o ckF[@sdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZ ddlZ ddlZ ddlZ ddlZddlZddlZddlmZmZmZmZddlmZddlmZmZeZddlm Z zddl!m"Z"Wn e#ye$Z"YnwgdZ%e &d Z'e &d Z(e &d Z)d d Z*  d]ddZ+d^ddZ,d^ddZ-d_ddZ.d^ddZ/Gddde0Z1ddZ2  !d`d"d#Z3  !dad$d%Z4  !dbd&d'Z5d(Z6d)Z7d*Z8e9Z:ed+d,Z;d^d-d.Zd3d4Z?dcd5d6Z@Gd7d8d8eAZBGd9d:d:ejCjDZEGd;d<dd>eAZG  ddd?d@ZHdAdBZIdCdDZJdEdFZKdedJdKZLdLdMZMdfdOdPZNdNdefdQdRZOdSdTZPdUdVZQGdWdXdXeAZRdYdZZSd[d\ZTdS)gz7 Some handy utility functions used by several classes. N)sixStringIOurllib encodebytes)contextmanager)md5sha512)json)JSONDecodeError)aclcorsdefaultObjectAcllocationlogging partNumberpolicyrequestPaymenttorrent versioning versionIdversionswebsiteuploadsuploadIdzresponse-content-typezresponse-content-languagezresponse-expireszresponse-cache-controlzresponse-content-dispositionzresponse-content-encodingdelete lifecycletaggingrestore storageClass websiteConfigcomposebilling userProjectencryptionConfigz(.)([A-Z][a-z]+)z([a-z])([0-9]+)z([a-z0-9])([A-Z])cCs(t|dkr|S|dtj|dfS)Nr)lenrparseunquote)nvr),/usr/lib/python3/dist-packages/boto/utils.py unquote_vks r+c Csz|stj}i}|D] }|}||dur+|dvs!||jr+t||||<q d|vr4d|d<d|vrsz$canonical_string..cSs g|] }|dtvrt|qS)r)qsa_of_interestr+r4r)r)r*r7 cSs|dSNrr))xr)r)r*sz"canonical_string..)keycSsg|]}d|qS)r2)joinr4r)r)r*r7s)botoprovider get_defaultlower startswith header_prefixstrstrip date_headersortedkeysr3r%sortr>) methodpathheadersexpiresr@interesting_headersr=lksorted_header_keysbufvaltqsar)r)r*canonical_stringrsL           rVcCs`|stj}|j}|}|D]}|tjjj j vr%||||<q|||||<q|SN) r?r@rAmetadata_prefixcopyrIrBs3r=Keybase_user_settable_fields)rMmetadatar@rX final_headerskr)r)r* merge_metas  r`c Cs|stj}|j}i}|D]4}||rDtj ||}t |t r7z| d}Wn t y6Ynw|||t|d<||=q|SNutf-8)r?r@rArXrIrBrCrr&r' isinstancebytesdecodeUnicodeDecodeErrorr%)rMr@rXr]hkeyrSr)r)r*get_aws_metadatas"    rhT c Cs8td|D]}z2tji}tj|}tj|}|j||d}|} t| t j s5t | dr5| d} | WStj jy_} z| } | dkrU|sUWYd} ~ dSWYd} ~ nd} ~ wtyv} z tjdWYd} ~ nd} ~ ww|d |krtjd ttd |tjd d dqtj ddS)a Retry a url. This is specifically used for accessing the metadata service on an instance. Since this address should never be proxied (for security reasons), we create a ProxyHandler with a NULL dictionary to override any proxy settings in the environment. r)timeoutrerbiNr/z&Caught exception reading instance datar$zSleeping before retryingBotomax_retry_delay<z'Unable to read instance data, giving up)rangerrequest ProxyHandler build_openerRequestopenreadrcr string_typeshasattrreerror HTTPErrorgetcode Exceptionr?log exceptiondebugtimesleepminconfigget) url retry_on_404 num_retriesrji proxy_handleropenerreqrresultecoder)r)r* retry_urls:           rcCs t|||SrW)LazyLoadMetadata)rrrjr)r)r*_get_instance_metadatas rcsheZdZdddZddZfddZddd Zfd d Zfd d ZfddZ fddZ Z S)rNc Cs||_||_i|_g|_||_tjj|j|j|jd}|r`|d}|D]=}| dr8|dd}|j |n#| d}|dkrR||dd}|d|d} n|}} | |j|<d||<q$dSdS) Nrrj /rr2r$z /openssh-key) _url _num_retries_leaves_dicts_timeoutr?utilsrr3endswithappendfind) selfrrrjdatafieldsfieldr=presourcer)r)r*__init__s*      zLazyLoadMetadata.__init__cCs|D]}||qdSrWr)rr=r)r)r* _materialize s zLazyLoadMetadata._materializec s||vr tt||Stt||}|dur|S||jvr|j|}d}td|jD]}z5tjj|j t j j |dd|j|j d}|rS|ddkrSt|}Wn|d}|dkra|d}Wnty}ztjd|jj|ftjd||}WYd}~nd}~wty}ztjd d |jj|}d}~wwtjd d |d |d |jkrttd|tjddd}t|q,tjdtjd|jj|f||||<n||jvrt|j |d|j||<tt||S)Nrz/:)safer{rzencountered '%s' exception: %szcorrupted JSON data found: %szencountered unretryablez '%s' exception, re-raisingz"Caught exception reading meta dataz for the '%s' tryr$rkrlrmrnz#Unable to read meta data, giving upr) superr __getitem__rrorr?rrrrr&quoterr loadsrr3r r|r~ __class____name__r{rxrrandomrrrrr) rr=rSrlast_exceptionrrr next_sleeprr)r*rs           zLazyLoadMetadata.__getitem__cCs"z||WSty|YSwrW)KeyError)rr=defaultr)r)r*rSs   zLazyLoadMetadata.getc|tt|SrW)rrrvaluesrrr)r*rYzLazyLoadMetadata.valuescrrW)rrritemsrrr)r*r]rzLazyLoadMetadata.itemscrrW)rrr__str__rrr)r*rarzLazyLoadMetadata.__str__crrW)rrr__repr__rrr)r*rerzLazyLoadMetadata.__repr__rW) r __module__ __qualname__rrrrrrrr __classcell__r)r)rr*rs   C   rcCsd|||fS)ax Builds an EC2 metadata URL for fetching information about an instance. Example: >>> _build_instance_metadata_url('http://169.254.169.254', 'latest', 'meta-data/') http://169.254.169.254/latest/meta-data/ :type url: string :param url: URL to metadata service, e.g. 'http://169.254.169.254' :type version: string :param version: Version of the metadata to get, e.g. 'latest' :type path: string :param path: Path of the metadata to get, e.g. 'meta-data/'. If a trailing slash is required it must be passed in with the path. :return: The full metadata URL z%s/%s/%sr))rversionrLr)r)r*_build_instance_metadata_urljsrlatesthttp://169.254.169.254 meta-data/cCsDzt|||}t|||dWStjjy!tjd|YdSw)a Returns the instance metadata as a nested Python dictionary. Simple values (e.g. local_hostname, hostname, etc.) will be stored as string values. Values such as ancestor-ami-ids will be stored in the dict as a list of string values. More complex fields such as public-keys and will be stored as nested dicts. If the timeout is specified, the connection to the specified url will time out after the specified number of seconds. rzBException caught when trying to retrieve instance metadata for: %sN)rrrrxURLErrorr?r|r})rrrrjr metadata_urlr)r)r*get_instance_metadatas rc Csi}t||d}z2t|||d}|d}|D] }t|d|d||d} | ddkr1t| } |r7| ||<q|WStjjyFYdSw)zF Returns the instance identity as a nested Python dictionary. zdynamic/instance-identity/rrrrrN)rrr3r rrrxr) rrrjriidbase_urlrrrrSr)r)r*get_instance_identitys$   rc Csbt||d}t|d||d}|r/|r/||}i}|D]}|d} | d|| d<q|S)Nz user-dataF)rrrjr2r$r)rrr3rF) rseprrjrud_url user_datalnvpairrTr)r)r*get_instance_userdatas   rz%Y-%m-%dT%H:%M:%SZz%Y-%m-%dT%H:%M:%S.%fZz%a, %d %b %Y %H:%M:%S %Zc cslt)ttj}zttj|VWttj|nttj|wWddS1s/wYdS)zE A context manager to set the locale in a threadsafe manner. N) LOCALE_LOCKlocale setlocaleLC_ALL)namesavedr)r)r*rs ""rcCs|st}tt|SrW)rgmtimestrftimeISO8601)tsr)r)r*get_tss rc CstdM|}ztj|t}|WWdStyPztj|t}|WYWdStyOtj|t}|YYWdSww1sTwYdS)NC)rrFdatetimestrptimer ValueError ISO8601_MSRFC1123)rdtr)r)r*parse_tss$     rcCsp|rd||f}|d}d}z"|ddD]}|r t||}qttd|dd|}q|WSYdS)Nz%s.%s.r$rr)r3getattr __import__r>) module_name class_namemodulescmr)r)r* find_classs   rcCs,d}|d7}tj|||||f}|S)z= Update your Dynamic DNS record with DNSMadeEasy.com z,https://www.dnsmadeeasy.com/servlet/updateipz$?username=%s&password=%s&id=%s&ip=%s)rrpurlopenru)usernamepassworddme_id ip_addressdme_urlsr)r)r* update_dmesrc Cstjd||durt}zb|dr;|tdddd\}}tj||d}| |}| |}| |n0|r^|r^t j } | d|||t j | } t j | } t j | t j |} || |dW|S)z Fetch a file based on the URI provided. If you do not pass in a file pointer a tempfile.NamedTemporaryFile, or None if the file could not be retrieved is returned. The URI can be either an HTTP url, or "s3://bucket_name/key_name" z Fetching %sNzs3://rr$)aws_access_key_idaws_secret_access_keyr)r?r|infotempfileNamedTemporaryFilerCr%r3 connect_s3 get_bucketget_keyget_contents_to_filerrpHTTPPasswordMgrWithDefaultRealm add_passwordHTTPBasicAuthHandlerrrinstall_openerrwriteruseekr}) urifilerr bucket_namekey_namerbucketr=passman authhandlerrrr)r)r* fetch_files0          r c@sTeZdZdddZdddZdd Zd d Zeeedd Zd dZ ee eddZ dS) ShellCommandTFNcCs0d|_||_t|_||_||_|j|ddS)Nr)cwd) exit_codecommandrlog_fpwait fail_fastrun)rrrrrr)r)r*r$s zShellCommand.__init__cCstjd|jtj|jdtjtjtj|d|_|jri|j durDt d|j }|j |d|j |d|j dus#tj|j |jj|_|jrf|jdkrftd|jd|j|jSdS)Nz running:%sT)shellstdinstdoutstderrrr$rzCommand z failed with status )r?r|rr subprocessPopenPIPEprocessrpollrr communicaterrgetvalue returncoderrr{)rrrTr)r)r*r,s0     zShellCommand.runcCstrW)AttributeErrorrvaluer)r)r* setReadOnlyBzShellCommand.setReadOnlycCs|jSrW)rrr)r)r* getStatusEszShellCommand.getStatuszThe exit code for the commandcCs |jSrW)rrrr)r)r* getOutputK zShellCommand.getOutputz*The STDIN and STDERR output of the command)TFNrW) rrrrrr$r&propertystatusr'outputr)r)r)r*r "s  r cs(eZdZdZfddZddZZS)AuthSMTPHandlera This class extends the SMTPHandler in the standard Python logging module to accept a username and password on the constructor and to then use those credentials to authenticate with the SMTP server. To use this, you could add something like this in your boto config file: [handler_hand07] class=boto.utils.AuthSMTPHandler level=WARN formatter=form07 args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') cs&tt|||||||_||_dS)z Initialize the handler. We have extended the constructor to accept a username/password for SMTP authentication. N)rr,rrr)rmailhostrrfromaddrtoaddrssubjectrr)r*r`s  zAuthSMTPHandler.__init__c Csz@|j}|s tj}t|j|}||j|j||}d|j d |j | |t j|f}||j |j ||WdSttfyJ||YdS)z Emit a record. Format the record and send it to the specified addressees. It would be really nice if I could add authorization to this class without having to resort to cut and paste inheritance but, no. z-From: %s To: %s Subject: %s Date: %s %s,N)mailportsmtplib SMTP_PORTSMTPr-loginrrformatr.r>r/ getSubjectemailr formatdatesendmailquitKeyboardInterrupt SystemExit handleError)rrecordportsmtpmsgr)r)r*emitms&   zAuthSMTPHandler.emit)rrr__doc__rrDrr)r)rr*r,Rs  r,c@speZdZdZGdddeZddZddZdd Zd d Z d d Z ddZ ddZ ddZ ddZddZdS)LRUCacheaqA dictionary-like object that stores only a certain number of items, and discards its least recently used item when full. >>> cache = LRUCache(3) >>> cache['A'] = 0 >>> cache['B'] = 1 >>> cache['C'] = 2 >>> len(cache) 3 >>> cache['A'] 0 Adding new items to the cache does not increase its size. Instead, the least recently used item is dropped: >>> cache['D'] = 3 >>> len(cache) 3 >>> 'B' in cache False Iterating over the cache returns the keys, starting with the most recently used: >>> for key in cache: ... print key D A C This code is based on the LRUCache class from Genshi which is based on `Myghty `_'s LRUCache from ``myghtyutils.util``, written by Mike Bayer and released under the MIT license (Genshi uses the BSD License). c@seZdZddZddZdS)zLRUCache._ItemcCsd|_|_||_||_dSrW)previousnextr=r#)rr=r#r)r)r*rs  zLRUCache._Item.__init__cC t|jSrW)reprr#rr)r)r*rr(zLRUCache._Item.__repr__N)rrrrrr)r)r)r*_Items rKcCst|_||_d|_d|_dSrW)dict_dictcapacityheadtail)rrNr)r)r*rs zLRUCache.__init__cCs ||jvSrW)rMrr)r)r* __contains__r(zLRUCache.__contains__ccs&|j}|r|jV|j}|sdSdSrW)rOr=rH)rcurr)r)r*__iter__s  zLRUCache.__iter__cCrIrW)r%rMrr)r)r*__len__r(zLRUCache.__len__cCs|j|}|||jSrW)rM _update_itemr#)rr=itemr)r)r*rs  zLRUCache.__getitem__cCsT|j|}|dur|||}||j|<||dS||_|||dSrW)rMrrK _insert_itemr#rU _manage_size)rr=r#rVr)r)r* __setitem__s     zLRUCache.__setitem__cCrIrW)rJrMrr)r)r*rr(zLRUCache.__repr__cCs:d|_|j|_|jdur||j_n||_||_|dSrW)rGrOrHrPrX)rrVr)r)r*rWs   zLRUCache._insert_itemcCs`t|j|jkr.|j|jj=|j|jkr|jj|_d|j_nd|_|_t|j|jksdSdSrW)r%rMrNrPr=rOrGrHrr)r)r*rXs     zLRUCache._manage_sizecCsV|j|krdS|j}|j|_|jdur||j_n||_d|_|j|_||j_|_dSrW)rOrGrHrP)rrVrGr)r)r*rUs   zLRUCache._update_itemN)rrrrEobjectrKrrQrSrTrrYrrWrXrUr)r)r)r*rFs%   rFc@s>eZdZdZeZd ddZddZddZd d Z d d Z dS)Passwordzp Password object that stores itself as hashed. Hash defaults to SHA512 if available, MD5 otherwise. NcCs||_|r ||_dSdS)zh Load the string from an initial value, this should be the raw hashed password. N)rEhashfunc)rrEr\r)r)r*rs zPassword.__init__cCs(t|ts |d}|||_dSra)rcrdencoder\ hexdigestrEr"r)r)r*sets  z Password.setcCs t|jSrWrErr)r)r*rr(zPassword.__str__cCs<|durdSt|ts|d}t||t|jkS)NFrb)rcrdr]rEr\r^)rotherr)r)r*__eq__s   zPassword.__eq__cCs|jrt|jSdSr:)rEr%rr)r)r*rTs zPassword.__len__NN) rrrrE_hashfnr\rr_rrbrTr)r)r)r*r[s   r[c Cs|pg}|rdtjdd|f}|stjddd}|rztjddd}tjj}||d<||d <||d <tjjd d |d <||d<|rQ| tjj ||rktjj dd}||tj|| ||D]}| |qmtjddd} tjddrt| ttjdd} nt| } tjddr| | | tjddd} tjddd} | r| | | | |||| WdStjdYdSdS)Nz[%s] %sInstancez instance-id Notificationsmtp_to smtp_fromr?FromzReply-ToToT) localtimeDateSubjecttexthtml smtp_host localhost smtp_portsmtp_tls smtp_userr/ smtp_passz notify failed)r?r get_valuer9mime multipart MIMEMultipartrr:attachrnMIMETextbaseMIMEBase set_payloadencoders encode_base64r3r5intgetboolehlostarttlsr6r; as_stringr<r|r}) r0body html_body to_string attachmentsappend_instance_id from_stringrCpartrpserverrtrur)r)r*notify&s`        rcCsDtjs t|tr |St|tjst|}t|tjr |d}|Sra)rPY2rcrdrv text_typer]r#r)r)r*get_utf8_value^s    rcCs*t|tst|trt|}|S|g}|SrW)rclisttuplerr)r)r*mklistks  rcCs(td|}td|}td|S)a~Convert camel case to a "pythonic" name. Examples:: pythonize_name('CamelCase') -> 'camel_case' pythonize_name('already_pythonized') -> 'already_pythonized' pythonize_name('HTTPRequest') -> 'http_request' pythonize_name('HTTPStatus200Ok') -> 'http_status_200_ok' pythonize_name('UPPER') -> 'upper' pythonize_name('') -> '' z\1_\2)_first_cap_regexsub_number_cap_regex_end_cap_regexrB)rs1s2r)r)r*pythonize_namets rF text/plain:cCstjj}|D]?\}}t||}|dd\}} |dkr'tjjj|| d} ntjj || } | |tj | | j dd|d|| q|} |rmt} tjd| d } z | | W| n| w| } | S) aDescription: :param content: A list of tuples of name-content pairs. This is used instead of a dict to ensure that scripts run in order :type list of tuples: :param compress: Use gzip to compress the scripts, defaults to no compression :type bool: :param deftype: The type that should be assumed if nothing else can be figured out :type str: :param delimiter: mime delimiter :type str: :return: Final mime multipart :rtype: str: rr$rn)_subtypezContent-Disposition attachment)filenamewb)modefileobj)r9rwrxryguess_mime_typer3rnr{r|r}r~rr add_headerrzrrgzipGzipFilercloser)contentcompressdeftype delimiterwrapperrcon definite_typemaintypesubtypemime_conrcontentrRgzr)r)r*write_mime_multiparts(       rcCs@ddddddd}|}|D]\}}||r|}|Sq|S)zDescription: Guess the mime type of a block of text :param content: content we're finding the type of :type str: :param deftype: Default mime type :type str: :rtype: : :return: ztext/x-include-urlztext/x-shellscriptztext/cloud-configztext/upstart-jobztext/part-handlerztext/cloud-boothook)z#includez#!z #cloud-configz #upstart-jobz #part-handlerz#cloud-boothook)rrC)rrstarts_with_mappingsrtype possible_typemimetyper)r)r*rs  r cCst|||tdS)a Compute MD5 hash on passed file and return results in a tuple of values. :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer will be reset to its current location before the method returns. :type buf_size: integer :param buf_size: Number of bytes per read request. :type size: int :param size: (optional) The Maximum number of bytes to read from the file pointer (fp). This is useful when uploading a file in multiple parts where the file is being split inplace into different parts. Less bytes may be available. :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element, the base64 encoded version of the plain digest as the second element and the data size as the third element. )hash_algorithm) compute_hashr)fpbuf_sizesizer)r)r* compute_md5src Cs|}|}|r||kr||}n||}|rIt|ts$|d}|||r6|t|8}|dkr6n|rB||krB||}n||}|s|}t|  d}|ddkrb|dd}||} | |||| fS)Nrbrrr) tellrurcrdr]updater%r^rdigestrer) rrrrhash_objsposr hex_digest base64_digest data_sizer)r)r*rs0                rcsfdd|DS)z Takes a specific header name and a dict of headers {"name": "value"}. Returns a list of matching header names, case-insensitive. cs g|] }|kr|qSr))rBr5hrr)r*r7r9z)find_matching_headers..r))rrMr)rr*find_matching_headerssrcs"t|}dfdd|DS)z Takes a specific header name and a dict of headers {"name": "value"}. Returns a string of all header values, comma-separated, that match the input header name, case-insensitive. r1c3s(|]}|durt|VqdSrWr`rrMr)r* s   z(merge_headers_by_name..)rr>)rrMmatching_headersr)rr*merge_headers_by_names rc@seZdZdZdddZdS) RequestHookz This can be extended and supplied to the connection object to gain access to request and response object after the request completes. One use for this would be to implement some specific request logging. FcCsdSrWr))rrpresponserxr)r)r*handle_request_data#r%zRequestHook.handle_request_dataN)F)rrrrErr)r)r)r*rsrcCs:|rt|ts dS|drdSt|ddkrdSdS)zQ Detect (naively) if the hostname is an IPV6 host. Return a boolean. F[Trrk)rcrErCr%r3hostnamer)r)r* host_is_ipv6's rcCs6|}t|r|ddddS|dddS)z Given a hostname that may have a port name, ensure that the port is trimmed returning only the host, including hostnames that are IPV6 and may include brackets. z]:r$rz[]r)rFrr3rr)r)r* parse_host>srrcrW)TriN)rrrNr)rrNr)rNrNr)NNN)NNNNT)Frr)rN)UrErrlogging.handlersrr? boto.providerrrr3rreemail.mime.multipartr9email.mime.baseemail.mime.text email.utilsemail.encodersr threadingr boto.compatrrrr contextlibrhashlibrrrdr boto.compat.jsonr ImportErrorrr8compilerrrr+rVr`rhrrrLrrrrrrrrLockrrrrrrr rZr handlers SMTPHandlerr,rFr[rrrrrrrrrrrrrr)r)r)r*s%       :   #v        #07w& 8  -