o .&aJr @sddlZddlZddlmZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZmZddlmZddlmZddlmZeeZd Zed d d ed Zd ZdddddddddZedZedZedZedZ ddZ!ddZ"Gdddej#Z$Gddde%Z&Gdd d ej'Z(d!d"Z)d#d$Z*d%d&Z+d'd(Z,dYd)d*Z-dZd,d-Z.Gd.d/d/e/Z0d0d1Z1ej2j3fd2d3Z4d4d5Z5Gd6d7d7e%Z6d8d9Z7Gd:d;d;e/Z8GdZ9e9Z:Gd?d@d@e/Z;GdAdBdBeZGdGdHdHe>Z?GdIdJdJe?Z@GdKdLdLe>ZAGdMdNdNeZBGdOdPdPeBZCGdQdRdReBZDGdSdTdTe=ZEGdUdVdVeZFGdWdXdXe/ZGdS)[N)datetime) namedtupledeque)parse)tzlocaltzutc)BaseSubscriber) bytes_print)queue)KiBMiBGiBTiBPiBEiBi)tzinfolii@l)kbmbgbtbkibmibgibtibzT^(?Parn:(aws).*:s3:[a-z\-0-9]*:[0-9]{12}:accesspoint[:/][^/]+)/?(?P.*)$z^(?Parn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:][a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P.*)$z^(?Parn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:][a-zA-Z0-9\-]{1,63}[/:]bucket[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P.*)$zs^(?Parn:(aws).*:s3-object-lambda:[a-z\-0-9]+:[0-9]{12}:accesspoint[/:][a-zA-Z0-9\-]{1,63})[/:]?(?P.*)$cCsvd}t|}|dkr dS||krd|SttD] \}}||d}t||||kr8d||||fSqdS)aConvert a size in bytes into a human readable format. For example:: >>> human_readable_size(1) '1 Byte' >>> human_readable_size(10) '10 Bytes' >>> human_readable_size(1024) '1.0 KiB' >>> human_readable_size(1024 * 1024) '1.0 MiB' :param value: The size in bytes. :return: The size in a human readable format based on base-2 units. rrz1 Bytez%d Bytesz%.1f %sN)float enumerateHUMANIZE_SUFFIXESround)valuebase bytes_intisuffixunitr'@/usr/lib/python3/dist-packages/awscli/customizations/s3/utils.pyhuman_readable_sizeCs r)cCs|}|dddkr|dd}n|dd}t|dko&|tv}|ssz0StablePriorityQueue.__init__..r)r Queue__init__range prioritiesdefault_priority)r;rM max_priorityr'r'r(rQs zStablePriorityQueue.__init__cCs d}|jD]}|t|7}q|Sr6)rSr.)r;sizebucketr'r'r(_qsizes zStablePriorityQueue._qsizecCs*tt|d|j|j}|j||dS)NPRIORITY)minr7rTrSr9)r;itempriorityr'r'r(_putszStablePriorityQueue._putcCs |jD] }|sq|SdSrB)rSpopleft)r;rWr'r'r(_gets  zStablePriorityQueue._getN)rrL)rCrDrErFrQrXr]r_r'r'r'r(rKs   rKcCs(t|r tdt|rtddS)NzRs3 commands do not support S3 Object Lambda resources. Use s3api commands instead.zOs3 commands do not support Outpost Bucket ARNs. Use s3control commands instead.)%_S3_OBJECT_LAMBDA_TO_BUCKET_KEY_REGEXmatchr1*_S3_OUTPOST_BUCKET_ARN_TO_BUCKET_KEY_REGEXs3_pathr'r'r(block_unsupported_resourcess  recCst|t|}|r|d|dfSt|}|r&|d|dfS|dd}|d}d}t|dkr<|d}||fS)z This is a helper function that given an s3 path such that the path is of the form: bucket/key It will return the bucket and the key represented by the s3 path rWkey/rr)re#_S3_ACCESSPOINT_TO_BUCKET_KEY_REGEXragroup_S3_OUTPOST_TO_BUCKET_KEY_REGEXsplitr.)rdra s3_componentsrWs3_keyr'r'r(find_bucket_keys    rocCs|dr |dd}t|S)zSplit s3 path into bucket and key prefix. This will also handle the s3:// prefix. :return: Tuple of ('bucketname', 'keyname') zs3://N) startswithrorcr'r'r(split_s3_bucket_keys  rrc Cstzt|}Wnty}ztd||fd}~wwz t|jt}Wnttt fy4d}Ynw|j |fS)z This is a helper function that given a local path return the size of the file in bytes and time of last modification. z(Could not retrieve file stat of "%s": %sN) osstatIOErrorr1r fromtimestampst_mtimerOSError OverflowErrorst_size)pathstatse update_timer'r'r( get_file_stats rc Cs|d}|d}|d}|d}|dur|d}dtjd}|dr-|t|dd}n |||d }|||d}|d rV|d} | |||||7} | |fS|d} | |fS) z This is a helper function that determines the destination path and compare key given parameters received from the ``FileFormat`` class. srcr8typeNr{rg)s3localdir_op use_src_name)rssepr.rlreplace) filessrc_pathrr8src_type dest_type sep_tablerel_path compare_key dest_pathr'r'r(find_dest_path_comp_keys&  rTcCs2d}|r |d|d}||}t|ddd}|S)zN This creates a ``PrintTask`` for whenever a warning is to be thrown. z warning: zSkipping file z. FT)messageerrorwarning) WarningResult)r{ error_message skip_file print_stringwarning_messager'r'r(create_warning0src@s"eZdZdZdddZddZdS)StdoutBytesWriterzd This class acts as a file-like object that performs the bytes_print function on write. NcC ||_dSrB)_stdout)r;stdoutr'r'r(rQB zStdoutBytesWriter.__init__cCst||jdS)zR Writes data to stdout as bytes. :param b: data to write N)r r)r;br'r'r(writeEszStdoutBytesWriter.writerB)rCrDrErFrQrr'r'r'r(r=s  rcCs6zt|dWStytjd|ddYdSw)zpGiven a filename, guess it's content type. If the type cannot be guessed, a value of None is returned. rz?Unable to guess content type for %s due to UnicodeDecodeError: T)exc_infoN) mimetypes guess_typeUnicodeDecodeErrorLOGGERdebug)filenamer'r'r(guess_content_typeNs  rcCsNztj|\}}tj||}tj||WSty&tj|YSw)zCross platform relative path of a filename. If no relative path can be calculated (i.e different drives on Windows), then instead of raising a ValueError, the absolute path is returned. )rsr{rlrelpathjoinr1abspath)rstartdirnamebasename relative_dirr'r'r( relative_pathds rc CsHz t|||fWdSty#}z |jtjkr|tdd}~ww)aB Set the utime of a file, and if it fails, raise a more explicit error. :param filename: the file to modify :param desired_time: the epoch timestamp to set for atime and mtime. :raises: SetFileUtimeError: if you do not have permission (errno 1) :raises: OSError: for all errors other than errno 1 zrThe file was downloaded, but attempting to modify the utime of the file failed. Is the file owned by another user?N)rsutimerxerrnoEPERMSetFileUtimeError)r desired_timer}r'r'r(set_file_utimets  rc@rG)rNrIr'r'r'r(rrJrcCst|tSrB)r astimezoner) date_stringr'r'r( _date_parsersrc@s*eZdZdZefddZ  dddZdS) BucketListerzList keys in a bucket.cC||_||_dSrB)_clientr)r;client date_parserr'r'r(rQ zBucketLister.__init__Nc cs|d|id}|dur||d<|dur|||jd}|jd i|}|D]#}|dg} | D]} |d| d} || d| d<| | fVq3q)dS) NPageSize)BucketPaginationConfigPrefixlist_objects_v2ContentsrgKey LastModifiedr')updater get_paginatorpaginategetr) r;rWprefix page_size extra_argskwargs paginatorpagespagecontentscontent source_pathr'r'r( list_objectss$    zBucketLister.list_objects)NNN)rCrDrErFrrQrr'r'r'r(rs  rcseZdZdfdd ZZS) PrintTaskFNcstt||||||S)aN :param message: An arbitrary string associated with the entry. This can be used to communicate the result of the task. :param error: Boolean indicating a failure. :param total_parts: The total number of parts for multipart transfers. :param warning: Boolean indicating a warning )superr__new__)clsrr total_partsr __class__r'r(rszPrintTask.__new__)FNN)rCrDrEr __classcell__r'r'rr(rsr)rrrrc@seZdZdZeddZeddZeddZedd Zed d Z ed d Z eddZ eddZ eddZ eddZeddZeddZeddZeddZeddZed d!Zed"d#Zed$d%Zed&d'Zed(d)Zd*S)+RequestParamsMapperaA utility class that maps CLI params to request params Each method in the class maps to a particular operation and will set the request parameters depending on the operation and CLI parameters provided. For each of the class's methods the parameters are as follows: :type request_params: dict :param request_params: A dictionary to be filled out with the appropriate parameters for the specified client operation using the current CLI parameters :type cli_params: dict :param cli_params: A dictionary of the current CLI params that will be used to generate the request parameters for the specified operation For example, take the mapping of request parameters for PutObject:: >>> cli_request_params = {'sse': 'AES256', 'storage_class': 'GLACIER'} >>> request_params = {} >>> RequestParamsMapper.map_put_object_params( request_params, cli_request_params) >>> print(request_params) {'StorageClass': 'GLACIER', 'ServerSideEncryption': 'AES256'} Note that existing parameters in ``request_params`` will be overridden if a parameter in ``cli_params`` maps to the existing parameter. cC@|||||||||||||||dS)z*Map CLI params to PutObject request paramsN)_set_general_object_params_set_metadata_params_set_sse_request_params_set_sse_c_request_params_set_request_payer_paramrrequest_params cli_paramsr'r'r(map_put_object_params    z)RequestParamsMapper.map_put_object_paramscC||||||dS)z*Map CLI params to GetObject request paramsNrrrr'r'r(map_get_object_params z)RequestParamsMapper.map_get_object_paramscCsV||||||||||||||||||||dS)z+Map CLI params to CopyObject request paramsN)r_set_metadata_directive_paramr!_auto_populate_metadata_directiver)_set_sse_c_and_copy_source_request_paramsrrr'r'r(map_copy_object_paramss     z*RequestParamsMapper.map_copy_object_paramscCr)z+Map CLI params to HeadObject request paramsNrrr'r'r(map_head_object_paramsrz*RequestParamsMapper.map_head_object_paramscCr)z6Map CLI params to CreateMultipartUpload request paramsN)rrrrrrr'r'r("map_create_multipart_upload_paramsrz6RequestParamsMapper.map_create_multipart_upload_paramscCr)z+Map CLI params to UploadPart request paramsNrrr'r'r(map_upload_part_paramsrz*RequestParamsMapper.map_upload_part_paramscCr)z/Map CLI params to UploadPartCopy request paramsN)rrrr'r'r(map_upload_part_copy_paramssz/RequestParamsMapper.map_upload_part_copy_paramscC|||dSrBrrr'r'r(map_delete_object_params z,RequestParamsMapper.map_delete_object_paramscCrrBrrr'r'r(map_list_objects_v2_paramsrz.RequestParamsMapper.map_list_objects_v2_paramscC|dr |d|d<dSdS)N request_payer RequestPayerrrr'r'r(r z,RequestParamsMapper._set_request_payer_paramc CsPddddddddd d }|D]}||r||}||||<q|||dS) NACL StorageClassWebsiteRedirectLocation ContentType CacheControlContentDispositionContentEncodingContentLanguageExpires) acl storage_classwebsite_redirect content_type cache_controlcontent_dispositioncontent_encodingcontent_languageexpires)r_set_grant_params)rrrgeneral_param_translationcli_param_namerequest_param_namer'r'r(rs   z.RequestParamsMapper._set_general_object_paramsc CsZ|dr)|dD]!}z |dd\}}Wn ty tdw||||<q dSdS)Ngrants=rz1grants should be of the form permission=principal)rrlr1_permission_to_param)rrrgrant permissiongranteer'r'r(r/s   z%RequestParamsMapper._set_grant_paramscCs8|dkrdS|dkr dS|dkrdS|dkrdStd ) Nread GrantReadfullGrantFullControlreadacl GrantReadACPwriteacl GrantWriteACPz5permission must be one of: read|readacl|writeacl|full)r1)rrr'r'r(r:sz(RequestParamsMapper._permission_to_paramcCr)NmetadataMetadatarrr'r'r(rGrz(RequestParamsMapper._set_metadata_paramscCs(|dr|dsd|d<dSdSdS)Nr'MetadataDirectiveREPLACEr)rrr'r'r(rLs  z5RequestParamsMapper._auto_populate_metadata_directivecCr)Nmetadata_directiver(rrr'r'r(rRs  z1RequestParamsMapper._set_metadata_directive_paramcCs4|dr |d|d<|dr|d|d<dSdS)NsseServerSideEncryptionsse_kms_key_id SSEKMSKeyIdrrr'r'r(rXs   z+RequestParamsMapper._set_sse_request_paramscC*|dr|d|d<|d|d<dSdS)Nsse_cSSECustomerAlgorithm sse_c_keySSECustomerKeyrrr'r'r(r_s  z-RequestParamsMapper._set_sse_c_request_paramscCr/)Nsse_c_copy_sourceCopySourceSSECustomerAlgorithmsse_c_copy_source_keyCopySourceSSECustomerKeyrrr'r'r(%_set_sse_c_copy_source_request_paramses  z9RequestParamsMapper._set_sse_c_copy_source_request_paramscCs||||||dSrB)rr8rr'r'r(rmrz=RequestParamsMapper._set_sse_c_and_copy_source_request_paramsN)rCrDrErF classmethodrrrrrrrrrrrrrrrrrrr8rr'r'r'r(rsT                   rc@ eZdZdZddZddZdS)ProvideSizeSubscriberzK A subscriber which provides the transfer size before it's queued. cCrrB)rV)r;rVr'r'r(rQxrzProvideSizeSubscriber.__init__cKs|j|jdSrB)metaprovide_transfer_sizerV)r;futurerr'r'r( on_queued{szProvideSizeSubscriber.on_queuedN)rCrDrErFrQr?r'r'r'r(r;ts r;c@s(eZdZdZddZddZddZdS) OnDoneFilteredSubscriberzSubscriber that differentiates between successes and failures It is really a convenience class so developers do not have to have to constantly remember to have a general try/except around future.result() c Ks\d}z|Wnty}z|}WYd}~nd}~ww|r'|||dS||dSrB)result Exception _on_failure _on_success)r;r>rfuture_exceptionr}r'r'r(on_dones z OnDoneFilteredSubscriber.on_donecCdSrBr'r;r>r'r'r(rDz$OnDoneFilteredSubscriber._on_successcCrGrBr'r;r>r}r'r'r(rCrIz$OnDoneFilteredSubscriber._on_failureN)rCrDrErFrFrDrCr'r'r'r(r@s  r@c@r:)DeleteSourceSubscriberz6A subscriber which deletes the source of the transfer.c CsBz||WdSty }z ||WYd}~dSd}~wwrB)_delete_sourcerB set_exceptionrJr'r'r(rDs z"DeleteSourceSubscriber._on_successcCtd)Nz_delete_source()NotImplementedErrorrHr'r'r(rLz%DeleteSourceSubscriber._delete_sourceN)rCrDrErFrDrLr'r'r'r(rKs rKc@s0eZdZdZddZddZddZdd Zd S) DeleteSourceObjectSubscriberz%A subscriber which deletes an object.cCrrB)r)r;rr'r'r(rQrz%DeleteSourceObjectSubscriber.__init__cC|jSrB)rWr; call_argsr'r'r( _get_bucketz(DeleteSourceObjectSubscriber._get_bucketcCrSrB)rfrTr'r'r(_get_keyrWz%DeleteSourceObjectSubscriber._get_keycCsN|jj}||||d}|jdr|jd|d<|jjdi|dS)N)rrrr')r<rUrVrXrrr delete_object)r;r>rUdelete_object_kwargsr'r'r(rLs z+DeleteSourceObjectSubscriber._delete_sourceN)rCrDrErFrQrVrXrLr'r'r'r(rRs  rRc@r:) DeleteCopySourceObjectSubscriberz+A subscriber which deletes the copy source.cC |jdS)Nr copy_sourcerTr'r'r(rVrz,DeleteCopySourceObjectSubscriber._get_bucketcCr\Nrr]rTr'r'r(rXrz)DeleteCopySourceObjectSubscriber._get_keyN)rCrDrErFrVrXr'r'r'r(r[s r[c@eZdZdZddZdS)DeleteSourceFileSubscriberz"A subscriber which deletes a file.cCst|jjjdSrB)rsremover<rUfileobjrHr'r'r(rLsz)DeleteSourceFileSubscriber._delete_sourceN)rCrDrErFrLr'r'r'r(ra rac@r:) BaseProvideContentTypeSubscriberz@A subscriber that provides content type when creating s3 objectscKs,t||}|dur||jjjd<dSdS)Nr)r _get_filenamer<rUr)r;r>r guessed_typer'r'r(r?sz*BaseProvideContentTypeSubscriber.on_queuedcCrN)Nz_get_filename()rOrHr'r'r(rfrQz.BaseProvideContentTypeSubscriber._get_filenameN)rCrDrErFr?rfr'r'r'r(res rec@eZdZddZdS)"ProvideUploadContentTypeSubscribercCs |jjjSrB)r<rUrcrHr'r'r(rfrz0ProvideUploadContentTypeSubscriber._get_filenameNrCrDrErfr'r'r'r(ri ric@rh) ProvideCopyContentTypeSubscribercCs|jjjdSr_)r<rUr^rHr'r'r(rfsz.ProvideCopyContentTypeSubscriber._get_filenameNrjr'r'r'r(rlrkrlc@r:)!ProvideLastModifiedTimeSubscriberz Sets utime for a downloaded filecCrrB)_last_modified_time _result_queue)r;last_modified_time result_queuer'r'r(rQrz*ProvideLastModifiedTimeSubscriber.__init__c Ksx|jjj}z|j}t|}t|t|WdSt y;}zd||f}|j t ||WYd}~dSd}~ww)NzNSuccessfully Downloaded %s but was unable to update the last modified time. %s) r<rUrcrn timetupletimemktimerr0rBroputr)r;r>rrlast_update_tuple mod_timestampr}rr'r'r(rDs    z-ProvideLastModifiedTimeSubscriber._on_successN)rCrDrErFrQrDr'r'r'r(rms rmc@r`)DirectoryCreatorSubscriberz4Creates a directory to download if it does not existc Ksxtj|jjj}ztj|st|WdSWdSty;}z|j t j ks0t d||fWYd}~dSd}~ww)Nz!Could not create directory %s: %s) rsr{rr<rUrcexistsmakedirsrxrEEXISTrH)r;r>rdr}r'r'r(r?s   z$DirectoryCreatorSubscriber.on_queuedN)rCrDrErFr?r'r'r'r(rxrdrxc@s"eZdZdZddZdddZdS)NonSeekableStreamaWrap a file like object as a non seekable stream. This class is used to wrap an existing file like object such that it only has a ``.read()`` method. There are some file like objects that aren't truly seekable but appear to be. For example, on windows, sys.stdin has a ``seek()`` method, and calling ``seek(0)`` even appears to work. However, subsequent ``.read()`` calls will just return an empty string. Consumers of these file like object have no way of knowing if these files are truly seekable or not, so this class can be used to force non-seekable behavior when you know for certain that a fileobj is non seekable. cCrrB)_fileobj)r;rcr'r'r(rQrzNonSeekableStream.__init__NcCs|dur |jS|j|SrB)r~r)r;amtr'r'r(rs  zNonSeekableStream.readrB)rCrDrErFrQrr'r'r'r(r}sr}rB)T)Hargparseloggingrrrrsrers collectionsrrdateutil.parserr dateutil.tzrrs3transfer.subscribersr awscli.compatr r getLoggerrCrr EPOCH_TIMEMAX_UPLOAD_SIZEr/compilerirkrbr`r)r4Actionr5rBrHrPrKrerorrrrrobjectrrr{curdirrrrrrrrrr;r@rKrRr[rarerirlrmrxr}r'r'r'r(s         ,    <