o .&a[@s:ddlZddlZddlmZddlmZmZmZmZm Z m Z ddl m Z ddl mZddl mZddl mZddl mZdd l mZdd l mZdd l mZdd l mZdd l mZddl mZddl mZddl mZddl mZddl mZddl mZddl mZddlmZddlmZddlm Z ddlm!Z!ddlm"Z"ddlm#Z#ddlm$Z$ddlm%Z%ddlm&Z&ddlm'Z'ddl(m)Z)e*e+Z,Gd d!d!e-Z.Gd"d#d#e-Z/Gd$d%d%e-Z0Gd&d'd'e0Z1Gd(d)d)e0Z2Gd*d+d+e0Z3Gd,d-d-e1Z4Gd.d/d/e2Z5Gd0d1d1e0Z6Gd2d3d3e0Z7dS)4N)TransferManager)human_readable_sizeMAX_UPLOAD_SIZEfind_bucket_key relative_pathcreate_warningNonSeekableStream)*create_transfer_config_from_runtime_config)UploadResultSubscriber)DownloadResultSubscriber)CopyResultSubscriber)UploadStreamResultSubscriber)DownloadStreamResultSubscriber)DeleteResultSubscriber) QueuedResult) SuccessResult) FailureResult) DryRunResult)ResultRecorder) ResultPrinter)OnlyShowErrorsResultPrinter)NoProgressResultPrinter)ResultProcessor)CommandResultRecorder)RequestParamsMapperStdoutBytesWriter)ProvideSizeSubscriber)"ProvideUploadContentTypeSubscriber) ProvideCopyContentTypeSubscriber)!ProvideLastModifiedTimeSubscriber)DirectoryCreatorSubscriber)DeleteSourceFileSubscriber)DeleteSourceObjectSubscriber) DeleteCopySourceObjectSubscriber)get_binary_stdinc@s(eZdZdZddZddZddZdS) S3TransferHandlerFactorycCs||_||_dS)aFactory for S3TransferHandlers :type cli_params: dict :param cli_params: The parameters provide to the CLI command :type runtime_config: RuntimeConfig :param runtime_config: The runtime config for the CLI command being run N) _cli_params_runtime_config)self cli_paramsruntime_configr-D/usr/lib/python3/dist-packages/awscli/customizations/s3/s3handler.py__init__:s z!S3TransferHandlerFactory.__init__c Csrt|j}|j|_|j|_t||}td|j|j t }|g}| ||t ||}t |||}t||j|S)agCreates a S3TransferHandler instance :type client: botocore.client.Client :param client: The client to power the S3TransferHandler :type result_queue: queue.Queue :param result_queue: The result queue to be used to process results for the S3TransferHandler :returns: A S3TransferHandler instance z7Using a multipart threshold of %s and a part size of %s)r r)MAX_IN_MEMORY_CHUNKSmax_in_memory_upload_chunksmax_in_memory_download_chunksrLOGGERdebugmultipart_thresholdmultipart_chunksizer_add_result_printerrrS3TransferHandlerr() r*client result_queuetransfer_configtransfer_managerresult_recorderresult_processor_handlersresult_processorcommand_result_recorderr-r-r.__call__Gs0   z!S3TransferHandlerFactory.__call__cCsh|jdrdS|jdrt|}n|jdrt|}n|jds)t|}nt|}||dS)Nquietonly_show_errors is_streamprogress)r(getrrrappend)r*r=r>result_printerr-r-r.r7ks       z,S3TransferHandlerFactory._add_result_printerN)__name__ __module__ __qualname__r0r/rAr7r-r-r-r.r&7s  $r&c@seZdZddZddZdS)r8cCsP||_||_|j|jj|f}t|t|t|t|t|t|t |g|_ dS)aBackend for performing S3 transfers :type transfer_manager: s3transfer.manager.TransferManager :param transfer_manager: Transfer manager to use for transfers :type cli_params: dict :param cli_params: The parameters passed to the CLI command in the form of a dictionary :type result_command_recorder: ResultCommandRecorder :param result_command_recorder: The result command recorder to be used to get the final result of the transfer N) _transfer_manager_result_command_recorderr:UploadStreamRequestSubmitterDownloadStreamRequestSubmitterUploadRequestSubmitterDownloadRequestSubmitterCopyRequestSubmitterDeleteRequestSubmitterLocalDeleteRequestSubmitter _submitters)r*r<r+result_command_recordersubmitter_argsr-r-r.r/zs  zS3TransferHandler.__init__c Cs|j>|j+d}|D]}|jD]}||r#||r!|d7}nqq |j|Wdn1s5wYWdn1sDwY|jS)aProcess iterable of FileInfos for transfer :type fileinfos: iterable of FileInfos param fileinfos: Set of FileInfos to submit to underlying transfer request submitters to make transfer API calls to S3 :rtype: CommandResult :returns: The result of the command that specifies the number of failures and warnings encountered. rN)rMrLrU can_submitsubmitnotify_total_submissionsget_command_result)r* fileinfostotal_submissionsfileinfo submitterr-r-r.calls$     zS3TransferHandler.callN)rIrJrKr/rar-r-r-r.r8ys %r8c@seZdZdZdZddZddZddZdd Zd d Z d d Z ddZ ddZ ddZ ddZddZddZddZddZddZdS) BaseTransferRequestSubmitterNcCs||_||_||_dS)a\Submits transfer requests to the TransferManager Given a FileInfo object and provided CLI parameters, it will add the necessary extra arguments and subscribers in making a call to the TransferManager. :type transfer_manager: s3transfer.manager.TransferManager :param transfer_manager: The underlying transfer manager :type result_queue: queue.Queue :param result_queue: The result queue to use :type cli_params: dict :param cli_params: The associated CLI parameters passed in to the command as a dictionary. N)rL _result_queuer()r*r<r:r+r-r-r.r/s z%BaseTransferRequestSubmitter.__init__cCs||}|s ||SdS)aSubmits a transfer request based on the FileInfo provided There is no guarantee that the transfer request will be made on behalf of the fileinfo as a fileinfo may be skipped based on circumstances in which the transfer is not possible. :type fileinfo: awscli.customizations.s3.fileinfo.FileInfo :param fileinfo: The FileInfo to be used to submit a transfer request to the underlying transfer manager. :rtype: s3transfer.futures.TransferFuture :returns: A TransferFuture representing the transfer if it the transfer was submitted. If it was not submitted nothing is returned. N)_warn_and_signal_if_skip _do_submit)r*r_ should_skipr-r-r.rZs  z#BaseTransferRequestSubmitter.submitcCtd)aChecks whether it can submit a particular FileInfo :type fileinfo: awscli.customizations.s3.fileinfo.FileInfo :param fileinfo: The FileInfo to check if the transfer request submitter can handle. :returns: True if it can use the provided FileInfo to make a transfer request to the underlying transfer manager. False, otherwise. z can_submit()NotImplementedErrorr*r_r-r-r.rYs z'BaseTransferRequestSubmitter.can_submitcCsi}|jr |||jg}||||jr2d|ji}|jddr'd|d<||jdi||jds?||||S||dS)Nr:is_moveFmove transfer_typedryrunr-) REQUEST_MAPPER_METHODr(_add_additional_subscribersRESULT_SUBSCRIBER_CLASSrcrFrG_submit_transfer_request_submit_dryrun)r*r_ extra_args subscribers result_kwargsr-r-r.res   z'BaseTransferRequestSubmitter._do_submitcCs@|j}|jddr d}||\}}|jt|||ddS)NrkFrlrmsrcdest)operation_namer(rF_format_src_destrcputr)r*r_rmrxryr-r-r.rss z+BaseTransferRequestSubmitter._submit_dryruncCdSNr-r*rur_r-r-r.rpz8BaseTransferRequestSubmitter._add_additional_subscriberscCrg)Nz_submit_transfer_request()rh)r*r_rtrur-r-r.rrz5BaseTransferRequestSubmitter._submit_transfer_requestcCs |D] }||r dSqdS)NT)_get_warning_handlers)r*r_warning_handlerr-r-r.rds z5BaseTransferRequestSubmitter._warn_and_signal_if_skipcCsgSr~r-r*r-r-r.rsz2BaseTransferRequestSubmitter._get_warning_handlerscCs|jdo |jd S)Nguess_mime_type content_type)r(rFrr-r-r._should_inject_content_type$s  z8BaseTransferRequestSubmitter._should_inject_content_typecCsf|jds1|s1td|j|jf|jds/td|jd|j|jf}|j |dSdS)Nforce_glacier_transferz@Encountered glacier object s3://%s. Not performing %s on object.ignore_glacier_warningss3://zObject is of storage class GLACIER. Unable to perform %s operations on GLACIER objects. You must restore the object to be able to perform the operation. See aws s3 %s help for additional parameter options to ignore or force these transfers.TF) r(rFis_glacier_compatibler3r4rxrzrrcr|)r*r_warningr-r-r. _warn_glacier*s"     z*BaseTransferRequestSubmitter._warn_glaciercCsDdtjj}tj|j|}|r t|jd}|j|dSdS)Nz..z#File references a parent directory.TF) ospathsepnormpath compare_key startswithrrcr|)r*r_ parent_prefix escapes_cwdrr-r-r._warn_parent_reference?s  z3BaseTransferRequestSubmitter._warn_parent_referencecCrg)zAReturns formatted versions of a fileinfos source and destination.r{rhrjr-r-r.r{Lsz-BaseTransferRequestSubmitter._format_src_destcCst|Sr~)rr*rr-r-r._format_local_pathPrz/BaseTransferRequestSubmitter._format_local_pathcCs|dr|Sd|S)Nr)rrr-r-r._format_s3_pathSs z,BaseTransferRequestSubmitter._format_s3_path)rIrJrKrorqr/rZrYrersrprrrdrrrrr{rrr-r-r-r.rbs$    rbc@sNeZdZejZeZddZddZ ddZ ddZ d d Z d d Z d dZdS)rPcC |jdkS)Nuploadrzrjr-r-r.rY] z!UploadRequestSubmitter.can_submitcCsF|t|j|r|t|jddr!|tdSdSNrkF)rGrsizerrr(rFr"rr-r-r.rp`s  z2UploadRequestSubmitter._add_additional_subscriberscC.t|j\}}||}|jj|||||dSN)fileobjbucketkeyrtru)rry _get_fileinrLr)r*r_rtrurrfileinr-r-r.rrg  z/UploadRequestSubmitter._submit_transfer_requestcC|jSr~)rxrjr-r-r.roz"UploadRequestSubmitter._get_fileincC|jgSr~)_warn_if_too_largerr-r-r.rrrz,UploadRequestSubmitter._get_warning_handlerscCsTt|dr&|jtkr(t|j}d|ttf}t||dd}|j|dSdSdS)Nrz&File %s exceeds s3 upload limit of %s.F) skip_file) getattrrrrrxrrrcr|)r*r_ file_pathwarning_messagerr-r-r.rus z)UploadRequestSubmitter._warn_if_too_largecC ||j}||j}||fSr~)rrxrryr*r_rxryr-r-r.r{  z'UploadRequestSubmitter._format_src_destN)rIrJrKrmap_put_object_paramsror rqrYrprrrrrr{r-r-r-r.rPYs rPc@sFeZdZejZeZddZddZ ddZ ddZ d d Z d d Z d S)rQcCr)Ndownloadrrjr-r-r.rYrz#DownloadRequestSubmitter.can_submitcCsV|t|j|t|t|j|j|jddr)|t |j dSdSr) rGrrr!r last_updatercr(rFr# source_clientrr-r-r.rps  z4DownloadRequestSubmitter._add_additional_subscriberscCrr)rrx _get_fileoutrLr)r*r_rtrurrfileoutr-r-r.rrrz1DownloadRequestSubmitter._submit_transfer_requestcCrr~)ryrjr-r-r.rrz%DownloadRequestSubmitter._get_fileoutcCs |j|jgSr~)rrrr-r-r.rs z.DownloadRequestSubmitter._get_warning_handlerscCrr~)rrxrryrr-r-r.r{rz)DownloadRequestSubmitter._format_src_destN)rIrJrKrmap_get_object_paramsror rqrYrprrrrr{r-r-r-r.rQs  rQc@s>eZdZejZeZddZddZ ddZ ddZ d d Z d S) rRcCr)Ncopyrrjr-r-r.rYrzCopyRequestSubmitter.can_submitcCsJ|t|j|r|t|jddr#|t|jdSdSr) rGrrrrr(rFr$rrr-r-r.rps  z0CopyRequestSubmitter._add_additional_subscribersc Cs@t|j\}}t|j\}}||d}|jj||||||jdS)N)BucketKey)rr copy_sourcertrur)rryrxrLrr) r*r_rtrurr source_bucket source_keyrr-r-r.rrs z-CopyRequestSubmitter._submit_transfer_requestcCrr~)rrr-r-r.rrz*CopyRequestSubmitter._get_warning_handlerscCs ||j}||j}||fSr~)rrxryrr-r-r.r{rz%CopyRequestSubmitter._format_src_destN) rIrJrKrmap_copy_object_paramsror rqrYrprrrr{r-r-r-r.rRs rRc@0eZdZeZddZddZddZddZd S) rNcC|jdko |jdS)NrrDrzr(rFrjr-r-r.rY  z'UploadStreamRequestSubmitter.can_submitcCs0|jdd}|dur|tt|dSdS)N expected_size)r(rFrGrint)r*rur_rr-r-r.rpsz8UploadStreamRequestSubmitter._add_additional_subscriberscCst}t|Sr~)r%r)r*r_ binary_stdinr-r-r.rsz(UploadStreamRequestSubmitter._get_fileincCdSN-r-rr-r-r.rrz/UploadStreamRequestSubmitter._format_local_pathN) rIrJrKr rqrYrprrr-r-r-r.rNs  rNc@r) rOcCr)NrrDrrjr-r-r.rYrz)DownloadStreamRequestSubmitter.can_submitcCr}r~r-rr-r-r.rprz:DownloadStreamRequestSubmitter._add_additional_subscriberscCstSr~rrjr-r-r.rrz+DownloadStreamRequestSubmitter._get_fileoutcCrrr-rr-r-r.rrz1DownloadStreamRequestSubmitter._format_local_pathN) rIrJrKrrqrYrprrr-r-r-r.rOs  rOc@s.eZdZejZeZddZddZ ddZ dS)rScC|jdko |jdkS)Ndeletes3rzsrc_typerjr-r-r.rY z!DeleteRequestSubmitter.can_submitcCs"t|j\}}|jj||||dS)N)rrrtru)rrxrLr)r*r_rtrurrr-r-r.rrs z/DeleteRequestSubmitter._submit_transfer_requestcC||jdfSr~)rrxrjr-r-r.r{z'DeleteRequestSubmitter._format_src_destN) rIrJrKrmap_delete_object_paramsrorrqrYrrr{r-r-r-r.rSs  rSc@s,eZdZdZdZddZddZddZdS)rTNcCr)Nrlocalrrjr-r-r.rY rz&LocalDeleteRequestSubmitter.can_submitc Cs||\}}d||d}zDz"|jtdddi|t|j|jtdi|WWdStyQ}z|jt dd|i|WYd}~WdSd}~wwYdS)Nrrwtotal_transfer_sizer exceptionTr-) r{rcr|rrremoverxr Exceptionr)r*r_rtrurxryrver-r-r.rrs,    z4LocalDeleteRequestSubmitter._submit_transfer_requestcCrr~)rrxrjr-r-r.r{-rz,LocalDeleteRequestSubmitter._format_src_dest)rIrJrKrorqrYrrr{r-r-r-r.rTs  rT)8loggingrs3transfer.managerrawscli.customizations.s3.utilsrrrrrr'awscli.customizations.s3.transferconfigr awscli.customizations.s3.resultsr r r r rrrrrrrrrrrrrrrrrr r!r"r#r$ awscli.compatr% getLoggerrIr3objectr&r8rbrPrQrRrNrOrSrTr-r-r-r.sX                                B?",$"