o .&a'e@sddlZddlmZddlmZddlmZddlmZddlmZddlmZddlm Z dd lm Z dd lm Z dd lm Z dd lm Z dd lmZddlmZddlmZddlmZddlmZGdddeZdS)N) BasicCommand)applicationutils)argumentschema) constants) emrfsutils)emrutils) exceptions) hbaseutils)helptext)instancegroupsutils)instancefleetsutils) steputils)Command) EC2_ROLE_NAME) EMR_ROLE_NAME)jsonc@seZdZdZejZgdejddejdde j ej ddej ddej dd d d ejd d d d dde jejdddejddejddejddejddejddd ejddejddeje jddd dejd dd dddejdd d d!ejd d"d d!dd#d d$ejd d%d d$dd&d'eje jd(d)ej e j!dd*ej"e j#dd+ej$e j%dd,e j&ej'dd-ej(dd.e j)ej*dd/ej+dd0ej,dd1ej-dd2ej.dd3e j/ej0dd4d5ej1d6d7e j2ej3dd8e j4ej5dd9e j6ej7dZ8e9:d:d;Z;e9:d:d<ZZ=d?d@Z>dAdBZ?dCdDZ@dEdFZAdGdHZBdIdJZCdKdLZDdMdNZEdOdPZFdQdRZGdSdTZHdUS)V CreateClusterzcreate-clusterz release-label)name help_textz ami-versionzinstance-groups)rschemarz instance-typezinstance-countzauto-terminate store_trueauto_terminate)raction group_namerzno-auto-terminate)rrrzinstance-fleetsrzDevelopment Cluster)rdefaultrzlog-urizlog-encryption-kms-key-idz service-rolezauto-scaling-rolezuse-default-roles)rrrconfigurationszec2-attributes)rrrztermination-protectedtermination_protectedzno-termination-protectedzscale-down-behaviorzvisible-to-all-users visibilityzno-visible-to-all-userszenable-debuggingdebugzno-enable-debuggingtags+)rnargsrrzbootstrap-actions applicationsemrfsstepszadditional-infozrestore-from-hbase-backupzsecurity-configurationz custom-ami-idzebs-root-volume-sizezrepo-upgrade-on-bootzkerberos-attributeszstep-concurrency-levelinteger)r cli_type_namerzmanaged-scaling-policyzplacement-group-configszauto-termination-policyemrzcreate-cluster-synopsis.txtzcreate-cluster-examples.rstcCsi}|j|d<||d}|jdur |jdur tjdd|d|jdur7|jdur7d|jvr7tjdd|d|jdurH|jdurHtjd d d i}|jdurXt |j|d <n t j |j|j |jd |d<|jdur|j|d<|jdurz t|j|d<Wn tytdw|jdur|jdurtd|j}|durtj|jd|j|d<t|d|jt|d|j|jdurt|d|j|jdurt|_|jduri|_t|jd<t|d|j|jdur|dD]}d|vr|j durt!qt|d|j |j"durt|d|j"|j#dur(|j$dur(d|_#t%|j#d|j$d|d<t%|j&d |j'd!|d"<|j(durO|j)durOd|_(t%|j(d#|j)d$|d%<t*|j+|d&<||d'<|jduru|j,||jd(t%|j-d)|j.d*}|jdur|durtj/|dur|j0|d+|1||gd,|j2dur|jdurt3j4|j5|j2|dd-\} } } |0|d.| |0|d/| |0|d+| ng|d0<|j2D] } |d06| q|j7} | dur t8j9| :d1| :d2d3}tj;tt|jN|jOdurt|d?|jO|jPdurt|d@|jP|Q|tR|jSdA||j5|jT|jU}|V|}tW|jSdA||dBS)CNNamez| Either choose --use-default-roles or use both --service-role and --ec2-attributes InstanceProfile=.Tz--use-default-rolesz--service-role)option1option2messageInstanceProfilez --ec2-attributes InstanceProfilez--instance-groupsz--instance-fleetsr)r*InstanceFleets)instance_groups instance_typeinstance_countInstanceGroups ReleaseLabelConfigurationsz=aws: error: invalid json argument for option --configurationsz\d?\..*) ami_version AmiVersionAdditionalInfoLogUriLogEncryptionKmsKeyId ServiceRoleAutoScalingPolicyAutoScalingRoleScaleDownBehaviorFz--no-auto-terminatez--auto-terminateKeepJobFlowAliveWhenNoStepsz--termination-protectedz--no-termination-protectedTerminationProtectedz--visible-to-all-usersz--no-visible-to-all-usersVisibleToAllUsersTags Instances)cluster parsed_attrsz--enable-debuggingz--no-enable-debuggingStepsrCkeyvalue)regionparsed_applicationsr5NewSupportedProductsBootstrapActions ApplicationsDir BackupVersion)dirbackup_version)jarraction_on_failureargs)rCparsed_boostrap_actions)rC emrfs_args release_label)parsed_step_listrIrWSecurityConfiguration CustomAmiIdEbsRootVolumeSizeRepoUpgradeOnBootKerberosAttributesStepConcurrencyLevelManagedScalingPolicyPlacementGroupConfigsAutoTerminationPolicy run_job_flowr)Xr#_validate_release_label_ami_versionuse_default_roles service_rolerMutualExclusiveOptionErrorec2_attributesr/instance_fleetsr "validate_and_build_instance_fleetsr "validate_and_build_instance_groupsr0r1rWrrloads ValueErrorr5rematchInvalidAmiVersionErrorr apply_dictadditional_infolog_urilog_encryption_kms_key_idrrkeysauto_scaling_roleMissingAutoScalingRoleErrorscale_down_behaviorno_auto_terminaterapply_boolean_optionsrno_termination_protectedvisible_to_all_usersno_visible_to_all_users parse_tagsr_build_ec2_attributesenable_debuggingno_enable_debugging LogUriError_update_cluster_dict_build_enable_debuggingr"rbuild_applicationsrIappendrestore_from_hbase_backupr $build_hbase_restore_from_backup_argsget build_steprHBASE_JAR_PATHHBASE_RESTORE_STEP_NAMECANCEL_AND_WAITbootstrap_actions_build_bootstrap_actionsr#_handle_emrfs_parametersr$r build_step_config_listsecurity_configuration custom_ami_idebs_root_volume_sizeintrepo_upgrade_on_bootkerberos_attributesstep_concurrency_levelmanaged_scaling_policyplacement_group_configsauto_termination_policy_validate_required_applicationscall_session endpoint_url verify_ssl_construct_resultdisplay_response)self parsed_argsparsed_globalsparamsservice_role_validation_messageinstances_configis_valid_ami_versioninstance_groupdebugging_enabledapp_listba_list step_list applicationhbase_restore_configrT step_config steps_listrun_job_flow_responseconstructed_resultrI/usr/lib/python3/dist-packages/awscli/customizations/emr/createcluster.py_run_main_commands                                                      zCreateCluster._run_main_commandcCs:d}d}|dur|d}|d}|dur||dSiS)N JobFlowId ClusterArn) ClusterIdr)r)rrun_job_flow_result jobFlowId clusterArnrrrr{s  zCreateCluster._construct_resultcCsx|}|d}d|vrd|vrtjdddd|vr&d|vr&tjdddd|vs.d|vr9d|vs6d|vr9tjtj|d|dd tj|d|d d tj|d|d d d|vrht|d <tj|d|d dd d|vr|t|d <tj|d|d dd tj|d |dd tj|d|dd tj|d|dd tj|d|dd tj|d|dd tj|d|dd tj|d|d|S)NrBSubnetId SubnetIdsr-AvailabilityZoneAvailabilityZonesKeyName Ec2KeyName src_paramssrc_key dest_paramsdest_key Ec2SubnetId Ec2SubnetIds Placementr, JobFlowRoleEmrManagedMasterSecurityGroupEmrManagedSlaveSecurityGroupServiceAccessSecurityGroupAdditionalMasterSecurityGroupsAdditionalSlaveSecurityGroupsrrGrH)rtrrfSubnetAndAzValidationErrorr apply_paramsdictapply)rrCrDrt instancesrrrr~s  z#CreateCluster._build_ec2_attributesc Cs|d}|dur g}g}t|t|tjkrtd|D]9}i}|ddur1|d|d<ntj|d<i}tj|d|ddtj|d|ddtj|d|d| |q||}t|d krf||d<|S) NrLzGaws: error: maximum number of bootstrap actions for a cluster exceeded.r(PathrArgsScriptBootstrapActionrr) rlenrMAX_BOOTSTRAP_ACTION_NUMBERrlBOOTSTRAP_ACTION_NAMErrrr) rrCrUcluster_ba_listrba ba_configscript_arg_configresultrrrrs@    z&CreateCluster._build_bootstrap_actionscCsL|jr tj}tjg}nt|j}tjtj|jdg}tj tj tj ||dS)N) relative_pathrI)rrSrRrT) rWrCOMMAND_RUNNERDEBUGGING_COMMANDrget_script_runnerrI build_s3_linkDEBUGGING_PATHrDEBUGGING_NAMETERMINATE_CLUSTER)rrrrRrTrrrrs  z%CreateCluster._build_enable_debuggingcCs@||vr|||7<|S|durt|dkr|||<|S)Nr)rtr)rrCrGrHrrrrs z"CreateCluster._update_cluster_dictcCsL|jdur|jdurtjddd|jdur"|jdur$tjddddSdS)Nz --ami-versionz--release-labelr-)r5rWrrfRequiredOptionsError)rrrrrrc s    z1CreateCluster._validate_release_label_ami_versioncCsrtg}|jdurtdd|jD}|||}|jdur+tj|vr+|tjt|dkr7t j |ddS)NcSsg|]}|dqS)r()lower).0apprrr szACreateCluster._validate_required_applications..r)r") setr"#_get_missing_applications_for_stepsrrHBASEaddtitlerrMissingApplicationsError)rrspecified_apps missing_appsrrrrs    z-CreateCluster._validate_required_applicationscCsttjtjtjg}tg}|jdurB|jD]+}t|t|kr#|S|d}|durA|}||vrA||vrA| |d q|S)NType) rrHIVEPIGIMPALAr$rrrrr)rrrallowed_app_stepsrstep step_typerrrr+s"    z1CreateCluster._get_missing_applications_for_stepscCs|jr dd|D}|S)NcSs$g|]}|jdkr|jdkr|qS)reinstance_profile)r)rxrrrr?s   zICreateCluster._filter_configurations_in_special_cases..)rd)rrrparsed_configsrrr'_filter_configurations_in_special_cases<sz5CreateCluster._filter_configurations_in_special_casescCsP|r||t|}|j|d|gddSt|j|}|j|d|ddS)Nr4rFrL)validate_no_emrfs_configurationrbuild_emrfs_confiurationrbuild_bootstrap_action_configsrI)rrCrVrWemrfs_configurationemrfs_ba_config_listrrrrDs    z&CreateCluster._handle_emrfs_parameterscCs<d|vr|dD]}|dur|dtjkrtjqdSdS)Nr4Classification)rr EMRFS_SITEr DuplicateEmrFsConfigurationError)rrCconfigrrrrTs z-CreateCluster.validate_no_emrfs_configurationN)I__name__ __module__ __qualname__NAMEr CREATE_CLUSTER_DESCRIPTION DESCRIPTION RELEASE_LABEL AMI_VERSIONrINSTANCE_GROUPS_SCHEMAINSTANCE_GROUPS INSTANCE_TYPEINSTANCE_COUNTAUTO_TERMINATEINSTANCE_FLEETS_SCHEMAINSTANCE_FLEETS CLUSTER_NAMELOG_URILOG_ENCRYPTION_KMS_KEY_ID SERVICE_ROLEAUTOSCALING_ROLEUSE_DEFAULT_ROLESCONFIGURATIONSEC2_ATTRIBUTESEC2_ATTRIBUTES_SCHEMATERMINATION_PROTECTEDSCALE_DOWN_BEHAVIOR VISIBILITY DEBUGGINGTAGS TAGS_SCHEMABOOTSTRAP_ACTIONSBOOTSTRAP_ACTIONS_SCHEMA APPLICATIONSAPPLICATIONS_SCHEMAEMR_FS EMR_FS_SCHEMA STEPS_SCHEMASTEPSADDITIONAL_INFO HBASE_RESTORE_FROM_BACKUP_SCHEMARESTORE_FROM_HBASESECURITY_CONFIG CUSTOM_AMI_IDEBS_ROOT_VOLUME_SIZEREPO_UPGRADE_ON_BOOTKERBEROS_ATTRIBUTES_SCHEMAKERBEROS_ATTRIBUTESSTEP_CONCURRENCY_LEVELMANAGED_SCALING_POLICY_SCHEMAMANAGED_SCALING_POLICYPLACEMENT_GROUP_CONFIGS_SCHEMAPLACEMENT_GROUP_CONFIGSAUTO_TERMINATION_POLICY_SCHEMAAUTO_TERMINATION_POLICY ARG_TABLEr FROM_FILESYNOPSISEXAMPLESrrr~rrrrcrrrrrrrrrr!s  !#&)+-0257:=@CFHKMOQSVY\_ c r F% r)rmawscli.customizations.commandsrawscli.customizations.emrrrrrrrr r r r r !awscli.customizations.emr.commandr#awscli.customizations.emr.constantsrrbotocore.compatrrrrrrs$