Merge branch 'release-1.7.33' into develop

* release-1.7.33:
  Bumping version to 1.7.33
  Update Changelog
  Enable use of managed policy for EMR.
  Use the latest policy in the example output for managed policy for EMR.i
  Fix example output for managed policy command for EMR.
  Fix example output for managed policy command for EMR.
  Fix documentation and example output for managed policy command for EMR.
  Enable use of managed policy for EMR.
  Add a unit test for Args to be mandatory in submitting Spark steps
  Add Args validation and update step type name for Spark
  Add support for Spark steps
  Bump dependancy of bcdoc

CHANGELOG.rst

@@ -2,6 +2,26 @@
 CHANGELOG
 =========
 
+1.7.33
+======
+
+* feature:``aws autoscaling``: Add support for attachinga and describing
+  load balancers.
+* feature:``aws ec2``: Add support for VPC flow logs and M4 instances.
+* feature:``aws emr``: Add Spark support and managed policy support.
+* feature:``aws ecs``: Add support for updating container agent.
+
+
+1.7.32
+======
+
+* feature:``aws logs``: Add support for ``put-subscription-filter``,
+  ``describe-subscription-filters``, and ``delete-subscription-filters``
+* feature:``aws storagegateway``: Add support for ``list-volume-initiators``
+* feature:``aws cognito-identity``: Add support for ``delete-identities``
+  and hiding disabled identities with the ``list-identities`` API operation
+
+
 1.7.31
 ======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.7.32'
+__version__ = '1.7.33'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/emr/constants.py

@@ -15,6 +15,8 @@
 
 EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
 EMR_ROLE_NAME = "EMR_DefaultRole"
+EC2_ROLE_ARN_PATTERN = "arn:{{region_suffix}}:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role"
+EMR_ROLE_ARN_PATTERN = "arn:{{region_suffix}}:iam::aws:policy/service-role/AmazonElasticMapReduceRole"
 
 # Action on failure
 CONTINUE = 'CONTINUE'
@@ -109,12 +111,14 @@
 STREAMING = 'streaming'
 GANGLIA = 'ganglia'
 HBASE = 'hbase'
+SPARK = 'spark'
 
 DEFAULT_CUSTOM_JAR_STEP_NAME = 'Custom JAR'
 DEFAULT_STREAMING_STEP_NAME = 'Streaming program'
 DEFAULT_HIVE_STEP_NAME = 'Hive program'
 DEFAULT_PIG_STEP_NAME = 'Pig program'
 DEFAULT_IMPALA_STEP_NAME = 'Impala program'
+DEFAULT_SPARK_STEP_NAME = 'Spark application'
 
 ARGS = '--args'
 RUN_HIVE_SCRIPT = '--run-hive-script'
@@ -124,7 +128,9 @@
 PIG_VERSIONS = '--pig-versions'
 PIG_STEP_CONFIG = 'PigStepConfig'
 RUN_IMPALA_SCRIPT = '--run-impala-script'
+SPARK_SUBMIT_COMMAND = '/home/hadoop/spark/bin/spark-submit'
 IMPALA_STEP_CONFIG = 'ImpalaStepConfig'
+SPARK_STEP_CONFIG = 'SparkStepConfig'
 STREAMING_STEP_CONFIG = 'StreamingStepConfig'
 CUSTOM_JAR_STEP_CONFIG = 'CustomJARStepConfig'
 
@@ -159,7 +165,7 @@
 
 LATEST = 'latest'
 
-APPLICATIONS = ["HIVE", "PIG", "HBASE", "GANGLIA", "IMPALA", "MAPR",
+APPLICATIONS = ["HIVE", "PIG", "HBASE", "GANGLIA", "IMPALA", "SPARK", "MAPR",
                 "MAPR_M3", "MAPR_M5", "MAPR_M7"]
 
 SSH_USER = 'hadoop'

awscli/customizations/emr/createdefaultroles.py

@@ -25,69 +25,15 @@
 from awscli.customizations.emr.command import Command
 from awscli.customizations.emr.constants import EC2
 from awscli.customizations.emr.constants import EC2_ROLE_NAME
+from awscli.customizations.emr.constants import EC2_ROLE_ARN_PATTERN
 from awscli.customizations.emr.constants import EMR
 from awscli.customizations.emr.constants import EMR_ROLE_NAME
+from awscli.customizations.emr.constants import EMR_ROLE_ARN_PATTERN
 from awscli.customizations.emr.exceptions import ResolveServicePrincipalError
 
 
 LOG = logging.getLogger(__name__)
 
-
-EC2_ROLE_POLICY = {
-    "Statement": [
-        {
-            "Action": [
-                "cloudwatch:*",
-                "dynamodb:*",
-                "ec2:Describe*",
-                "elasticmapreduce:Describe*",
-                "rds:Describe*",
-                "s3:*",
-                "sdb:*",
-                "sns:*",
-                "sqs:*"
-            ],
-            "Effect": "Allow",
-            "Resource": ["*"]
-        }
-    ]
-}
-
-
-EMR_ROLE_POLICY = {
-    "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Action": [
-                "ec2:AuthorizeSecurityGroupIngress",
-                "ec2:CancelSpotInstanceRequests",
-                "ec2:CreateSecurityGroup",
-                "ec2:CreateTags",
-                "ec2:Describe*",
-                "ec2:DeleteTags",
-                "ec2:ModifyImageAttribute",
-                "ec2:ModifyInstanceAttribute",
-                "ec2:RequestSpotInstances",
-                "ec2:RunInstances",
-                "ec2:TerminateInstances",
-                "iam:PassRole",
-                "iam:ListRolePolicies",
-                "iam:GetRole",
-                "iam:GetRolePolicy",
-                "iam:ListInstanceProfiles",
-                "s3:Get*",
-                "s3:List*",
-                "s3:CreateBucket",
-                "sdb:BatchPutAttributes",
-                "sdb:Select"
-            ],
-            "Effect": "Allow",
-            "Resource": "*"
-        }
-    ]
-}
-
-
 def assume_role_policy(serviceprincipal):
     return {
         "Version": "2008-10-17",
@@ -101,6 +47,22 @@ def assume_role_policy(serviceprincipal):
         ]
     }
 
+def get_service_role_policy_arn(region):
+    region_suffix = _get_policy_arn_suffix(region)
+    return EMR_ROLE_ARN_PATTERN.replace("{{region_suffix}}", region_suffix)
+
+def get_ec2_role_policy_arn(region):
+    region_suffix = _get_policy_arn_suffix(region)
+    return EC2_ROLE_ARN_PATTERN.replace("{{region_suffix}}", region_suffix)
+
+def _get_policy_arn_suffix(region):
+    region_string = region.lower()
+    if region_string.startswith("cn-"):
+        return "aws-cn"
+    elif region_string.startswith("us-gov"):
+        return "aws-us-gov"
+    else:
+        return "aws"
 
 def get_service_principal(service, endpoint_host):
     return service + '.' + _get_suffix(endpoint_host)
@@ -139,7 +101,9 @@ class CreateDefaultRoles(Command):
     DESCRIPTION = ('Creates the default IAM role ' +
                    EC2_ROLE_NAME + ' and ' +
                    EMR_ROLE_NAME + ' which can be used when'
-                   ' creating the cluster using the create-cluster command.\n'
+                   ' creating the cluster using the create-cluster command.'
+                   ' The default roles for EMR use managed policies, which'
+                   ' are updated automatically to support future EMR functionality.\n'
                    '\nIf you do not have a Service Role and Instance Profile '
                    'variable set for your create-cluster command in the AWS '
                    'CLI config file, create-default-roles will automatically '
@@ -161,7 +125,10 @@ class CreateDefaultRoles(Command):
 
     def _run_main_command(self, parsed_args, parsed_globals):
         ec2_result = None
+        ec2_policy = None
         emr_result = None
+        emr_policy = None
+
         self.iam_endpoint_url = parsed_args.iam_endpoint
 
         self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
@@ -182,10 +149,10 @@ def _run_main_command(self, parsed_args, parsed_globals):
         else:
             LOG.debug('Role ' + role_name + ' does not exist.'
                       ' Creating default role for EC2: ' + role_name)
+            role_arn = get_ec2_role_policy_arn(self.region)
             ec2_result = self._create_role_with_role_policy(
-                role_name, role_name, EC2,
-                emrutils.dict_to_string(EC2_ROLE_POLICY),
-                parsed_globals)
+                role_name, EC2, role_arn, parsed_globals)
+            ec2_policy = self._get_role_policy(role_arn, parsed_globals);
 
         # Check if the default EC2 Instance Profile for EMR exists.
         instance_profile_name = EC2_ROLE_NAME
@@ -207,17 +174,16 @@ def _run_main_command(self, parsed_args, parsed_globals):
         else:
             LOG.debug('Role ' + role_name + ' does not exist.'
                       ' Creating default role for EMR: ' + role_name)
+            role_arn = get_service_role_policy_arn(self.region)
             emr_result = self._create_role_with_role_policy(
-                role_name, role_name, EMR,
-                emrutils.dict_to_string(EMR_ROLE_POLICY),
-                parsed_globals)
+                role_name, EMR, role_arn, parsed_globals)
+            emr_policy = self._get_role_policy(role_arn, parsed_globals)
 
         configutils.update_roles(self._session)
-
         emrutils.display_response(
             self._session,
             'create_role',
-            self._construct_result(ec2_result, emr_result),
+            self._construct_result(ec2_result, ec2_policy, emr_result, emr_policy),
             parsed_globals)
 
         return 0
@@ -229,19 +195,18 @@ def _check_for_iam_endpoint(self, region, iam_endpoint):
             if iam_endpoint is None:
                 raise exceptions.UnknownIamEndpointError(region=region)
 
-    def _construct_result(self, ec2_response, emr_response):
+    def _construct_result(self, ec2_response, ec2_policy, emr_response, emr_policy):
         result = []
         self._construct_role_and_role_policy_structure(
-            result, ec2_response, EC2_ROLE_POLICY)
+            result, ec2_response, ec2_policy)
         self._construct_role_and_role_policy_structure(
-            result, emr_response, EMR_ROLE_POLICY)
+            result, emr_response, emr_policy)
         return result
 
     def _construct_role_and_role_policy_structure(
-            self, list, response, role_policy):
-        if response is not None and response[1] is not None:
-            list.append({'Role': response[1]['Role'],
-                         'RolePolicy': role_policy})
+            self, list, response, policy):
+        if response is not None and response['Role'] is not None:
+            list.append({'Role': response['Role'], 'RolePolicy': policy})
             return list
 
     def _check_if_role_exists(self, role_name, parsed_globals):
@@ -251,7 +216,7 @@ def _check_if_role_exists(self, role_name, parsed_globals):
         except Exception as e:
             role_not_found_msg = 'The role with name ' + role_name +\
                                  ' cannot be found'
-            if role_not_found_msg in e.message:
+            if role_not_found_msg in e.error_message:
                 # No role error.
                 return False
             else:
@@ -270,7 +235,7 @@ def _check_if_instance_profile_exists(self, instance_profile_name,
             profile_not_found_msg = 'Instance Profile ' +\
                                     instance_profile_name +\
                                     ' cannot be found.'
-            if profile_not_found_msg in e.message:
+            if profile_not_found_msg in e.error_message:
                 # No instance profile error.
                 return False
             else:
@@ -279,9 +244,18 @@ def _check_if_instance_profile_exists(self, instance_profile_name,
 
         return True
 
+    def _get_role_policy(self, arn, parsed_globals):
+        parameters = {}
+        parameters['PolicyArn'] = arn
+        policy_details = self._call_iam_operation('GetPolicy',
+                                                   parameters,
+                                                   parsed_globals)
+        parameters["VersionId"] = policy_details["Policy"]["DefaultVersionId"]
+        policy_version_details = self._call_iam_operation('GetPolicyVersion', parameters, parsed_globals)
+        return policy_version_details["PolicyVersion"]["Document"]
+
     def _create_role_with_role_policy(
-            self, role_name, policy_name, service_name, policy_document,
-            parsed_globals):
+            self, role_name, service_name, role_arn, parsed_globals):
         service_principal = get_service_principal(service_name,
                                                   self.emr_endpoint_url)
         LOG.debug(service_principal)
@@ -295,10 +269,9 @@ def _create_role_with_role_policy(
                                                         parsed_globals)
 
         parameters = {}
-        parameters['PolicyDocument'] = policy_document
-        parameters['PolicyName'] = policy_name
+        parameters['PolicyArn'] = role_arn
         parameters['RoleName'] = role_name
-        self._call_iam_operation('PutRolePolicy', parameters, parsed_globals)
+        self._call_iam_operation('AttachRolePolicy', parameters, parsed_globals)
 
         return create_role_response
 

awscli/customizations/emr/steputils.py

@@ -35,6 +35,8 @@ def build_step_config_list(parsed_step_list, region):
             step_config = build_pig_step(parsed_step=step, region=region)
         elif step_type == constants.IMPALA:
             step_config = build_impala_step(parsed_step=step, region=region)
+        elif step_type == constants.SPARK:
+            step_config = build_spark_step(parsed_step=step, region=region)
         else:
             raise exceptions.UnknownStepTypeError(step_type=step_type)
 
@@ -164,6 +166,24 @@ def build_impala_step(parsed_step, region):
         action_on_failure=action_on_failure)
 
 
+def build_spark_step(parsed_step, region):
+    name = _apply_default_value(
+        arg=parsed_step.get('Name'),
+        value=constants.DEFAULT_SPARK_STEP_NAME)
+    action_on_failure = _apply_default_value(
+        arg=parsed_step.get('ActionOnFailure'),
+        value=constants.DEFAULT_FAILURE_ACTION)
+    args_list=[constants.SPARK_SUBMIT_COMMAND]
+    args = parsed_step.get('Args')
+    emrutils.check_required_field(
+        structure=constants.SPARK_STEP_CONFIG, name='Args', value=args)
+    args_list += args
+    return emrutils.build_step(
+        jar=emrutils.get_script_runner(region),
+        args=args_list,
+        name=name,
+        action_on_failure=action_on_failure)
+
 def _apply_default_value(arg, value):
     if arg is None:
         arg = value