Merge branch 'release-1.7.14' into develop

* release-1.7.14: (30 commits)
  Bumping version to 1.7.14
  Update changelog
  Switch datapipeline list-runs to use clients
  Use choose-signer event for --no-sign-request
  Put imports in proper order
  Add issue to changelog
  Fix a bug in when setting nested config as last item
  Disable s3 streaming memory test on RHEL systems
  Fix htmlgen
  Fix tests after rebasing s3 branch from develop
  Update code from feedback
  Added some missing tests
  Touched up code based on pr-check script
  Add some test for multipart copies
  Switch make_loc_files to use FileCreator
  Removed all s3 services, operations, and endpoints
  Converted all tests for s3 client switch over
  Transitioned rest of s3 commands off of clients
  Remove use of clone client for s3 commands
  Add client support for s3 website
  ...

CHANGELOG.rst

@@ -2,6 +2,21 @@
 CHANGELOG
 =========
 
+1.7.14
+======
+
+* feature:``aws logs``: Update ``aws logs`` command to the latest model.
+* feature:``aws ec2``: Add paginators for the ``describe-snapshots``
+  sub-command.
+* feature:``aws cloudtrail``: Add support for the new ``lookup-events``
+  sub-command.
+* bugfix:``aws configure set``: Fix issue when setting nested configuration
+  values
+* feature:``aws s3``: Add support for ``--metadata-directive`` that allows
+  metadata to be copied or replaced for single part copies.
+  (`issue 1188 <https://github.com/aws/aws-cli/pull/1188>`__)
+
+
 1.7.13
 ======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.7.13'
+__version__ = '1.7.14'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/configure.py

@@ -17,8 +17,8 @@
 
 from botocore.exceptions import ProfileNotFound
 
-from awscli.customizations.commands import BasicCommand
 from awscli.compat import raw_input
+from awscli.customizations.commands import BasicCommand
 
 
 logger = logging.getLogger(__name__)
@@ -31,6 +31,7 @@ def register_configure_cmd(cli):
 
 
 class ConfigValue(object):
+
     def __init__(self, value, config_type, config_variable):
         self.value = value
         self.config_type = config_type
@@ -54,6 +55,7 @@ def _mask_value(current_value):
 
 
 class InteractivePrompter(object):
+
     def get_value(self, current_value, config_name, prompt_text=''):
         if config_name in ('aws_access_key_id', 'aws_secret_access_key'):
             current_value = _mask_value(current_value)
@@ -218,6 +220,10 @@ def _update_subattributes(self, index, contents, values, starting_indent):
                 # write out all the values now.
                 self._insert_new_values(i - 1, contents, values, '    ')
                 break
+        else:
+            if starting_indent != current_indent:
+                # The option is the last option in the file
+                self._insert_new_values(i, contents, values, '    ')
         return i
 
     def _insert_new_values(self, line_number, contents, new_values, indent=''):

awscli/customizations/datapipeline/__init__.py

@@ -340,15 +340,14 @@ def __init__(self, session, formatter=None):
         self._formatter = formatter
 
     def _run_main(self, parsed_args, parsed_globals, **kwargs):
-        self._set_session_objects(parsed_globals)
+        self._set_client(parsed_globals)
         self._parse_type_args(parsed_args)
         self._list_runs(parsed_args)
 
-    def _set_session_objects(self, parsed_globals):
+    def _set_client(self, parsed_globals):
         # This is called from _run_main and is used to ensure that we have
         # a service/endpoint object to work with.
-        self.service = self._session.get_service('datapipeline')
-        self.endpoint = self.service.get_endpoint(
+        self.client = self._session.create_client('datapipeline',
             region_name=parsed_globals.region,
             endpoint_url=parsed_globals.endpoint_url,
             verify=parsed_globals.verify_ssl)
@@ -389,15 +388,13 @@ def _list_runs(self, parsed_args):
         self._formatter.display_objects_to_user(converted)
 
     def _describe_objects(self, pipeline_id, object_ids):
-        operation = self.service.get_operation('DescribeObjects')
-        http_parsed, parsed = operation.call(
-            self.endpoint, pipeline_id=pipeline_id, object_ids=object_ids)
+        parsed = self.client.describe_objects(
+            pipelineId=pipeline_id, objectIds=object_ids)
         return parsed
 
     def _query_objects(self, pipeline_id, query):
-        operation = self.service.get_operation('QueryObjects')
-        paginator = operation.paginate(
-            self.endpoint, pipeline_id=pipeline_id,
+        paginator = self.client.get_paginator('query_objects').paginate(
+            pipelineId=pipeline_id,
             sphere='INSTANCE', query=query)
         parsed = paginator.build_full_result()
         return parsed['ids']

awscli/customizations/globalargs.py

@@ -13,7 +13,7 @@
 import sys
 import os
 
-import botocore
+from botocore.handlers import disable_signing
 import jmespath
 
 from awscli.compat import urlparse
@@ -72,8 +72,4 @@ def no_sign_request(parsed_args, session, **kwargs):
         # we need to set the signature_version to None for
         # any service created.  This ensures that get_endpoint()
         # will not look for auth.
-        session.register('service-created', disable_signing)
-
-
-def disable_signing(service, **kwargs):
-    service.signature_version = botocore.UNSIGNED
+        session.register('choose-signer', disable_signing)

awscli/customizations/s3/filegenerator.py

@@ -114,10 +114,9 @@ class FileGenerator(object):
     under the same common prefix.  The generator yields corresponding
     ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
     """
-    def __init__(self, service, endpoint, operation_name,
-                 follow_symlinks=True, page_size=None, result_queue=None):
-        self._service = service
-        self._endpoint = endpoint
+    def __init__(self, client, operation_name, follow_symlinks=True,
+                 page_size=None, result_queue=None):
+        self._client = client
         self.operation_name = operation_name
         self.follow_symlinks = follow_symlinks
         self.page_size = page_size
@@ -279,8 +278,7 @@ def list_objects(self, s3_path, dir_op):
         if not dir_op and prefix:
             yield self._list_single_object(s3_path)
         else:
-            operation = self._service.get_operation('ListObjects')
-            lister = BucketLister(operation, self._endpoint)
+            lister = BucketLister(self._client)
             for key in lister.list_objects(bucket=bucket, prefix=prefix,
                                            page_size=self.page_size):
                 source_path, size, last_update = key
@@ -304,10 +302,8 @@ def _list_single_object(self, s3_path):
         # IAM policies with the smallest set of permissions needed) and
         # instead use a HeadObject request.
         bucket, key = find_bucket_key(s3_path)
-        operation = self._service.get_operation('HeadObject')
         try:
-            response = operation.call(
-                self._endpoint, bucket=bucket, key=key)[1]
+            response = self._client.head_object(Bucket=bucket, Key=key)
         except ClientError as e:
             # We want to try to give a more helpful error message.
             # This is what the customer is going to see so we want to
@@ -318,7 +314,6 @@ def _list_single_object(self, s3_path):
             if e.http_status_code == 404:
                 # The key does not exist so we'll raise a more specific
                 # error message here.
-                copy_fields['error_code'] = 'NoSuchKey'
                 copy_fields['error_message'] = 'Key "%s" does not exist' % key
             else:
                 reason = six.moves.http_client.responses[