Merge branch 'release-1.2.7'

* release-1.2.7: (21 commits)
  Bumping version to 1.2.7
  Update changelog with the latest changes
  Change mututally to mutually
  Allow the --protocol option of ec2 create-network-acl-entry command to accept tcp|udp|icmp|all in addition to the numeric protocol numbers.  The docs for the command already say it does this but the actual EC2 operation supports only integer values.
  Fix bug when filtering s3 locations
  Clean up existing filter unit tests
  Add unittest for website redirect location
  Add unit test for recursive download
  Remove extraneous log statements.
  Fix an assumption in argprocess.py that all map types will have an enum of possible keys.  Also change the import of json to come from the botocore.compat module and also set the object_pairs_hook to use OrderedDict so unit tests can be compared properly.  Dependent on boto/botocore#185. Fixes #407.
  Add tests that verify what ops were called
  Add a short circuit path for listing a specific s3 object
  Remove newline between task fail and error messages
  Add attributes to exception in errorhandler
  Remove src/dest resource existence check
  If --private-ip-address is specified with any other options that require the creation of a NetworkInterfaces structure, move the value of --private-ip-address into the NetworkInterfaces structure.  Fixes #520.
  Log CLI/botocore version in the debug logs
  Add changelog entry for issue 516
  Handle a list of strings from a dict in text output
  Remove tutorial in favor of user guide
  ...

CHANGELOG.rst

@@ -2,6 +2,30 @@
 CHANGELOG
 =========
 
+Next Release (TBD)
+==================
+
+* Allow tcp, udp, icmp, all for ``--protocol`` param of
+  the ``ec2 create-network-acl-entry`` command
+  (`issue 508 <https://github.com/aws/aws-cli/issues/508>`__)
+* Fix bug when filtering ``s3://`` locations with the
+  ``--include/--exclude`` params
+  (issue 531 <https://github.com/aws/aws-cli/pull/531>`__)
+* Fix an issue with map type parameters raising uncaught
+  exceptions in commands such as `sns create-platform-application`
+  (`issue 407 <https://github.com/aws/aws-cli/issues/407>`__)
+* Fix an issue when both ``--private-ip-address`` and
+  ``--associate-public-ip-address`` are specified in the
+  ``ec2 run-instances`` command
+  (`issue 520 <https://github.com/aws/aws-cli/issues/520>`__)
+* Fix an issue where ``--output text`` was not providing
+  a starting identifier for certain rows
+  (`issue 516 <https://github.com/aws/aws-cli/pull/516>`__)
+* Update the ``support`` command to the latest version
+* Update the ``--query`` syntax to support flattening sublists
+  (`boto/jmespath#20 <https://github.com/boto/jmespath/pull/20>`__)
+
+
 1.2.6
 =====
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.2.6'
+__version__ = '1.2.7'
 
 #
 # Get our data path to be added to botocore's search path

awscli/argprocess.py

@@ -12,10 +12,11 @@
 # language governing permissions and limitations under the License.
 """Module for processing CLI args."""
 import os
-import json
 import logging
 import six
 
+from botocore.compat import OrderedDict, json
+
 from awscli import utils
 from awscli import SCALAR_TYPES, COMPLEX_TYPES
 
@@ -243,7 +244,7 @@ def _key_value_parse(self, param, value):
         # that is, csv key value pairs, where the key and values
         # are separated by '='.  All of this should be whitespace
         # insensitive.
-        parsed = {}
+        parsed = OrderedDict()
         parts = self._split_on_commas(value)
         valid_names = self._create_name_to_params(param)
         for part in parts:
@@ -253,18 +254,19 @@ def _key_value_parse(self, param, value):
                 raise ParamSyntaxError(part)
             key = key.strip()
             value = value.strip()
-            if key not in valid_names:
+            if valid_names and key not in valid_names:
                 raise ParamUnknownKeyError(param, key, valid_names)
-            sub_param = valid_names[key]
-            if sub_param is not None:
-                value = unpack_scalar_cli_arg(sub_param, value)
+            if valid_names:
+                sub_param = valid_names[key]
+                if sub_param is not None:
+                    value = unpack_scalar_cli_arg(sub_param, value)
             parsed[key] = value
         return parsed
 
     def _create_name_to_params(self, param):
         if param.type == 'structure':
             return dict([(p.name, p) for p in param.members])
-        elif param.type == 'map':
+        elif param.type == 'map' and hasattr(param.keys, 'enum'):
             return dict([(v, None) for v in param.keys.enum])
 
     def _docs_list_scalar_list_parse(self, param):
@@ -351,7 +353,7 @@ def unpack_cli_arg(parameter, value):
 def unpack_complex_cli_arg(parameter, value):
     if parameter.type == 'structure' or parameter.type == 'map':
         if value.lstrip()[0] == '{':
-            d = json.loads(value)
+            d = json.loads(value, object_pairs_hook=OrderedDict)
         else:
             msg = 'The value for parameter "%s" must be JSON or path to file.' % (
                 parameter.cli_name)
@@ -360,11 +362,11 @@ def unpack_complex_cli_arg(parameter, value):
     elif parameter.type == 'list':
         if isinstance(value, six.string_types):
             if value.lstrip()[0] == '[':
-                return json.loads(value)
+                return json.loads(value, object_pairs_hook=OrderedDict)
         elif isinstance(value, list) and len(value) == 1:
             single_value = value[0].strip()
             if single_value and single_value[0] == '[':
-                return json.loads(value[0])
+                return json.loads(value[0], object_pairs_hook=OrderedDict)
         return [unpack_cli_arg(parameter.members, v) for v in value]
 
 

awscli/arguments.py

@@ -143,7 +143,7 @@ def group_name(self):
 
         This base class has no default behavior for groups, code
         that consumes argument objects can use them for whatever
-        purposes they like (documentation, mututally exclusive group
+        purposes they like (documentation, mutually exclusive group
         validation, etc.).
 
         """

awscli/clidriver.py

@@ -14,6 +14,7 @@
 import logging
 
 import botocore.session
+from botocore import __version__ as botocore_version
 from botocore.hooks import HierarchicalEmitter
 from botocore import xform_name
 from botocore.compat import copy_kwargs, OrderedDict
@@ -219,6 +220,9 @@ def _handle_top_level_args(self, args):
             # loading of plugins, etc.
             self.session.set_debug_logger(logger_name='botocore')
             self.session.set_debug_logger(logger_name='awscli')
+            LOG.debug("CLI version: %s, botocore version: %s",
+                      self.session.user_agent(),
+                      botocore_version)
         else:
             self.session.set_stream_logger(logger_name='awscli',
                                            log_level=logging.ERROR)

awscli/customizations/ec2protocolarg.py

@@ -0,0 +1,35 @@
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""
+This customization allows the user to specify the values "tcp", "udp",
+or "icmp" as values for the --protocol parameter.  The actual Protocol
+parameter of the operation accepts only integer protocol numbers.
+"""
+
+def _fix_args(operation, endpoint, params, **kwargs):
+    if 'protocol' in params:
+        if params['protocol'] == 'tcp':
+            params['protocol'] = '6'
+        elif params['protocol'] == 'udp':
+            params['protocol'] = '17'
+        elif params['protocol'] == 'icmp':
+            params['protocol'] = '1'
+        elif params['protocol'] == 'all':
+            params['protocol'] = '-1'
+
+
+def register_protocol_args(cli):
+    ('before-parameter-build.ec2.RunInstances', _fix_args),
+    cli.register('before-parameter-build.ec2.CreateNetworkAclEntry',
+                 _fix_args)
+

awscli/customizations/ec2runinstances.py

@@ -100,6 +100,11 @@ def _fix_args(operation, endpoint, params, **kwargs):
             if 'security_group_ids' in params:
                 ni[0]['Groups'] = params['security_group_ids']
                 del params['security_group_ids']
+            if 'private_ip_address' in params:
+                ip_addr = {'PrivateIpAddress': params['private_ip_address'],
+                           'Primary': True}
+                ni[0]['PrivateIpAddresses'] = [ip_addr]
+                del params['private_ip_address']
 
 
 EVENTS = [

awscli/customizations/s3/filegenerator.py

@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 import os
 import sys
+import datetime
 
 from six import text_type
 from dateutil.parser import parse
@@ -152,27 +153,39 @@ def list_objects(self, s3_path, dir_op):
         common prefix.  It yields the file's source path, size, and last
         update.
         """
-        operation = self._service.get_operation('ListObjects')
+        # Short circuit path: if we are not recursing into the s3
+        # bucket and a specific path was given, we can just yield
+        # that path and not have to call any operation in s3.
         bucket, prefix = find_bucket_key(s3_path)
-        iterator = operation.paginate(self._endpoint, bucket=bucket,
-                                      prefix=prefix)
-        for html_response, response_data in iterator:
-            contents = response_data['Contents']
-            for content in contents:
-                src_path = bucket + '/' + content['Key']
-                size = content['Size']
-                last_update = parse(content['LastModified'])
-                last_update = last_update.astimezone(tzlocal())
-                if size == 0 and src_path.endswith('/'):
-                    if self.operation_name == 'delete':
-                        # This is to filter out manually created folders
-                        # in S3.  They have a size zero and would be
-                        # undesirably downloaded.  Local directories
-                        # are automatically created when they do not
-                        # exist locally.  But user should be able to
-                        # delete them.
+        if not dir_op and prefix:
+            # Then a specific path was specified so we yield that
+            # exact path.  The size doesn't matter, but the last_update
+            # is normally set to the last_modified time we get back
+            # from s3 for the specific object.  We lose that here, but
+            # on the plus side, we don't need to require ListObjects
+            # permission to download a single file.
+            yield s3_path, 1, datetime.datetime.now()
+        else:
+            operation = self._service.get_operation('ListObjects')
+            iterator = operation.paginate(self._endpoint, bucket=bucket,
+                                        prefix=prefix)
+            for html_response, response_data in iterator:
+                contents = response_data['Contents']
+                for content in contents:
+                    src_path = bucket + '/' + content['Key']
+                    size = content['Size']
+                    last_update = parse(content['LastModified'])
+                    last_update = last_update.astimezone(tzlocal())
+                    if size == 0 and src_path.endswith('/'):
+                        if self.operation_name == 'delete':
+                            # This is to filter out manually created folders
+                            # in S3.  They have a size zero and would be
+                            # undesirably downloaded.  Local directories
+                            # are automatically created when they do not
+                            # exist locally.  But user should be able to
+                            # delete them.
+                            yield src_path, size, last_update
+                    elif not dir_op and s3_path != src_path:
+                        pass
+                    else:
                         yield src_path, size, last_update
-                elif not dir_op and s3_path != src_path:
-                    pass
-                else:
-                    yield src_path, size, last_update

awscli/customizations/s3/filters.py

@@ -10,10 +10,14 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import logging
 import fnmatch
 import os
 
 
+LOG = logging.getLogger(__name__)
+
+
 class Filter(object):
     """
     This is a universal exclude/include filter.
@@ -55,13 +59,21 @@ def call(self, file_infos):
 
                 else:
                     path_pattern = pattern[1].replace(os.sep, '/')
-                    full_path_pattern = path_pattern
-
+                    full_path_pattern = os.path.join(file_path.split('/')[0],
+                                                     path_pattern)
                 is_match = fnmatch.fnmatch(file_path, full_path_pattern)
                 if is_match and pattern_type == '--include':
                     file_status = (file_info, True)
+                    LOG.debug("%s matched include filter: %s",
+                              file_path, full_path_pattern)
                 elif is_match and pattern_type == '--exclude':
                     file_status = (file_info, False)
-
+                    LOG.debug("%s matched exclude filter: %s",
+                              file_path, full_path_pattern)
+                else:
+                    LOG.debug("%s did not match %s filter: %s",
+                              file_path, pattern_type[2:], full_path_pattern)
+            LOG.debug("=%s final filtered status, should_include: %s",
+                      file_path, file_status[1])
             if file_status[1]:
                 yield file_info

awscli/customizations/s3/s3.py

@@ -639,24 +639,12 @@ def add_paths(self, paths):
         the destination always have some value.
         """
         self.check_path_type(paths)
-        self.check_src_path(paths)
         src_path = paths[0]
         self.parameters['src'] = src_path
         if len(paths) == 2:
             self.parameters['dest'] = paths[1]
         elif len(paths) == 1:
             self.parameters['dest'] = paths[0]
-        self.check_dest_path(self.parameters['dest'])
-
-    def check_dest_path(self, destination):
-        if destination.startswith('s3://') and \
-                self.cmd in ['cp', 'sync', 'mv']:
-            bucket, key = find_bucket_key(destination[5:])
-            # A bucket is not always provided (like 'aws s3 ls')
-            # so only verify the bucket exists if we actually have
-            # a bucket.
-            if bucket:
-                self._verify_bucket_exists(bucket)
 
     def _verify_bucket_exists(self, bucket_name):
         session = self.session
@@ -706,36 +694,7 @@ def check_src_path(self, paths):
         """
         src_path = paths[0]
         dir_op = self.parameters['dir_op']
-        if src_path.startswith('s3://'):
-            if self.cmd in ['mb', 'rb']:
-                return
-            session = self.session
-            service = session.get_service('s3')
-            endpoint = service.get_endpoint(self.parameters['region'])
-            src_path = src_path[5:]
-            if dir_op:
-                if not src_path.endswith('/'):
-                    src_path += '/'  # all prefixes must end with a /
-            bucket, key = find_bucket_key(src_path)
-            operation = service.get_operation('ListObjects')
-            response_data = operation.call(endpoint, bucket=bucket, prefix=key,
-                                           delimiter='/')[1]
-            check_error(response_data)
-            contents = response_data['Contents']
-            common_prefixes = response_data['CommonPrefixes']
-            if not dir_op:
-                if contents:
-                    if contents[0]['Key'] == key:
-                        pass
-                    else:
-                        raise Exception("Error: S3 Object does not exist")
-                else:
-                    raise Exception('Error: S3 Object does not exist')
-            else:
-                if not contents and not common_prefixes:
-                    raise Exception('Error: S3 Prefix does not exist')
-
-        else:
+        if not src_path.startswith('s3://'):
             src_path = os.path.abspath(src_path)
             if os.path.exists(src_path):
                 if os.path.isdir(src_path) and not dir_op:

awscli/customizations/s3/tasks.py

@@ -70,6 +70,7 @@ def _execute_task(self, attempts, last_error=''):
             self._queue_print_message(self.filename, failed=True,
                                       dryrun=self.parameters['dryrun'],
                                       error_message=last_error)
+            return
         filename = self.filename
         try:
             if not self.parameters['dryrun']:
@@ -99,7 +100,7 @@ def _queue_print_message(self, filename, failed, dryrun,
                 message = print_operation(filename, failed,
                                           self.parameters['dryrun'])
                 if error_message is not None:
-                    message += '\n' + error_message
+                    message += ' ' + error_message
                 result = {'message': message, 'error': failed}
                 self.result_queue.put(result)
         except Exception as e: