Added a --source-region parameter.

awscli/customizations/s3/filegenerator.py

@@ -17,7 +17,6 @@
 from dateutil.parser import parse
 from dateutil.tz import tzlocal
 
-from awscli.customizations.s3.fileinfo import FileInfo
 from awscli.customizations.s3.utils import find_bucket_key, get_file_stat
 from awscli.customizations.s3.utils import BucketLister
 from awscli.errorhandler import ClientError
@@ -46,6 +45,20 @@ def __init__(self, directory, filename):
         super(FileDecodingError, self).__init__(self.error_message)
 
 
+class FileStat(object):
+    def __init__(self, src, dest=None, compare_key=None, size=None,
+                 last_update=None, src_type=None, dest_type=None,
+                 operation_name=None):
+        self.src = src
+        self.dest = dest
+        self.compare_key = compare_key
+        self.size = size
+        self.last_update = last_update
+        self.src_type = src_type
+        self.dest_type = dest_type
+        self.operation_name = operation_name
+
+
 class FileGenerator(object):
     """
     This is a class the creates a generator to yield files based on information
@@ -54,7 +67,8 @@ class FileGenerator(object):
     under the same common prefix.  The generator yields corresponding
     ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
     """
-    def __init__(self, service, endpoint, operation_name, follow_symlinks=True):
+    def __init__(self, service, endpoint, operation_name,
+                 follow_symlinks=True):
         self._service = service
         self._endpoint = endpoint
         self.operation_name = operation_name
@@ -86,10 +100,9 @@ def call(self, files):
                                               sep_table[dest_type])
             else:
                 dest_path = dest['path']
-            yield FileInfo(src=src_path, dest=dest_path,
+            yield FileStat(src=src_path, dest=dest_path,
                            compare_key=compare_key, size=size,
                            last_update=last_update, src_type=src_type,
-                           service=self._service, endpoint=self._endpoint,
                            dest_type=dest_type,
                            operation_name=self.operation_name)
 

awscli/customizations/s3/fileinfo.py

@@ -140,7 +140,7 @@ class FileInfo(TaskInfo):
     def __init__(self, src, dest=None, compare_key=None, size=None,
                  last_update=None, src_type=None, dest_type=None,
                  operation_name=None, service=None, endpoint=None,
-                 parameters=None):
+                 parameters=None, source_endpoint=None):
         super(FileInfo, self).__init__(src, src_type=src_type,
                                        operation_name=operation_name,
                                        service=service,
@@ -156,6 +156,7 @@ def __init__(self, src, dest=None, compare_key=None, size=None,
         else:
             self.parameters = {'acl': None,
                                'sse': None}
+        self.source_endpoint = source_endpoint
 
     def _permission_to_param(self, permission):
         if permission == 'read':
@@ -256,7 +257,8 @@ def delete(self):
         """
         if (self.src_type == 's3'):
             bucket, key = find_bucket_key(self.src)
-            params = {'endpoint': self.endpoint, 'bucket': bucket, 'key': key}
+            params = {'endpoint': self.source_endpoint, 'bucket': bucket,
+                      'key': key}
             response_data, http = operate(self.service, 'DeleteObject',
                                           params)
         else:

awscli/customizations/s3/fileinfobuilder.py

@@ -0,0 +1,49 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from awscli.customizations.s3.fileinfo import FileInfo
+
+
+class FileInfoBuilder(object):
+    """
+    This class takes a ``FileBase`` object's attributes and generates
+    a ``FileInfo`` object so that the operation can be performed.
+    """
+    def __init__(self, service, endpoint, source_endpoint=None,
+                 parameters = None):
+        self._service = service
+        self._endpoint = endpoint
+        self._source_endpoint = endpoint
+        if source_endpoint:
+            self._source_endpoint = source_endpoint
+        self._parameters = parameters 
+
+    def call(self, files):
+        for file_base in files:
+            file_info = self._inject_info(file_base)
+            yield file_info            
+
+    def _inject_info(self, file_base):
+        file_info_attr = {}
+        file_info_attr['src'] = file_base.src
+        file_info_attr['dest'] = file_base.dest
+        file_info_attr['compare_key'] = file_base.compare_key
+        file_info_attr['size'] = file_base.size
+        file_info_attr['last_update'] = file_base.last_update
+        file_info_attr['src_type'] = file_base.src_type
+        file_info_attr['dest_type'] = file_base.dest_type
+        file_info_attr['operation_name'] = file_base.operation_name
+        file_info_attr['service'] = self._service
+        file_info_attr['endpoint'] = self._endpoint
+        file_info_attr['source_endpoint'] = self._source_endpoint
+        file_info_attr['parameters'] = self._parameters
+        return FileInfo(**file_info_attr)

awscli/customizations/s3/subcommands.py

@@ -19,6 +19,7 @@
 
 from awscli.customizations.commands import BasicCommand
 from awscli.customizations.s3.comparator import Comparator
+from awscli.customizations.s3.fileinfobuilder import FileInfoBuilder
 from awscli.customizations.s3.fileformat import FileFormat
 from awscli.customizations.s3.filegenerator import FileGenerator
 from awscli.customizations.s3.fileinfo import TaskInfo
@@ -163,6 +164,16 @@
 CONTENT_LANGUAGE = {'name': 'content-language', 'nargs': 1,
                     'help_text': ("The language the content is in.")}
 
+SOURCE_REGION = {'name': 'source-region', 'nargs': 1,
+                 'help_text': (
+                     "When transferring objects from an s3 bucket to an s3 "
+                     "bucket, this specifies the region of the source bucket."
+                     " Note the region specified by ``--region`` or through "
+                     "configuration of the CLI refers to the region of the "
+                     "destination bucket.  If ``--source-region`` is not "
+                     "specified the region of the source will be the same "
+                     "as the region of the destination bucket.")}
+
 EXPIRES = {'name': 'expires', 'nargs': 1, 'help_text': ("The date and time at "
            "which the object is no longer cacheable.")}
 
@@ -198,20 +209,22 @@
                  FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
                  SSE, STORAGE_CLASS, GRANTS, WEBSITE_REDIRECT, CONTENT_TYPE,
                  CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING,
-                 CONTENT_LANGUAGE, EXPIRES]
+                 CONTENT_LANGUAGE, EXPIRES, SOURCE_REGION]
 
 SYNC_ARGS = [DELETE, EXACT_TIMESTAMPS, SIZE_ONLY] + TRANSFER_ARGS
 
 
+def get_endpoint(service, region, endpoint_url, verify):
+    return service.get_endpoint(region_name=region, endpoint_url=endpoint_url,
+                                verify=verify)
+
+
 class S3Command(BasicCommand):
     def _run_main(self, parsed_args, parsed_globals):
         self.service = self._session.get_service('s3')
-        self.endpoint = self._get_endpoint(self.service, parsed_globals)
-
-    def _get_endpoint(self, service, parsed_globals):
-        return service.get_endpoint(region_name=parsed_globals.region,
-                                    endpoint_url=parsed_globals.endpoint_url,
-                                    verify=parsed_globals.verify_ssl)
+        self.endpoint = get_endpoint(self.service, parsed_globals.region,
+                                     parsed_globals.endpoint_url,
+                                     parsed_globals.verify_ssl)
 
 
 class ListCommand(S3Command):
@@ -363,6 +376,7 @@ def _run_main(self, parsed_args, parsed_globals):
         cmd_params.check_force(parsed_globals)
         cmd = CommandArchitecture(self._session, self.NAME,
                                   cmd_params.parameters)
+        cmd.set_endpoints()
         cmd.create_instructions()
         return cmd.run()
 
@@ -463,10 +477,25 @@ def __init__(self, session, cmd, parameters):
         self.parameters = parameters
         self.instructions = []
         self._service = self.session.get_service('s3')
-        self._endpoint = self._service.get_endpoint(
-            region_name=self.parameters['region'],
+        self._endpoint = None
+        self._source_endpoint = None
+
+    def set_endpoints(self):
+        self._endpoint = get_endpoint(
+            self._service,
+            region=self.parameters['region'],
             endpoint_url=self.parameters['endpoint_url'],
-            verify=self.parameters['verify_ssl'])
+            verify=self.parameters['verify_ssl']
+        )
+        self._source_endpoint = self._endpoint
+        if self.parameters['source_region']:
+            if self.parameters['paths_type'] == 's3s3':
+                self._source_endpoint = get_endpoint(
+                    self._service,
+                    region=self.parameters['source_region'][0],
+                    endpoint_url=None,
+                    verify=self.parameters['verify_ssl']
+                )
 
     def create_instructions(self):
         """
@@ -482,6 +511,8 @@ def create_instructions(self):
             self.instructions.append('filters')
         if self.cmd == 'sync':
             self.instructions.append('comparator')
+        if self.cmd not in ['mb', 'rb']:
+            self.instructions.append('file_info_builder')
         self.instructions.append('s3_handler')
 
     def run(self):
@@ -524,7 +555,8 @@ def run(self):
             'rb': 'remove_bucket'
         }
         operation_name = cmd_translation[paths_type][self.cmd]
-        file_generator = FileGenerator(self._service, self._endpoint,
+        file_generator = FileGenerator(self._service,
+                                       self._source_endpoint,
                                        operation_name,
                                        self.parameters['follow_symlinks'])
         rev_generator = FileGenerator(self._service, self._endpoint, '',
@@ -534,6 +566,8 @@ def run(self):
                              operation_name=operation_name,
                              service=self._service,
                              endpoint=self._endpoint)]
+        file_info_builder = FileInfoBuilder(self._service, self._endpoint,
+                                 self._source_endpoint, self.parameters) 
         s3handler = S3Handler(self.session, self.parameters)
 
         command_dict = {}
@@ -544,21 +578,25 @@ def run(self):
                             'filters': [create_filter(self.parameters),
                                         create_filter(self.parameters)],
                             'comparator': [Comparator(self.parameters)],
+                            'file_info_builder': [file_info_builder],
                             's3_handler': [s3handler]}
         elif self.cmd == 'cp':
             command_dict = {'setup': [files],
                             'file_generator': [file_generator],
                             'filters': [create_filter(self.parameters)],
+                            'file_info_builder': [file_info_builder],
                             's3_handler': [s3handler]}
         elif self.cmd == 'rm':
             command_dict = {'setup': [files],
                             'file_generator': [file_generator],
                             'filters': [create_filter(self.parameters)],
+                            'file_info_builder': [file_info_builder],
                             's3_handler': [s3handler]}
         elif self.cmd == 'mv':
             command_dict = {'setup': [files],
                             'file_generator': [file_generator],
                             'filters': [create_filter(self.parameters)],
+                            'file_info_builder': [file_info_builder],
                             's3_handler': [s3handler]}
         elif self.cmd == 'mb':
             command_dict = {'setup': [taskinfo],
@@ -610,6 +648,8 @@ def __init__(self, session, cmd, parameters, usage):
             self.parameters['dir_op'] = False
         if 'follow_symlinks' not in parameters:
             self.parameters['follow_symlinks'] = True
+        if 'source_region' not in parameters:
+            self.parameters['source_region'] = None
         if self.cmd in ['sync', 'mb', 'rb']:
             self.parameters['dir_op'] = True
 

awscli/testutils.py

@@ -333,6 +333,9 @@ def create_file(self, filename, contents, mtime=None):
             os.makedirs(os.path.dirname(full_path))
         with open(full_path, 'w') as f:
             f.write(contents)
+        current_time = os.path.getmtime(full_path)
+        # Subtract a few years off the last modification date.
+        os.utime(full_path, (current_time, current_time - 100000000))
         if mtime is not None:
             os.utime(full_path, (mtime, mtime))
         return full_path

tests/integration/customizations/s3/test_filegenerator.py

@@ -22,8 +22,7 @@
 
 import botocore.session
 from awscli import EnvironmentVariables
-from awscli.customizations.s3.filegenerator import FileGenerator
-from awscli.customizations.s3.fileinfo import FileInfo
+from awscli.customizations.s3.filegenerator import FileGenerator, FileStat
 from tests.unit.customizations.s3 import make_s3_files, s3_cleanup, \
     compare_files
 
@@ -52,14 +51,14 @@ def test_s3_file(self):
         result_list = list(
             FileGenerator(self.service, self.endpoint, '').call(
                 input_s3_file))
-        file_info = FileInfo(src=self.file1, dest='text1.txt',
+        file_stat = FileStat(src=self.file1, dest='text1.txt',
                              compare_key='text1.txt',
                              size=expected_file_size,
                              last_update=result_list[0].last_update,
                              src_type='s3',
                              dest_type='local', operation_name='')
 
-        expected_list = [file_info]
+        expected_list = [file_stat]
         self.assertEqual(len(result_list), 1)
         compare_files(self, result_list[0], expected_list[0])
 
@@ -75,22 +74,22 @@ def test_s3_directory(self):
         result_list = list(
             FileGenerator(self.service, self.endpoint, '').call(
                 input_s3_file))
-        file_info = FileInfo(src=self.file2,
+        file_stat = FileStat(src=self.file2,
                              dest='another_directory' + os.sep + 'text2.txt',
                              compare_key='another_directory/text2.txt',
                              size=21,
                              last_update=result_list[0].last_update,
                              src_type='s3',
                              dest_type='local', operation_name='')
-        file_info2 = FileInfo(src=self.file1,
+        file_stat2 = FileStat(src=self.file1,
                               dest='text1.txt',
                               compare_key='text1.txt',
                               size=15,
                               last_update=result_list[1].last_update,
                               src_type='s3',
                               dest_type='local', operation_name='')
 
-        expected_result = [file_info, file_info2]
+        expected_result = [file_stat, file_stat2]
         self.assertEqual(len(result_list), 2)
         compare_files(self, result_list[0], expected_result[0])
         compare_files(self, result_list[1], expected_result[1])
@@ -109,37 +108,32 @@ def test_s3_delete_directory(self):
                           'delete').call(
                 input_s3_file))
 
-        file_info1 = FileInfo(
+        file_stat1 = FileStat(
             src=self.bucket + '/another_directory/',
             dest='another_directory' + os.sep,
             compare_key='another_directory/',
             size=0,
             last_update=result_list[0].last_update,
             src_type='s3',
-            dest_type='local', operation_name='delete',
-            service=self.service, endpoint=self.endpoint)
-        file_info2 = FileInfo(
+            dest_type='local', operation_name='delete')
+        file_stat2 = FileStat(
             src=self.file2,
             dest='another_directory' + os.sep + 'text2.txt',
             compare_key='another_directory/text2.txt',
             size=21,
             last_update=result_list[1].last_update,
             src_type='s3',
-            dest_type='local', operation_name='delete',
-            service=self.service,
-            endpoint=self.endpoint)
-        file_info3 = FileInfo(
+            dest_type='local', operation_name='delete')
+        file_stat3 = FileStat(
             src=self.file1,
             dest='text1.txt',
             compare_key='text1.txt',
             size=15,
             last_update=result_list[2].last_update,
             src_type='s3',
-            dest_type='local', operation_name='delete',
-            service=self.service,
-            endpoint=self.endpoint)
+            dest_type='local', operation_name='delete')
 
-        expected_list = [file_info1, file_info2, file_info3]
+        expected_list = [file_stat1, file_stat2, file_stat3]
         self.assertEqual(len(result_list), 3)
         compare_files(self, result_list[0], expected_list[0])
         compare_files(self, result_list[1], expected_list[1])