diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index 824d537996b0..2e53787e33e1 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -17,7 +17,6 @@ from dateutil.parser import parse from dateutil.tz import tzlocal -from awscli.customizations.s3.fileinfo import FileInfo from awscli.customizations.s3.utils import find_bucket_key, get_file_stat from awscli.customizations.s3.utils import BucketLister from awscli.errorhandler import ClientError @@ -46,6 +45,20 @@ def __init__(self, directory, filename): super(FileDecodingError, self).__init__(self.error_message) +class FileBase(object): + def __init__(self, src, dest=None, compare_key=None, size=None, + last_update=None, src_type=None, dest_type=None, + operation_name=None): + self.src = src + self.dest = dest + self.compare_key = compare_key + self.size = size + self.last_update = last_update + self.src_type = src_type + self.dest_type = dest_type + self.operation_name = operation_name + + class FileGenerator(object): """ This is a class the creates a generator to yield files based on information @@ -55,12 +68,9 @@ class FileGenerator(object): ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``. """ def __init__(self, service, endpoint, operation_name, - follow_symlinks=True, source_endpoint=None): + follow_symlinks=True): self._service = service self._endpoint = endpoint - self._source_endpoint = endpoint - if source_endpoint: - self._source_endpoint = source_endpoint self.operation_name = operation_name self.follow_symlinks = follow_symlinks @@ -90,13 +100,11 @@ def call(self, files): sep_table[dest_type]) else: dest_path = dest['path'] - yield FileInfo(src=src_path, dest=dest_path, + yield FileBase(src=src_path, dest=dest_path, compare_key=compare_key, size=size, last_update=last_update, src_type=src_type, - service=self._service, endpoint=self._endpoint, dest_type=dest_type, - operation_name=self.operation_name, - source_endpoint=self._source_endpoint) + operation_name=self.operation_name) def list_files(self, path, dir_op): """ @@ -195,7 +203,7 @@ def list_objects(self, s3_path, dir_op): yield self._list_single_object(s3_path) else: operation = self._service.get_operation('ListObjects') - lister = BucketLister(operation, self._source_endpoint) + lister = BucketLister(operation, self._endpoint) for key in lister.list_objects(bucket=bucket, prefix=prefix): source_path, size, last_update = key if size == 0 and source_path.endswith('/'): @@ -221,7 +229,7 @@ def _list_single_object(self, s3_path): operation = self._service.get_operation('HeadObject') try: response = operation.call( - self._source_endpoint, bucket=bucket, key=key)[1] + self._endpoint, bucket=bucket, key=key)[1] except ClientError as e: # We want to try to give a more helpful error message. # This is what the customer is going to see so we want to diff --git a/awscli/customizations/s3/infosetter.py b/awscli/customizations/s3/infosetter.py new file mode 100644 index 000000000000..bb4c69909d88 --- /dev/null +++ b/awscli/customizations/s3/infosetter.py @@ -0,0 +1,49 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from awscli.customizations.s3.fileinfo import FileInfo + + +class InfoSetter(object): + """ + This class takes a ``FileBase`` object's attributes and generates + a ``FileInfo`` object so that the operation can be performed. + """ + def __init__(self, service, endpoint, source_endpoint=None, + parameters = None): + self._service = service + self._endpoint = endpoint + self._source_endpoint = endpoint + if source_endpoint: + self._source_endpoint = source_endpoint + self._parameters = parameters + + def call(self, files): + for file_base in files: + file_info = self.inject_info(file_base) + yield file_info + + def inject_info(self, file_base): + file_info_attr = {} + file_info_attr['src'] = file_base.src + file_info_attr['dest'] = file_base.dest + file_info_attr['compare_key'] = file_base.compare_key + file_info_attr['size'] = file_base.size + file_info_attr['last_update'] = file_base.last_update + file_info_attr['src_type'] = file_base.src_type + file_info_attr['dest_type'] = file_base.dest_type + file_info_attr['operation_name'] = file_base.operation_name + file_info_attr['service'] = self._service + file_info_attr['endpoint'] = self._endpoint + file_info_attr['source_endpoint'] = self._source_endpoint + file_info_attr['parameters'] = self._parameters + return FileInfo(**file_info_attr) diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 98f9b0865bf9..ec68c312adc4 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -19,6 +19,7 @@ from awscli.customizations.commands import BasicCommand from awscli.customizations.s3.comparator import Comparator +from awscli.customizations.s3.infosetter import InfoSetter from awscli.customizations.s3.fileformat import FileFormat from awscli.customizations.s3.filegenerator import FileGenerator from awscli.customizations.s3.fileinfo import TaskInfo @@ -486,6 +487,7 @@ def set_endpoints(self): endpoint_url=self.parameters['endpoint_url'], verify=self.parameters['verify_ssl'] ) + self._source_endpoint = self._endpoint if self.parameters['source_region']: if self.parameters['paths_type'] == 's3s3': self._source_endpoint = get_endpoint( @@ -509,6 +511,8 @@ def create_instructions(self): self.instructions.append('filters') if self.cmd == 'sync': self.instructions.append('comparator') + if self.cmd not in ['mb', 'rb']: + self.instructions.append('info_setter') self.instructions.append('s3_handler') def run(self): @@ -551,10 +555,10 @@ def run(self): 'rb': 'remove_bucket' } operation_name = cmd_translation[paths_type][self.cmd] - file_generator = FileGenerator(self._service, self._endpoint, + file_generator = FileGenerator(self._service, + self._source_endpoint, operation_name, - self.parameters['follow_symlinks'], - self._source_endpoint) + self.parameters['follow_symlinks']) rev_generator = FileGenerator(self._service, self._endpoint, '', self.parameters['follow_symlinks']) taskinfo = [TaskInfo(src=files['src']['path'], @@ -562,6 +566,8 @@ def run(self): operation_name=operation_name, service=self._service, endpoint=self._endpoint)] + info_setter = InfoSetter(self._service, self._endpoint, + self._source_endpoint, self.parameters) s3handler = S3Handler(self.session, self.parameters) command_dict = {} @@ -572,21 +578,25 @@ def run(self): 'filters': [create_filter(self.parameters), create_filter(self.parameters)], 'comparator': [Comparator(self.parameters)], + 'info_setter': [info_setter], 's3_handler': [s3handler]} elif self.cmd == 'cp': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], + 'info_setter': [info_setter], 's3_handler': [s3handler]} elif self.cmd == 'rm': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], + 'info_setter': [info_setter], 's3_handler': [s3handler]} elif self.cmd == 'mv': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], + 'info_setter': [info_setter], 's3_handler': [s3handler]} elif self.cmd == 'mb': command_dict = {'setup': [taskinfo], diff --git a/tests/integration/customizations/s3/test_filegenerator.py b/tests/integration/customizations/s3/test_filegenerator.py index 23b758ea35ea..59353fc3d39b 100644 --- a/tests/integration/customizations/s3/test_filegenerator.py +++ b/tests/integration/customizations/s3/test_filegenerator.py @@ -22,8 +22,7 @@ import botocore.session from awscli import EnvironmentVariables -from awscli.customizations.s3.filegenerator import FileGenerator -from awscli.customizations.s3.fileinfo import FileInfo +from awscli.customizations.s3.filegenerator import FileGenerator, FileBase from tests.unit.customizations.s3 import make_s3_files, s3_cleanup, \ compare_files @@ -52,16 +51,14 @@ def test_s3_file(self): result_list = list( FileGenerator(self.service, self.endpoint, '').call( input_s3_file)) - file_info = FileInfo(src=self.file1, dest='text1.txt', + file_base = FileBase(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=expected_file_size, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='', - endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='local', operation_name='') - expected_list = [file_info] + expected_list = [file_base] self.assertEqual(len(result_list), 1) compare_files(self, result_list[0], expected_list[0]) @@ -77,26 +74,22 @@ def test_s3_directory(self): result_list = list( FileGenerator(self.service, self.endpoint, '').call( input_s3_file)) - file_info = FileInfo(src=self.file2, + file_base = FileBase(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='', - endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info2 = FileInfo(src=self.file1, + dest_type='local', operation_name='') + file_base2 = FileBase(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=15, last_update=result_list[1].last_update, src_type='s3', - dest_type='local', operation_name='', - endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='local', operation_name='') - expected_result = [file_info, file_info2] + expected_result = [file_base, file_base2] self.assertEqual(len(result_list), 2) compare_files(self, result_list[0], expected_result[0]) compare_files(self, result_list[1], expected_result[1]) @@ -115,40 +108,32 @@ def test_s3_delete_directory(self): 'delete').call( input_s3_file)) - file_info1 = FileInfo( + file_base1 = FileBase( src=self.bucket + '/another_directory/', dest='another_directory' + os.sep, compare_key='another_directory/', size=0, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info2 = FileInfo( + dest_type='local', operation_name='delete') + file_base2 = FileBase( src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[1].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, - endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info3 = FileInfo( + dest_type='local', operation_name='delete') + file_base3 = FileBase( src=self.file1, dest='text1.txt', compare_key='text1.txt', size=15, last_update=result_list[2].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, - endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='local', operation_name='delete') - expected_list = [file_info1, file_info2, file_info3] + expected_list = [file_base1, file_base2, file_base3] self.assertEqual(len(result_list), 3) compare_files(self, result_list[0], expected_list[0]) compare_files(self, result_list[1], expected_list[1]) diff --git a/tests/unit/customizations/s3/__init__.py b/tests/unit/customizations/s3/__init__.py index 4c4f995407d2..3743c757f1de 100644 --- a/tests/unit/customizations/s3/__init__.py +++ b/tests/unit/customizations/s3/__init__.py @@ -150,7 +150,7 @@ def s3_cleanup(bucket, session, key1='text1.txt', key2='text2.txt'): def compare_files(self, result_file, ref_file): """ - Ensures that the FileInfo's properties are what they + Ensures that the FileBase's properties are what they are suppose to be. """ self.assertEqual(result_file.src, ref_file.src) @@ -161,16 +161,6 @@ def compare_files(self, result_file, ref_file): self.assertEqual(result_file.src_type, ref_file.src_type) self.assertEqual(result_file.dest_type, ref_file.dest_type) self.assertEqual(result_file.operation_name, ref_file.operation_name) - compare_endpoints(self, result_file.endpoint, ref_file.endpoint) - compare_endpoints(self, result_file.source_endpoint, - ref_file.source_endpoint) - - -def compare_endpoints(self, endpoint, ref_endpoint): - self.assertEqual(endpoint.region_name, ref_endpoint.region_name) - if getattr(endpoint, 'endpoint_url', None): - self.assertEqual(endpoint.endpoint_url, ref_endpoint.endpoint_url) - self.assertEqual(endpoint.verify, ref_endpoint.verify) def list_contents(bucket, session): diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index d0d5652d1bc9..45d5cf5e6960 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -14,7 +14,7 @@ import unittest from awscli.customizations.s3.comparator import Comparator -from awscli.customizations.s3.fileinfo import FileInfo +from awscli.customizations.s3.filegenerator import FileBase class ComparatorTest(unittest.TestCase): @@ -30,16 +30,14 @@ def test_compare_key_equal(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='upload') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -56,16 +54,14 @@ def test_compare_size(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=11, last_update=time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='upload') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -84,16 +80,14 @@ def test_compare_lastmod_upload(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='upload') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -112,16 +106,14 @@ def test_compare_lastmod_copy(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', - dest_type='s3', operation_name='copy', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='copy') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -140,16 +132,14 @@ def test_compare_lastmod_download(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='local', operation_name='download') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') src_files.append(src_file) dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -159,16 +149,14 @@ def test_compare_lastmod_download(self): self.assertEqual(result_list, ref_list) # If the source is newer than the destination do not download. - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='local', operation_name='download') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') src_files = [] dest_files = [] src_files.append(src_file) @@ -189,16 +177,14 @@ def test_compare_key_less(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='bomparator_test.py', size=10, last_update=time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='upload') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) dest_file.operation = 'delete' @@ -219,16 +205,14 @@ def test_compare_key_greater(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) - dest_file = FileInfo(src='', dest='', + dest_type='s3', operation_name='upload') + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) src_file.operation = 'upload' @@ -250,11 +234,10 @@ def test_empty_src(self): ref_list = [] result_list = [] time = datetime.datetime.now() - dest_file = FileInfo(src='', dest='', + dest_file = FileBase(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') dest_files.append(dest_file) dest_file.operation = 'delete' ref_list.append(dest_file) @@ -273,11 +256,10 @@ def test_empty_dest(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) + dest_type='s3', operation_name='upload') src_files.append(src_file) ref_list.append(src_file) files = self.comparator.call(iter(src_files), iter(dest_files)) @@ -312,17 +294,15 @@ def test_compare_size_only_dest_older_than_src(self): time_src = datetime.datetime.now() time_dst = time_src + datetime.timedelta(days=1) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) + dest_type='s3', operation_name='upload') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 0) @@ -335,17 +315,15 @@ def test_compare_size_only_src_older_than_dest(self): time_dst = datetime.datetime.now() time_src = time_dst + datetime.timedelta(days=1) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', - dest_type='s3', operation_name='upload', - service=None, endpoint=None) + dest_type='s3', operation_name='upload') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', - dest_type='local', operation_name='', - service=None, endpoint=None) + dest_type='local', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 0) @@ -364,17 +342,15 @@ def test_compare_exact_timestamps_dest_older(self): time_src = datetime.datetime.now() time_dst = time_src - datetime.timedelta(days=1) - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) + dest_type='local', operation_name='download') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 1) @@ -388,17 +364,15 @@ def test_compare_exact_timestamps_src_older(self): time_src = datetime.datetime.now() - datetime.timedelta(days=1) time_dst = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) + dest_type='local', operation_name='download') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 1) @@ -411,17 +385,15 @@ def test_compare_exact_timestamps_same_age_same_size(self): """ time_both = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) + dest_type='local', operation_name='download') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 0) @@ -434,17 +406,15 @@ def test_compare_exact_timestamps_same_age_diff_size(self): """ time_both = datetime.datetime.now() - src_file = FileInfo(src='', dest='', + src_file = FileBase(src='', dest='', compare_key='test.py', size=20, last_update=time_both, src_type='s3', - dest_type='local', operation_name='download', - service=None, endpoint=None) + dest_type='local', operation_name='download') - dst_file = FileInfo(src='', dest='', + dst_file = FileBase(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', - dest_type='s3', operation_name='', - service=None, endpoint=None) + dest_type='s3', operation_name='') files = self.comparator.call(iter([src_file]), iter([dst_file])) self.assertEqual(sum(1 for _ in files), 1) diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py index 56ebc2d3754e..8ede65de6536 100644 --- a/tests/unit/customizations/s3/test_filegenerator.py +++ b/tests/unit/customizations/s3/test_filegenerator.py @@ -20,8 +20,7 @@ import mock from awscli.customizations.s3.filegenerator import FileGenerator, \ - FileDecodingError -from awscli.customizations.s3.fileinfo import FileInfo + FileDecodingError, FileBase from awscli.customizations.s3.utils import get_file_stat import botocore.session from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ @@ -61,13 +60,11 @@ def test_local_file(self): for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) - file_info = FileInfo(src=self.local_file, dest='bucket/text1.txt', + file_base = FileBase(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', - dest_type='s3', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - ref_list = [file_info] + dest_type='s3', operation_name='') + ref_list = [file_base] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -88,24 +85,20 @@ def test_local_directory(self): for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) - file_info = FileInfo(src=self.local_file, dest='bucket/text1.txt', + file_base = FileBase(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', - dest_type='s3', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='s3', operation_name='') path = self.local_dir + 'another_directory' + os.sep \ + 'text2.txt' size, last_update = get_file_stat(path) - file_info2 = FileInfo(src=path, + file_base2 = FileBase(src=path, dest='bucket/another_directory/text2.txt', compare_key='another_directory/text2.txt', size=size, last_update=last_update, src_type='local', - dest_type='s3', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - ref_list = [file_info2, file_info] + dest_type='s3', operation_name='') + ref_list = [file_base2, file_base] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -224,12 +217,12 @@ def test_no_follow_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_infos = FileGenerator(self.service, self.endpoint, + file_bases = FileGenerator(self.service, self.endpoint, '', False).call(input_local_dir) self.filenames.sort() result_list = [] - for file_info in file_infos: - result_list.append(getattr(file_info, 'src')) + for file_base in file_bases: + result_list.append(getattr(file_base, 'src')) self.assertEqual(len(result_list), len(self.filenames)) # Just check to make sure the right local files are generated. for i in range(len(result_list)): @@ -246,13 +239,13 @@ def test_follow_bad_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_infos = FileGenerator(self.service, self.endpoint, + file_bases = FileGenerator(self.service, self.endpoint, '', True).call(input_local_dir) result_list = [] rc = 0 try: - for file_info in file_infos: - result_list.append(getattr(file_info, 'src')) + for file_base in file_bases: + result_list.append(getattr(file_base, 'src')) rc = 1 except OSError as e: pass @@ -271,13 +264,13 @@ def test_follow_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_infos = FileGenerator(self.service, self.endpoint, + file_bases = FileGenerator(self.service, self.endpoint, '', True).call(input_local_dir) all_filenames = self.filenames + self.symlink_files all_filenames.sort() result_list = [] - for file_info in file_infos: - result_list.append(getattr(file_info, 'src')) + for file_base in file_bases: + result_list.append(getattr(file_base, 'src')) self.assertEqual(len(result_list), len(all_filenames)) # Just check to make sure the right local files are generated. for i in range(len(result_list)): @@ -379,7 +372,6 @@ def setUp(self): self.file2 = self.bucket + '/' + 'another_directory/text2.txt' self.service = self.session.get_service('s3') self.endpoint = self.service.get_endpoint('us-east-1') - self.source_endpoint = self.service.get_endpoint('us-west-1') def tearDown(self): s3_cleanup(self.bucket, self.session) @@ -393,22 +385,19 @@ def test_s3_file(self): 'dest': {'path': 'text1.txt', 'type': 'local'}, 'dir_op': False, 'use_src_name': False} params = {'region': 'us-east-1'} - file_gen = FileGenerator(self.service, self.endpoint, - '', source_endpoint=self.source_endpoint) + file_gen = FileGenerator(self.service, self.endpoint, '') files = file_gen.call(input_s3_file) result_list = [] for filename in files: result_list.append(filename) - file_info = FileInfo(src=self.file1, dest='text1.txt', + file_base = FileBase(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.source_endpoint) + dest_type='local', operation_name='') - ref_list = [file_info] + ref_list = [file_base] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -428,27 +417,23 @@ def test_s3_directory(self): result_list = [] for filename in files: result_list.append(filename) - file_info = FileInfo(src=self.file2, + file_base = FileBase(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info2 = FileInfo(src=self.file1, + dest_type='local', operation_name='') + file_base2 = FileBase(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[1].size, last_update=result_list[1].last_update, src_type='s3', - dest_type='local', operation_name='', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='local', operation_name='') - ref_list = [file_info, file_info2] + ref_list = [file_base, file_base2] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -468,35 +453,29 @@ def test_s3_delete_directory(self): for filename in files: result_list.append(filename) - file_info1 = FileInfo(src=self.bucket + '/another_directory/', + file_base1 = FileBase(src=self.bucket + '/another_directory/', dest='another_directory' + os.sep, compare_key='another_directory/', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info2 = FileInfo(src=self.file2, + dest_type='local', operation_name='delete') + file_base2 = FileBase(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=result_list[1].size, last_update=result_list[1].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) - file_info3 = FileInfo(src=self.file1, + dest_type='local', operation_name='delete') + file_base3 = FileBase(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[2].size, last_update=result_list[2].last_update, src_type='s3', - dest_type='local', operation_name='delete', - service=self.service, endpoint=self.endpoint, - source_endpoint=self.endpoint) + dest_type='local', operation_name='delete') - ref_list = [file_info1, file_info2, file_info3] + ref_list = [file_base1, file_base2, file_base3] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) diff --git a/tests/unit/customizations/s3/test_filters.py b/tests/unit/customizations/s3/test_filters.py index a7f19bb05c2d..b3f49e5029b5 100644 --- a/tests/unit/customizations/s3/test_filters.py +++ b/tests/unit/customizations/s3/test_filters.py @@ -14,7 +14,7 @@ from awscli.testutils import unittest import platform -from awscli.customizations.s3.fileinfo import FileInfo +from awscli.customizations.s3.filegenerator import FileBase from awscli.customizations.s3.filters import Filter @@ -29,27 +29,26 @@ def platform_path(filepath): class FiltersTest(unittest.TestCase): def setUp(self): self.local_files = [ - self.file_info('test.txt'), - self.file_info('test.jpg'), - self.file_info(os.path.join('directory', 'test.jpg')), + self.file_base('test.txt'), + self.file_base('test.jpg'), + self.file_base(os.path.join('directory', 'test.jpg')), ] self.s3_files = [ - self.file_info('bucket/test.txt', src_type='s3'), - self.file_info('bucket/test.jpg', src_type='s3'), - self.file_info('bucket/key/test.jpg', src_type='s3'), + self.file_base('bucket/test.txt', src_type='s3'), + self.file_base('bucket/test.jpg', src_type='s3'), + self.file_base('bucket/key/test.jpg', src_type='s3'), ] - def file_info(self, filename, src_type='local'): + def file_base(self, filename, src_type='local'): if src_type == 'local': filename = os.path.abspath(filename) dest_type = 's3' else: dest_type = 'local' - return FileInfo(src=filename, dest='', + return FileBase(src=filename, dest='', compare_key='', size=10, last_update=0, src_type=src_type, - dest_type=dest_type, operation_name='', - service=None, endpoint=None) + dest_type=dest_type, operation_name='') def create_filter(self, filters=None, root=None, dst_root=None): if root is None: @@ -117,15 +116,15 @@ def test_prefix_filtering_consistent(self): # The same filter should work for both local and remote files. # So if I have a directory with 2 files: local_files = [ - self.file_info('test1.txt'), - self.file_info('nottest1.txt'), + self.file_base('test1.txt'), + self.file_base('nottest1.txt'), ] - # And the same 2 files remote (note that the way FileInfo objects + # And the same 2 files remote (note that the way FileBase objects # are constructed, we'll have the bucket name but no leading '/' # character): remote_files = [ - self.file_info('bucket/test1.txt', src_type='s3'), - self.file_info('bucket/nottest1.txt', src_type='s3'), + self.file_base('bucket/test1.txt', src_type='s3'), + self.file_base('bucket/nottest1.txt', src_type='s3'), ] # If I apply the filter to the local to the local files. exclude_filter = self.create_filter([['exclude', 't*']]) @@ -144,9 +143,9 @@ def test_prefix_filtering_consistent(self): def test_bucket_exclude_with_prefix(self): s3_files = [ - self.file_info('bucket/dir1/key1.txt', src_type='s3'), - self.file_info('bucket/dir1/key2.txt', src_type='s3'), - self.file_info('bucket/dir1/notkey3.txt', src_type='s3'), + self.file_base('bucket/dir1/key1.txt', src_type='s3'), + self.file_base('bucket/dir1/key2.txt', src_type='s3'), + self.file_base('bucket/dir1/notkey3.txt', src_type='s3'), ] filtered_files = list( self.create_filter([['exclude', 'dir1/*']], @@ -161,7 +160,7 @@ def test_bucket_exclude_with_prefix(self): def test_root_dir(self): p = platform_path - local_files = [self.file_info(p('/foo/bar/baz.txt'), src_type='local')] + local_files = [self.file_base(p('/foo/bar/baz.txt'), src_type='local')] local_filter = self.create_filter([['exclude', 'baz.txt']], root=p('/foo/bar/')) filtered = list(local_filter.call(local_files)) diff --git a/tests/unit/customizations/s3/test_infosetter.py b/tests/unit/customizations/s3/test_infosetter.py new file mode 100644 index 000000000000..2cf197a07fe2 --- /dev/null +++ b/tests/unit/customizations/s3/test_infosetter.py @@ -0,0 +1,38 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import mock + +from awscli.testutils import unittest +from awscli.customizations.s3.filegenerator import FileBase +from awscli.customizations.s3.fileinfo import FileInfo +from awscli.customizations.s3.infosetter import InfoSetter + + +class TestInfoSetter(unittest.TestCase): + def test_info_setter(self): + info_setter = InfoSetter(service='service', endpoint='endpoint', + source_endpoint='source_endpoint', + parameters='parameters') + files = [FileBase(src='src', dest='dest', compare_key='compare_key', + size='size', last_update='last_update', + src_type='src_type', dest_type='dest_type', + operation_name='operation_name')] + file_infos = info_setter.call(files) + for file_info in file_infos: + attributes = file_info.__dict__.keys() + for key in attributes: + self.assertEqual(getattr(file_info, key), str(key)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index 05a230a1a7de..9ce310586daf 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -132,7 +132,7 @@ def test_set_endpoint_no_source(self): endpoint = cmd_arc._endpoint source_endpoint = cmd_arc._source_endpoint self.assertEqual(endpoint.region_name, 'us-west-1') - self.assertEqual(source_endpoint, None) + self.assertEqual(source_endpoint.region_name, 'us-west-1') def test_set_endpoint_with_source(self): cmd_arc = CommandArchitecture(self.session, 'sync', @@ -154,10 +154,11 @@ def test_create_instructions(self): """ cmds = ['cp', 'mv', 'rm', 'sync', 'mb', 'rb'] - instructions = {'cp': ['file_generator', 's3_handler'], - 'mv': ['file_generator', 's3_handler'], - 'rm': ['file_generator', 's3_handler'], - 'sync': ['file_generator', 'comparator', 's3_handler'], + instructions = {'cp': ['file_generator', 'info_setter', 's3_handler'], + 'mv': ['file_generator', 'info_setter', 's3_handler'], + 'rm': ['file_generator', 'info_setter', 's3_handler'], + 'sync': ['file_generator', 'comparator', + 'info_setter', 's3_handler'], 'mb': ['s3_handler'], 'rb': ['s3_handler']} @@ -175,7 +176,7 @@ def test_create_instructions(self): cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, ['file_generator', 'filters', - 's3_handler']) + 'info_setter', 's3_handler']) def test_run_cp_put(self): # This ensures that the architecture sets up correctly for a ``cp`` put