diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index 2e53787e33e1..7984a8a31e5a 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -45,7 +45,7 @@ def __init__(self, directory, filename): super(FileDecodingError, self).__init__(self.error_message) -class FileBase(object): +class FileStat(object): def __init__(self, src, dest=None, compare_key=None, size=None, last_update=None, src_type=None, dest_type=None, operation_name=None): @@ -100,7 +100,7 @@ def call(self, files): sep_table[dest_type]) else: dest_path = dest['path'] - yield FileBase(src=src_path, dest=dest_path, + yield FileStat(src=src_path, dest=dest_path, compare_key=compare_key, size=size, last_update=last_update, src_type=src_type, dest_type=dest_type, diff --git a/awscli/customizations/s3/infosetter.py b/awscli/customizations/s3/fileinfobuilder.py similarity index 94% rename from awscli/customizations/s3/infosetter.py rename to awscli/customizations/s3/fileinfobuilder.py index bb4c69909d88..8bc2042615ef 100644 --- a/awscli/customizations/s3/infosetter.py +++ b/awscli/customizations/s3/fileinfobuilder.py @@ -13,7 +13,7 @@ from awscli.customizations.s3.fileinfo import FileInfo -class InfoSetter(object): +class FileInfoBuilder(object): """ This class takes a ``FileBase`` object's attributes and generates a ``FileInfo`` object so that the operation can be performed. @@ -29,10 +29,10 @@ def __init__(self, service, endpoint, source_endpoint=None, def call(self, files): for file_base in files: - file_info = self.inject_info(file_base) + file_info = self._inject_info(file_base) yield file_info - def inject_info(self, file_base): + def _inject_info(self, file_base): file_info_attr = {} file_info_attr['src'] = file_base.src file_info_attr['dest'] = file_base.dest diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index ec68c312adc4..a4f9586aa9dc 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -19,7 +19,7 @@ from awscli.customizations.commands import BasicCommand from awscli.customizations.s3.comparator import Comparator -from awscli.customizations.s3.infosetter import InfoSetter +from awscli.customizations.s3.fileinfobuilder import FileInfoBuilder from awscli.customizations.s3.fileformat import FileFormat from awscli.customizations.s3.filegenerator import FileGenerator from awscli.customizations.s3.fileinfo import TaskInfo @@ -512,7 +512,7 @@ def create_instructions(self): if self.cmd == 'sync': self.instructions.append('comparator') if self.cmd not in ['mb', 'rb']: - self.instructions.append('info_setter') + self.instructions.append('file_info_builder') self.instructions.append('s3_handler') def run(self): @@ -566,7 +566,7 @@ def run(self): operation_name=operation_name, service=self._service, endpoint=self._endpoint)] - info_setter = InfoSetter(self._service, self._endpoint, + file_info_builder = FileInfoBuilder(self._service, self._endpoint, self._source_endpoint, self.parameters) s3handler = S3Handler(self.session, self.parameters) @@ -578,25 +578,25 @@ def run(self): 'filters': [create_filter(self.parameters), create_filter(self.parameters)], 'comparator': [Comparator(self.parameters)], - 'info_setter': [info_setter], + 'file_info_builder': [file_info_builder], 's3_handler': [s3handler]} elif self.cmd == 'cp': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], - 'info_setter': [info_setter], + 'file_info_builder': [file_info_builder], 's3_handler': [s3handler]} elif self.cmd == 'rm': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], - 'info_setter': [info_setter], + 'file_info_builder': [file_info_builder], 's3_handler': [s3handler]} elif self.cmd == 'mv': command_dict = {'setup': [files], 'file_generator': [file_generator], 'filters': [create_filter(self.parameters)], - 'info_setter': [info_setter], + 'file_info_builder': [file_info_builder], 's3_handler': [s3handler]} elif self.cmd == 'mb': command_dict = {'setup': [taskinfo], diff --git a/tests/integration/customizations/s3/test_filegenerator.py b/tests/integration/customizations/s3/test_filegenerator.py index 59353fc3d39b..0c3912402533 100644 --- a/tests/integration/customizations/s3/test_filegenerator.py +++ b/tests/integration/customizations/s3/test_filegenerator.py @@ -22,7 +22,7 @@ import botocore.session from awscli import EnvironmentVariables -from awscli.customizations.s3.filegenerator import FileGenerator, FileBase +from awscli.customizations.s3.filegenerator import FileGenerator, FileStat from tests.unit.customizations.s3 import make_s3_files, s3_cleanup, \ compare_files @@ -51,14 +51,14 @@ def test_s3_file(self): result_list = list( FileGenerator(self.service, self.endpoint, '').call( input_s3_file)) - file_base = FileBase(src=self.file1, dest='text1.txt', + file_stat = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=expected_file_size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') - expected_list = [file_base] + expected_list = [file_stat] self.assertEqual(len(result_list), 1) compare_files(self, result_list[0], expected_list[0]) @@ -74,14 +74,14 @@ def test_s3_directory(self): result_list = list( FileGenerator(self.service, self.endpoint, '').call( input_s3_file)) - file_base = FileBase(src=self.file2, + file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=21, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') - file_base2 = FileBase(src=self.file1, + file_stat2 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=15, @@ -89,7 +89,7 @@ def test_s3_directory(self): src_type='s3', dest_type='local', operation_name='') - expected_result = [file_base, file_base2] + expected_result = [file_stat, file_stat2] self.assertEqual(len(result_list), 2) compare_files(self, result_list[0], expected_result[0]) compare_files(self, result_list[1], expected_result[1]) @@ -108,7 +108,7 @@ def test_s3_delete_directory(self): 'delete').call( input_s3_file)) - file_base1 = FileBase( + file_stat1 = FileStat( src=self.bucket + '/another_directory/', dest='another_directory' + os.sep, compare_key='another_directory/', @@ -116,7 +116,7 @@ def test_s3_delete_directory(self): last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='delete') - file_base2 = FileBase( + file_stat2 = FileStat( src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', @@ -124,7 +124,7 @@ def test_s3_delete_directory(self): last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='delete') - file_base3 = FileBase( + file_stat3 = FileStat( src=self.file1, dest='text1.txt', compare_key='text1.txt', @@ -133,7 +133,7 @@ def test_s3_delete_directory(self): src_type='s3', dest_type='local', operation_name='delete') - expected_list = [file_base1, file_base2, file_base3] + expected_list = [file_stat1, file_stat2, file_stat3] self.assertEqual(len(result_list), 3) compare_files(self, result_list[0], expected_list[0]) compare_files(self, result_list[1], expected_list[1]) diff --git a/tests/integration/customizations/s3/test_plugin.py b/tests/integration/customizations/s3/test_plugin.py index b96b3f968a4c..56801f0a9970 100644 --- a/tests/integration/customizations/s3/test_plugin.py +++ b/tests/integration/customizations/s3/test_plugin.py @@ -26,7 +26,8 @@ import botocore.session import six -from awscli.testutils import unittest, aws, FileCreator +from awscli.testutils import unittest, FileCreator +from awscli.testutils import aws as _aws from tests.unit.customizations.s3 import create_bucket as _create_bucket from awscli.customizations.s3 import constants @@ -41,6 +42,14 @@ def cd(directory): os.chdir(original) +def aws(command, collect_memory=False, env_vars=None, wait_for_finish=True): + if not env_vars: + env_vars = os.environ.copy() + env_vars['AWS_DEFAULT_REGION'] = "us-west-2" + return _aws(command, collect_memory=collect_memory, env_vars=env_vars, + wait_for_finish=wait_for_finish) + + class BaseS3CLICommand(unittest.TestCase): """Base class for aws s3 command. @@ -53,7 +62,7 @@ def setUp(self): self.session = botocore.session.get_session() self.service = self.session.get_service('s3') self.regions = {} - self.region = 'us-east-1' + self.region = 'us-west-2' self.endpoint = self.service.get_endpoint(self.region) self.extra_setup() @@ -597,6 +606,9 @@ def test_sync_with_delete_option_with_same_prefix(self): class TestSourceRegion(BaseS3CLICommand): def extra_setup(self): name_comp = [] + # This creates a non DNS compatible bucket name by making two random + # sequences of characters and joining them with a period and + # adding a .com at the end. for i in range(2): name_comp.append(''.join(random.sample(string.ascii_lowercase + string.digits,10))) @@ -607,7 +619,7 @@ def extra_setup(self): string.digits,10))) self.dest_name = '.'.join(name_comp + ['com']) self.src_region = 'us-west-1' - self.dest_region = 'us-west-2' + self.dest_region = 'us-east-1' self.src_bucket = self.create_bucket(self.src_name, self.src_region) self.dest_bucket = self.create_bucket(self.dest_name, self.dest_region) diff --git a/tests/unit/customizations/s3/__init__.py b/tests/unit/customizations/s3/__init__.py index 3743c757f1de..5ada082edd3e 100644 --- a/tests/unit/customizations/s3/__init__.py +++ b/tests/unit/customizations/s3/__init__.py @@ -110,7 +110,7 @@ def create_bucket(session, name=None, region=None): """ service = session.get_service('s3') if not region: - region = 'us-east-1' + region = 'us-west-2' endpoint = service.get_endpoint(region) if name: bucket_name = name @@ -150,7 +150,7 @@ def s3_cleanup(bucket, session, key1='text1.txt', key2='text2.txt'): def compare_files(self, result_file, ref_file): """ - Ensures that the FileBase's properties are what they + Ensures that the FileStat's properties are what they are suppose to be. """ self.assertEqual(result_file.src, ref_file.src) diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index 45d5cf5e6960..b8909b248e82 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -14,7 +14,7 @@ import unittest from awscli.customizations.s3.comparator import Comparator -from awscli.customizations.s3.filegenerator import FileBase +from awscli.customizations.s3.filegenerator import FileStat class ComparatorTest(unittest.TestCase): @@ -30,11 +30,11 @@ def test_compare_key_equal(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -54,11 +54,11 @@ def test_compare_size(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=11, last_update=time, src_type='local', dest_type='s3', operation_name='upload') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -80,11 +80,11 @@ def test_compare_lastmod_upload(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', dest_type='s3', operation_name='upload') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -106,11 +106,11 @@ def test_compare_lastmod_copy(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', dest_type='s3', operation_name='copy') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='s3', operation_name='') @@ -132,11 +132,11 @@ def test_compare_lastmod_download(self): result_list = [] time = datetime.datetime.now() future_time = time + datetime.timedelta(0, 3) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='download') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='local', dest_type='s3', operation_name='') @@ -149,11 +149,11 @@ def test_compare_lastmod_download(self): self.assertEqual(result_list, ref_list) # If the source is newer than the destination do not download. - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=future_time, src_type='s3', dest_type='local', operation_name='download') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='') @@ -177,11 +177,11 @@ def test_compare_key_less(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='bomparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -205,11 +205,11 @@ def test_compare_key_greater(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -234,7 +234,7 @@ def test_empty_src(self): ref_list = [] result_list = [] time = datetime.datetime.now() - dest_file = FileBase(src='', dest='', + dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, last_update=time, src_type='s3', dest_type='local', operation_name='') @@ -256,7 +256,7 @@ def test_empty_dest(self): ref_list = [] result_list = [] time = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='domparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') @@ -294,12 +294,12 @@ def test_compare_size_only_dest_older_than_src(self): time_src = datetime.datetime.now() time_dst = time_src + datetime.timedelta(days=1) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', dest_type='s3', operation_name='upload') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', dest_type='local', operation_name='') @@ -315,12 +315,12 @@ def test_compare_size_only_src_older_than_dest(self): time_dst = datetime.datetime.now() time_src = time_dst + datetime.timedelta(days=1) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='local', dest_type='s3', operation_name='upload') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='s3', dest_type='local', operation_name='') @@ -342,12 +342,12 @@ def test_compare_exact_timestamps_dest_older(self): time_src = datetime.datetime.now() time_dst = time_src - datetime.timedelta(days=1) - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='download') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', dest_type='s3', operation_name='') @@ -364,12 +364,12 @@ def test_compare_exact_timestamps_src_older(self): time_src = datetime.datetime.now() - datetime.timedelta(days=1) time_dst = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_src, src_type='s3', dest_type='local', operation_name='download') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_dst, src_type='local', dest_type='s3', operation_name='') @@ -385,12 +385,12 @@ def test_compare_exact_timestamps_same_age_same_size(self): """ time_both = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='s3', dest_type='local', operation_name='download') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', dest_type='s3', operation_name='') @@ -406,12 +406,12 @@ def test_compare_exact_timestamps_same_age_diff_size(self): """ time_both = datetime.datetime.now() - src_file = FileBase(src='', dest='', + src_file = FileStat(src='', dest='', compare_key='test.py', size=20, last_update=time_both, src_type='s3', dest_type='local', operation_name='download') - dst_file = FileBase(src='', dest='', + dst_file = FileStat(src='', dest='', compare_key='test.py', size=10, last_update=time_both, src_type='local', dest_type='s3', operation_name='') diff --git a/tests/unit/customizations/s3/test_filegenerator.py b/tests/unit/customizations/s3/test_filegenerator.py index 8ede65de6536..96b2a2739c2b 100644 --- a/tests/unit/customizations/s3/test_filegenerator.py +++ b/tests/unit/customizations/s3/test_filegenerator.py @@ -20,7 +20,7 @@ import mock from awscli.customizations.s3.filegenerator import FileGenerator, \ - FileDecodingError, FileBase + FileDecodingError, FileStat from awscli.customizations.s3.utils import get_file_stat import botocore.session from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ @@ -60,11 +60,11 @@ def test_local_file(self): for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) - file_base = FileBase(src=self.local_file, dest='bucket/text1.txt', + file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') - ref_list = [file_base] + ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -85,20 +85,20 @@ def test_local_directory(self): for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) - file_base = FileBase(src=self.local_file, dest='bucket/text1.txt', + file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') path = self.local_dir + 'another_directory' + os.sep \ + 'text2.txt' size, last_update = get_file_stat(path) - file_base2 = FileBase(src=path, + file_stat2 = FileStat(src=path, dest='bucket/another_directory/text2.txt', compare_key='another_directory/text2.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') - ref_list = [file_base2, file_base] + ref_list = [file_stat2, file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -217,12 +217,12 @@ def test_no_follow_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_bases = FileGenerator(self.service, self.endpoint, + file_stats = FileGenerator(self.service, self.endpoint, '', False).call(input_local_dir) self.filenames.sort() result_list = [] - for file_base in file_bases: - result_list.append(getattr(file_base, 'src')) + for file_stat in file_stats: + result_list.append(getattr(file_stat, 'src')) self.assertEqual(len(result_list), len(self.filenames)) # Just check to make sure the right local files are generated. for i in range(len(result_list)): @@ -239,13 +239,13 @@ def test_follow_bad_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_bases = FileGenerator(self.service, self.endpoint, + file_stats = FileGenerator(self.service, self.endpoint, '', True).call(input_local_dir) result_list = [] rc = 0 try: - for file_base in file_bases: - result_list.append(getattr(file_base, 'src')) + for file_stat in file_stats: + result_list.append(getattr(file_stat, 'src')) rc = 1 except OSError as e: pass @@ -264,13 +264,13 @@ def test_follow_symlink(self): 'dest': {'path': self.bucket, 'type': 's3'}, 'dir_op': True, 'use_src_name': True} - file_bases = FileGenerator(self.service, self.endpoint, + file_stats = FileGenerator(self.service, self.endpoint, '', True).call(input_local_dir) all_filenames = self.filenames + self.symlink_files all_filenames.sort() result_list = [] - for file_base in file_bases: - result_list.append(getattr(file_base, 'src')) + for file_stat in file_stats: + result_list.append(getattr(file_stat, 'src')) self.assertEqual(len(result_list), len(all_filenames)) # Just check to make sure the right local files are generated. for i in range(len(result_list)): @@ -390,14 +390,14 @@ def test_s3_file(self): result_list = [] for filename in files: result_list.append(filename) - file_base = FileBase(src=self.file1, dest='text1.txt', + file_stat = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') - ref_list = [file_base] + ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -417,7 +417,7 @@ def test_s3_directory(self): result_list = [] for filename in files: result_list.append(filename) - file_base = FileBase(src=self.file2, + file_stat = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', @@ -425,7 +425,7 @@ def test_s3_directory(self): last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='') - file_base2 = FileBase(src=self.file1, + file_stat2 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[1].size, @@ -433,7 +433,7 @@ def test_s3_directory(self): src_type='s3', dest_type='local', operation_name='') - ref_list = [file_base, file_base2] + ref_list = [file_stat, file_stat2] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) @@ -453,21 +453,21 @@ def test_s3_delete_directory(self): for filename in files: result_list.append(filename) - file_base1 = FileBase(src=self.bucket + '/another_directory/', + file_stat1 = FileStat(src=self.bucket + '/another_directory/', dest='another_directory' + os.sep, compare_key='another_directory/', size=result_list[0].size, last_update=result_list[0].last_update, src_type='s3', dest_type='local', operation_name='delete') - file_base2 = FileBase(src=self.file2, + file_stat2 = FileStat(src=self.file2, dest='another_directory' + os.sep + 'text2.txt', compare_key='another_directory/text2.txt', size=result_list[1].size, last_update=result_list[1].last_update, src_type='s3', dest_type='local', operation_name='delete') - file_base3 = FileBase(src=self.file1, + file_stat3 = FileStat(src=self.file1, dest='text1.txt', compare_key='text1.txt', size=result_list[2].size, @@ -475,7 +475,7 @@ def test_s3_delete_directory(self): src_type='s3', dest_type='local', operation_name='delete') - ref_list = [file_base1, file_base2, file_base3] + ref_list = [file_stat1, file_stat2, file_stat3] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i]) diff --git a/tests/unit/customizations/s3/test_infosetter.py b/tests/unit/customizations/s3/test_fileinfobuilder.py similarity index 73% rename from tests/unit/customizations/s3/test_infosetter.py rename to tests/unit/customizations/s3/test_fileinfobuilder.py index 2cf197a07fe2..439c006ad136 100644 --- a/tests/unit/customizations/s3/test_infosetter.py +++ b/tests/unit/customizations/s3/test_fileinfobuilder.py @@ -13,17 +13,17 @@ import mock from awscli.testutils import unittest -from awscli.customizations.s3.filegenerator import FileBase +from awscli.customizations.s3.filegenerator import FileStat from awscli.customizations.s3.fileinfo import FileInfo -from awscli.customizations.s3.infosetter import InfoSetter +from awscli.customizations.s3.fileinfobuilder import FileInfoBuilder -class TestInfoSetter(unittest.TestCase): +class TestFileInfoBuilder(unittest.TestCase): def test_info_setter(self): - info_setter = InfoSetter(service='service', endpoint='endpoint', - source_endpoint='source_endpoint', - parameters='parameters') - files = [FileBase(src='src', dest='dest', compare_key='compare_key', + info_setter = FileInfoBuilder(service='service', endpoint='endpoint', + source_endpoint='source_endpoint', + parameters='parameters') + files = [FileStat(src='src', dest='dest', compare_key='compare_key', size='size', last_update='last_update', src_type='src_type', dest_type='dest_type', operation_name='operation_name')] diff --git a/tests/unit/customizations/s3/test_filters.py b/tests/unit/customizations/s3/test_filters.py index b3f49e5029b5..eaa5f89e485b 100644 --- a/tests/unit/customizations/s3/test_filters.py +++ b/tests/unit/customizations/s3/test_filters.py @@ -14,7 +14,7 @@ from awscli.testutils import unittest import platform -from awscli.customizations.s3.filegenerator import FileBase +from awscli.customizations.s3.filegenerator import FileStat from awscli.customizations.s3.filters import Filter @@ -29,23 +29,23 @@ def platform_path(filepath): class FiltersTest(unittest.TestCase): def setUp(self): self.local_files = [ - self.file_base('test.txt'), - self.file_base('test.jpg'), - self.file_base(os.path.join('directory', 'test.jpg')), + self.file_stat('test.txt'), + self.file_stat('test.jpg'), + self.file_stat(os.path.join('directory', 'test.jpg')), ] self.s3_files = [ - self.file_base('bucket/test.txt', src_type='s3'), - self.file_base('bucket/test.jpg', src_type='s3'), - self.file_base('bucket/key/test.jpg', src_type='s3'), + self.file_stat('bucket/test.txt', src_type='s3'), + self.file_stat('bucket/test.jpg', src_type='s3'), + self.file_stat('bucket/key/test.jpg', src_type='s3'), ] - def file_base(self, filename, src_type='local'): + def file_stat(self, filename, src_type='local'): if src_type == 'local': filename = os.path.abspath(filename) dest_type = 's3' else: dest_type = 'local' - return FileBase(src=filename, dest='', + return FileStat(src=filename, dest='', compare_key='', size=10, last_update=0, src_type=src_type, dest_type=dest_type, operation_name='') @@ -116,15 +116,15 @@ def test_prefix_filtering_consistent(self): # The same filter should work for both local and remote files. # So if I have a directory with 2 files: local_files = [ - self.file_base('test1.txt'), - self.file_base('nottest1.txt'), + self.file_stat('test1.txt'), + self.file_stat('nottest1.txt'), ] - # And the same 2 files remote (note that the way FileBase objects + # And the same 2 files remote (note that the way FileStat objects # are constructed, we'll have the bucket name but no leading '/' # character): remote_files = [ - self.file_base('bucket/test1.txt', src_type='s3'), - self.file_base('bucket/nottest1.txt', src_type='s3'), + self.file_stat('bucket/test1.txt', src_type='s3'), + self.file_stat('bucket/nottest1.txt', src_type='s3'), ] # If I apply the filter to the local to the local files. exclude_filter = self.create_filter([['exclude', 't*']]) @@ -143,9 +143,9 @@ def test_prefix_filtering_consistent(self): def test_bucket_exclude_with_prefix(self): s3_files = [ - self.file_base('bucket/dir1/key1.txt', src_type='s3'), - self.file_base('bucket/dir1/key2.txt', src_type='s3'), - self.file_base('bucket/dir1/notkey3.txt', src_type='s3'), + self.file_stat('bucket/dir1/key1.txt', src_type='s3'), + self.file_stat('bucket/dir1/key2.txt', src_type='s3'), + self.file_stat('bucket/dir1/notkey3.txt', src_type='s3'), ] filtered_files = list( self.create_filter([['exclude', 'dir1/*']], @@ -160,7 +160,7 @@ def test_bucket_exclude_with_prefix(self): def test_root_dir(self): p = platform_path - local_files = [self.file_base(p('/foo/bar/baz.txt'), src_type='local')] + local_files = [self.file_stat(p('/foo/bar/baz.txt'), src_type='local')] local_filter = self.create_filter([['exclude', 'baz.txt']], root=p('/foo/bar/')) filtered = list(local_filter.call(local_files)) diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index 9ce310586daf..5a6c2c978397 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -154,11 +154,14 @@ def test_create_instructions(self): """ cmds = ['cp', 'mv', 'rm', 'sync', 'mb', 'rb'] - instructions = {'cp': ['file_generator', 'info_setter', 's3_handler'], - 'mv': ['file_generator', 'info_setter', 's3_handler'], - 'rm': ['file_generator', 'info_setter', 's3_handler'], + instructions = {'cp': ['file_generator', 'file_info_builder', + 's3_handler'], + 'mv': ['file_generator', 'file_info_builder', + 's3_handler'], + 'rm': ['file_generator', 'file_info_builder', + 's3_handler'], 'sync': ['file_generator', 'comparator', - 'info_setter', 's3_handler'], + 'file_info_builder', 's3_handler'], 'mb': ['s3_handler'], 'rb': ['s3_handler']} @@ -176,7 +179,8 @@ def test_create_instructions(self): cmd_arc = CommandArchitecture(self.session, 'cp', params) cmd_arc.create_instructions() self.assertEqual(cmd_arc.instructions, ['file_generator', 'filters', - 'info_setter', 's3_handler']) + 'file_info_builder', + 's3_handler']) def test_run_cp_put(self): # This ensures that the architecture sets up correctly for a ``cp`` put