Optional s3 sync flag to address timestamp issues described in aws#599

awscli/customizations/s3/comparator.py

@@ -38,6 +38,10 @@ def __init__(self, params=None):
         if 'size_only' in params:
             self.compare_on_size_only = params['size_only']
 
+        self.match_exact_timestamps = False
+        if 'exact_timestamps' in params:
+            self.match_exact_timestamps = params['exact_timestamps']
+
     def call(self, src_files, dest_files):
         """
         This function preforms the actual comparisons.  The parameters it takes
@@ -198,6 +202,11 @@ def compare_time(self, src_file, dest_file):
                 # at the source location.
                 return False
         elif cmd == "download":
+            if self.match_exact_timestamps:
+                # An update is needed unless the
+                # timestamps match exactly.
+                return total_seconds(delta) == 0
+
             if total_seconds(delta) <= 0:
                 return True
             else:

awscli/customizations/s3/s3.py

@@ -796,6 +796,7 @@ def add_verify_ssl(self, parsed_globals):
                                 'sse', 'storage-class', 'content-type',
                                 'cache-control', 'content-disposition',
                                 'content-encoding', 'content-language',
+                                'exact-timestamps',
                                 'expires', 'size-only']},
             'ls': {'options': {'nargs': '?', 'default': 's3://'},
                    'params': ['recursive'], 'default': 's3://',
@@ -850,6 +851,11 @@ def add_verify_ssl(self, parsed_globals):
                'size-only': {'options': {'action': 'store_true'}, 'documents':
                    ('Makes the size of each key the only criteria used to '
                     'decide whether to sync from source to destination.')},
+               'exact-timestamps': {'options': {'action': 'store_true'}, 'documents':
+                   ('When syncing from S3 to local, same-sized items will be '
+                    'ignored only when the timestamps match exactly. The '
+                    'default behavior is to ignore same-sized items unless '
+                    'the local version is newer than the S3 version.')},
                'index-document': {'options': {}, 'documents':
                    ('A suffix that is appended to a request that is for a '
                     'directory on the website endpoint (e.g. if the suffix '

tests/unit/customizations/s3/test_comparator.py

@@ -351,5 +351,104 @@ def test_compare_size_only_src_older_than_dest(self):
         self.assertEqual(sum(1 for _ in files), 0)
 
 
+class ComparatorExactTimestampsTest(unittest.TestCase):
+    def setUp(self):
+        self.comparator = Comparator({'exact_timestamps': True})
+
+    def test_compare_exact_timestamps_dest_older(self):
+        """
+        Confirm that same-sized files are synced when
+        the destination is older than the source and
+        `exact_timestamps` is set.
+        """
+        time_src = datetime.datetime.now()
+        time_dst = time_src - datetime.timedelta(days=1)
+
+        src_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_src, src_type='s3',
+                            dest_type='local', operation_name='download',
+                            service=None, endpoint=None)
+
+        dst_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_dst, src_type='local',
+                            dest_type='s3', operation_name='',
+                            service=None, endpoint=None)
+
+        files = self.comparator.call(iter([src_file]), iter([dst_file]))
+        self.assertEqual(sum(1 for _ in files), 1)
+
+    def test_compare_exact_timestamps_src_older(self):
+        """
+        Confirm that same-sized files are synced when
+        the source is older than the destination and
+        `exact_timestamps` is set.
+        """
+        time_src = datetime.datetime.now() - datetime.timedelta(days=1)
+        time_dst = datetime.datetime.now()
+
+        src_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_src, src_type='s3',
+                            dest_type='local', operation_name='download',
+                            service=None, endpoint=None)
+
+        dst_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_dst, src_type='local',
+                            dest_type='s3', operation_name='',
+                            service=None, endpoint=None)
+
+        files = self.comparator.call(iter([src_file]), iter([dst_file]))
+        self.assertEqual(sum(1 for _ in files), 1)
+
+    def test_compare_exact_timestamps_same_age_same_size(self):
+        """
+        Confirm that same-sized files are not synced when
+        the source and destination are the same age and
+        `exact_timestamps` is set.
+        """
+        time_both = datetime.datetime.now()
+
+        src_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_both, src_type='s3',
+                            dest_type='local', operation_name='download',
+                            service=None, endpoint=None)
+
+        dst_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_both, src_type='local',
+                            dest_type='s3', operation_name='',
+                            service=None, endpoint=None)
+
+        files = self.comparator.call(iter([src_file]), iter([dst_file]))
+        self.assertEqual(sum(1 for _ in files), 0)
+
+    def test_compare_exact_timestamps_same_age_diff_size(self):
+        """
+        Confirm that files of differing sizes are synced when
+        the source and destination are the same age and
+        `exact_timestamps` is set.
+        """
+        time_both = datetime.datetime.now()
+
+        src_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=20,
+                            last_update=time_both, src_type='s3',
+                            dest_type='local', operation_name='download',
+                            service=None, endpoint=None)
+
+        dst_file = FileInfo(src='', dest='',
+                            compare_key='test.py', size=10,
+                            last_update=time_both, src_type='local',
+                            dest_type='s3', operation_name='',
+                            service=None, endpoint=None)
+
+        files = self.comparator.call(iter([src_file]), iter([dst_file]))
+        self.assertEqual(sum(1 for _ in files), 1)
+
+
 if __name__ == "__main__":
     unittest.main()