aws_s3: Add latest choice on overwrite parameter (ansible-collections…

…#595)

aws_s3: Add latest choice on overwrite parameter

SUMMARY

This PR adds latest choice on overwrite parameter to get remote object only when it's the latest. If a local object is the latest, or local and remote objects have identical last-modified timestamps, overwriting is ignored.

ISSUE TYPE


Feature Pull Request

COMPONENT NAME

aws_s3
ADDITIONAL INFORMATION


Motivation: when using the different choice, getting a large sized object takes few minutes whether local and remote objects are identical or not because of MD5 calculation.

Reviewed-by: Alina Buzachis <None>
Reviewed-by: na4da <None>
Reviewed-by: Jill R <None>
Reviewed-by: None <None>

changelogs/fragments/595-aws_s3-add-latest-choice-on-overwrite-parameter.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- aws_s3 - add latest choice on ``overwrite`` parameter to get latest object on S3 (https://github.com/ansible-collections/amazon.aws/pull/595).

plugins/modules/aws_s3.py

@@ -102,12 +102,13 @@
   overwrite:
     description:
       - Force overwrite either locally on the filesystem or remotely with the object/key. Used with C(PUT) and C(GET) operations.
-      - Must be a Boolean, C(always), C(never) or C(different).
+      - Must be a Boolean, C(always), C(never), C(different) or C(latest).
       - C(true) is the same as C(always).
       - C(false) is equal to C(never).
       - When this is set to C(different) the MD5 sum of the local file is compared with the 'ETag' of the object/key in S3.
         The ETag may or may not be an MD5 digest of the object data. See the ETag response header here
         U(https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html).
+      - (C(GET) mode only) When this is set to C(latest) the last modified timestamp of local file is compared with the 'LastModified' of the object/key in S3.
     default: 'always'
     aliases: ['force']
     type: str
@@ -424,6 +425,26 @@ def get_etag(s3, bucket, obj, version=None):
         return None
 
 
+def get_s3_last_modified_timestamp(s3, bucket, obj, version=None):
+    if version:
+        key_check = s3.head_object(Bucket=bucket, Key=obj, VersionId=version)
+    else:
+        key_check = s3.head_object(Bucket=bucket, Key=obj)
+    if not key_check:
+        return None
+    return key_check['LastModified'].timestamp()
+
+
+def is_local_object_latest(module, s3, bucket, obj, version=None, local_file=None):
+    s3_last_modified = get_s3_last_modified_timestamp(s3, bucket, obj, version)
+    if os.path.exists(local_file) is False:
+        return False
+    else:
+        local_last_modified = os.path.getmtime(local_file)
+
+    return s3_last_modified <= local_last_modified
+
+
 def bucket_check(module, s3, bucket, validate=True):
     exists = True
     try:
@@ -946,7 +967,7 @@ def main():
 
     validate_bucket_name(module, bucket)
 
-    if overwrite not in ['always', 'never', 'different']:
+    if overwrite not in ['always', 'never', 'different', 'latest']:
         if module.boolean(overwrite):
             overwrite = 'always'
         else:
@@ -1020,8 +1041,10 @@ def main():
         if dest and path_check(dest) and overwrite != 'always':
             if overwrite == 'never':
                 module.exit_json(msg="Local object already exists and overwrite is disabled.", changed=False)
-            if etag_compare(module, s3, bucket, obj, version=version, local_file=dest):
+            if overwrite == 'different' and etag_compare(module, s3, bucket, obj, version=version, local_file=dest):
                 module.exit_json(msg="Local and remote object are identical, ignoring. Use overwrite=always parameter to force.", changed=False)
+            if overwrite == 'latest' and is_local_object_latest(module, s3, bucket, obj, version=version, local_file=dest):
+                module.exit_json(msg="Local object is latest, ignoreing. Use overwrite=always parameter to force.", changed=False)
 
         try:
             download_s3file(module, s3, bucket, obj, dest, retries, version=version)

tests/integration/targets/aws_s3/tasks/main.yml

@@ -274,6 +274,39 @@
         that:
           - result is changed
 
+    - name: test get with overwrite=latest and identical files
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        mode: get
+        dest: "{{ tmpdir.path }}/download.txt"
+        object: delete.txt
+        overwrite: latest
+      retries: 3
+      delay: 3
+      register: result
+
+    - assert:
+        that:
+          - result is not changed
+
+    - name: modify mtime for local file to past
+      shell: touch -mt 197001010900.00 "{{ tmpdir.path }}/download.txt"
+
+    - name: test get with overwrite=latest and files that mtimes are different
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        mode: get
+        dest: "{{ tmpdir.path }}/download.txt"
+        object: delete.txt
+        overwrite: latest
+      retries: 3
+      delay: 3
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
     - name: test geturl of the object
       aws_s3:
         bucket: "{{ bucket_name }}"