From 4000d5824d9bdcd9cf07843c6b42d7a41a0abaa9 Mon Sep 17 00:00:00 2001 From: na4da <34153527+na4da@users.noreply.github.com> Date: Thu, 13 Jan 2022 08:49:20 +0900 Subject: [PATCH] aws_s3: Add latest choice on overwrite parameter (#595) aws_s3: Add latest choice on overwrite parameter SUMMARY This PR adds latest choice on overwrite parameter to get remote object only when it's the latest. If a local object is the latest, or local and remote objects have identical last-modified timestamps, overwriting is ignored. ISSUE TYPE Feature Pull Request COMPONENT NAME aws_s3 ADDITIONAL INFORMATION Motivation: when using the different choice, getting a large sized object takes few minutes whether local and remote objects are identical or not because of MD5 calculation. Reviewed-by: Alina Buzachis Reviewed-by: na4da Reviewed-by: Jill R Reviewed-by: None --- ...d-latest-choice-on-overwrite-parameter.yml | 2 ++ plugins/modules/aws_s3.py | 29 ++++++++++++++-- .../integration/targets/aws_s3/tasks/main.yml | 33 +++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/595-aws_s3-add-latest-choice-on-overwrite-parameter.yml diff --git a/changelogs/fragments/595-aws_s3-add-latest-choice-on-overwrite-parameter.yml b/changelogs/fragments/595-aws_s3-add-latest-choice-on-overwrite-parameter.yml new file mode 100644 index 00000000000..74b96e93e24 --- /dev/null +++ b/changelogs/fragments/595-aws_s3-add-latest-choice-on-overwrite-parameter.yml @@ -0,0 +1,2 @@ +minor_changes: +- aws_s3 - add latest choice on ``overwrite`` parameter to get latest object on S3 (https://github.com/ansible-collections/amazon.aws/pull/595). diff --git a/plugins/modules/aws_s3.py b/plugins/modules/aws_s3.py index a4582a41984..614ee741a32 100644 --- a/plugins/modules/aws_s3.py +++ b/plugins/modules/aws_s3.py @@ -102,12 +102,13 @@ overwrite: description: - Force overwrite either locally on the filesystem or remotely with the object/key. Used with C(PUT) and C(GET) operations. - - Must be a Boolean, C(always), C(never) or C(different). + - Must be a Boolean, C(always), C(never), C(different) or C(latest). - C(true) is the same as C(always). - C(false) is equal to C(never). - When this is set to C(different) the MD5 sum of the local file is compared with the 'ETag' of the object/key in S3. The ETag may or may not be an MD5 digest of the object data. See the ETag response header here U(https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html). + - (C(GET) mode only) When this is set to C(latest) the last modified timestamp of local file is compared with the 'LastModified' of the object/key in S3. default: 'always' aliases: ['force'] type: str @@ -424,6 +425,26 @@ def get_etag(s3, bucket, obj, version=None): return None +def get_s3_last_modified_timestamp(s3, bucket, obj, version=None): + if version: + key_check = s3.head_object(Bucket=bucket, Key=obj, VersionId=version) + else: + key_check = s3.head_object(Bucket=bucket, Key=obj) + if not key_check: + return None + return key_check['LastModified'].timestamp() + + +def is_local_object_latest(module, s3, bucket, obj, version=None, local_file=None): + s3_last_modified = get_s3_last_modified_timestamp(s3, bucket, obj, version) + if os.path.exists(local_file) is False: + return False + else: + local_last_modified = os.path.getmtime(local_file) + + return s3_last_modified <= local_last_modified + + def bucket_check(module, s3, bucket, validate=True): exists = True try: @@ -946,7 +967,7 @@ def main(): validate_bucket_name(module, bucket) - if overwrite not in ['always', 'never', 'different']: + if overwrite not in ['always', 'never', 'different', 'latest']: if module.boolean(overwrite): overwrite = 'always' else: @@ -1020,8 +1041,10 @@ def main(): if dest and path_check(dest) and overwrite != 'always': if overwrite == 'never': module.exit_json(msg="Local object already exists and overwrite is disabled.", changed=False) - if etag_compare(module, s3, bucket, obj, version=version, local_file=dest): + if overwrite == 'different' and etag_compare(module, s3, bucket, obj, version=version, local_file=dest): module.exit_json(msg="Local and remote object are identical, ignoring. Use overwrite=always parameter to force.", changed=False) + if overwrite == 'latest' and is_local_object_latest(module, s3, bucket, obj, version=version, local_file=dest): + module.exit_json(msg="Local object is latest, ignoreing. Use overwrite=always parameter to force.", changed=False) try: download_s3file(module, s3, bucket, obj, dest, retries, version=version) diff --git a/tests/integration/targets/aws_s3/tasks/main.yml b/tests/integration/targets/aws_s3/tasks/main.yml index f7b7a3f7345..a36c659d4ef 100644 --- a/tests/integration/targets/aws_s3/tasks/main.yml +++ b/tests/integration/targets/aws_s3/tasks/main.yml @@ -274,6 +274,39 @@ that: - result is changed + - name: test get with overwrite=latest and identical files + aws_s3: + bucket: "{{ bucket_name }}" + mode: get + dest: "{{ tmpdir.path }}/download.txt" + object: delete.txt + overwrite: latest + retries: 3 + delay: 3 + register: result + + - assert: + that: + - result is not changed + + - name: modify mtime for local file to past + shell: touch -mt 197001010900.00 "{{ tmpdir.path }}/download.txt" + + - name: test get with overwrite=latest and files that mtimes are different + aws_s3: + bucket: "{{ bucket_name }}" + mode: get + dest: "{{ tmpdir.path }}/download.txt" + object: delete.txt + overwrite: latest + retries: 3 + delay: 3 + register: result + + - assert: + that: + - result is changed + - name: test geturl of the object aws_s3: bucket: "{{ bucket_name }}"