From f9eef86b7da36d3f09e147dfcefca95506ab5db9 Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Mon, 10 Sep 2018 16:03:12 -0700 Subject: [PATCH 1/6] Adding s3_list_objects_encoding_type_url handler to ListObjectsV2 --- botocore/handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/botocore/handlers.py b/botocore/handlers.py index 253d5368f8..f09030ec46 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -880,6 +880,8 @@ def remove_subscribe_to_shard(class_attributes, **kwargs): ('before-parameter-build.s3.ListObjects', set_list_objects_encoding_type_url), + ('before-parameter-build.s3.ListObjectsV2', + set_list_objects_encoding_type_url), ('before-call.s3.PutBucketTagging', calculate_md5), ('before-call.s3.PutBucketLifecycle', calculate_md5), ('before-call.s3.PutBucketLifecycleConfiguration', calculate_md5), From 3231b9a2c768d50f6477bfbc0b7279612366c620 Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Mon, 10 Sep 2018 19:52:49 -0700 Subject: [PATCH 2/6] registering response decode for ListObjectsV2 --- botocore/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/botocore/handlers.py b/botocore/handlers.py index f09030ec46..9139b83060 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -945,6 +945,7 @@ def remove_subscribe_to_shard(class_attributes, **kwargs): ('before-parameter-build.route53', fix_route53_ids), ('before-parameter-build.glacier', inject_account_id), ('after-call.s3.ListObjects', decode_list_object), + ('after-call.s3.ListObjectsV2', decode_list_object), # Cloudsearchdomain search operation will be sent by HTTP POST ('request-created.cloudsearchdomain.Search', From 8ef58cfad1836441516792b0280862b8847867ff Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Tue, 11 Sep 2018 16:04:05 -0700 Subject: [PATCH 3/6] added decode_list_object_v2 handler and tests --- botocore/handlers.py | 23 +++++++++++++- tests/unit/test_handlers.py | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/botocore/handlers.py b/botocore/handlers.py index 9139b83060..abb1f7e6b2 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -745,6 +745,27 @@ def decode_list_object(parsed, context, **kwargs): member[child_key] = unquote_str(member[child_key]) +def decode_list_object_v2(parsed, context, **kwargs): + # From the documentation: If you specify encoding-type request parameter, + # Amazon S3 includes this element in the response, and returns encoded key + # name values in the following response elements: + # Delimiter, Prefix, ContinuationToken, Key, and StartAfter. + + if parsed.get('EncodingType') == 'url' and \ + context.get('encoding_type_auto_set'): + # URL decode top-level keys in the response if present. + top_level_keys = ['Delimiter', 'Prefix', 'ContinuationToken', 'StartAfter'] + for key in top_level_keys: + if key in parsed: + parsed[key] = unquote_str(parsed[key]) + # URL decode nested keys from the response if present. + nested_keys = [('Contents', 'Key'), ('CommonPrefixes', 'Prefix')] + for (top_key, child_key) in nested_keys: + if top_key in parsed: + for member in parsed[top_key]: + member[child_key] = unquote_str(member[child_key]) + + def convert_body_to_file_like_object(params, **kwargs): if 'Body' in params: if isinstance(params['Body'], six.string_types): @@ -945,7 +966,7 @@ def remove_subscribe_to_shard(class_attributes, **kwargs): ('before-parameter-build.route53', fix_route53_ids), ('before-parameter-build.glacier', inject_account_id), ('after-call.s3.ListObjects', decode_list_object), - ('after-call.s3.ListObjectsV2', decode_list_object), + ('after-call.s3.ListObjectsV2', decode_list_object_v2), # Cloudsearchdomain search operation will be sent by HTTP POST ('request-created.cloudsearchdomain.Search', diff --git a/tests/unit/test_handlers.py b/tests/unit/test_handlers.py index c427ca3ac3..99ca9e5396 100644 --- a/tests/unit/test_handlers.py +++ b/tests/unit/test_handlers.py @@ -873,6 +873,69 @@ def test_decode_list_objects_with_delimiter(self): handlers.decode_list_object(parsed, context=context) self.assertEqual(parsed['Delimiter'], u'\xe7\xf6s% asd\x08 c') + def test_decode_list_objects_v2(self): + parsed = { + 'Contents': [{'Key': "%C3%A7%C3%B6s%25asd%08"}], + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['Contents'][0]['Key'], u'\xe7\xf6s%asd\x08') + + def test_decode_list_objects_v2_does_not_decode_without_context(self): + parsed = { + 'Contents': [{'Key': "%C3%A7%C3%B6s%25asd"}], + 'EncodingType': 'url', + } + handlers.decode_list_object_v2(parsed, context={}) + self.assertEqual(parsed['Contents'][0]['Key'], u'%C3%A7%C3%B6s%25asd') + + def test_decode_list_objects_v2_with_delimiter(self): + parsed = { + 'Delimiter': "%C3%A7%C3%B6s%25%20asd%08+c", + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['Delimiter'], u'\xe7\xf6s% asd\x08 c') + + def test_decode_list_objects_v2_with_prefix(self): + parsed = { + 'Prefix': "%C3%A7%C3%B6s%25%20asd%08+c", + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['Prefix'], u'\xe7\xf6s% asd\x08 c') + + def test_decode_list_objects_v2_with_continuationtoken(self): + parsed = { + 'ContinuationToken': "%C3%A7%C3%B6s%25%20asd%08+c", + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['ContinuationToken'], u'\xe7\xf6s% asd\x08 c') + + def test_decode_list_objects_v2_with_startafter(self): + parsed = { + 'StartAfter': "%C3%A7%C3%B6s%25%20asd%08+c", + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['StartAfter'], u'\xe7\xf6s% asd\x08 c') + + def test_decode_list_objects_v2_with_common_prefixes(self): + parsed = { + 'CommonPrefixes': [{'Prefix': "%C3%A7%C3%B6s%25%20asd%08+c"}], + 'EncodingType': 'url', + } + context = {'encoding_type_auto_set': True} + handlers.decode_list_object_v2(parsed, context=context) + self.assertEqual(parsed['CommonPrefixes'][0]['Prefix'], + u'\xe7\xf6s% asd\x08 c') + def test_get_bucket_location_optional(self): # This handler should no-op if another hook (i.e. stubber) has already # filled in response From a0d2f3f7b85d0d9c3be3388111306538b1aede20 Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Wed, 12 Sep 2018 11:34:22 -0700 Subject: [PATCH 4/6] generalized decode_list_object --- botocore/handlers.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/botocore/handlers.py b/botocore/handlers.py index abb1f7e6b2..7a81fed8ec 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -730,42 +730,38 @@ def decode_list_object(parsed, context, **kwargs): # Amazon S3 includes this element in the response, and returns encoded key # name values in the following response elements: # Delimiter, Marker, Prefix, NextMarker, Key. - if parsed.get('EncodingType') == 'url' and \ - context.get('encoding_type_auto_set'): - # URL decode top-level keys in the response if present. - top_level_keys = ['Delimiter', 'Marker', 'NextMarker'] - for key in top_level_keys: - if key in parsed: - parsed[key] = unquote_str(parsed[key]) - # URL decode nested keys from the response if present. - nested_keys = [('Contents', 'Key'), ('CommonPrefixes', 'Prefix')] - for (top_key, child_key) in nested_keys: - if top_key in parsed: - for member in parsed[top_key]: - member[child_key] = unquote_str(member[child_key]) - + _decode_list_object( + top_level_keys=['Delimiter', 'Marker', 'NextMarker'], + nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], + parsed=parsed, + context=context + ) def decode_list_object_v2(parsed, context, **kwargs): # From the documentation: If you specify encoding-type request parameter, # Amazon S3 includes this element in the response, and returns encoded key # name values in the following response elements: # Delimiter, Prefix, ContinuationToken, Key, and StartAfter. - + _decode_list_object( + top_level_keys=['Delimiter', 'Prefix', 'ContinuationToken', 'StartAfter'], + nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')], + parsed=parsed, + context=context + ) + +def _decode_list_object(top_level_keys, nested_keys, parsed, context): if parsed.get('EncodingType') == 'url' and \ context.get('encoding_type_auto_set'): # URL decode top-level keys in the response if present. - top_level_keys = ['Delimiter', 'Prefix', 'ContinuationToken', 'StartAfter'] for key in top_level_keys: if key in parsed: parsed[key] = unquote_str(parsed[key]) # URL decode nested keys from the response if present. - nested_keys = [('Contents', 'Key'), ('CommonPrefixes', 'Prefix')] for (top_key, child_key) in nested_keys: if top_key in parsed: for member in parsed[top_key]: member[child_key] = unquote_str(member[child_key]) - def convert_body_to_file_like_object(params, **kwargs): if 'Body' in params: if isinstance(params['Body'], six.string_types): From 321aece184894f07ed7534d7ddc966e3e661d6bd Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Wed, 12 Sep 2018 11:51:31 -0700 Subject: [PATCH 5/6] list_objects_v2 decode integration test --- tests/integration/test_s3.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py index 5c25ede927..b6be23d66a 100644 --- a/tests/integration/test_s3.py +++ b/tests/integration/test_s3.py @@ -416,6 +416,21 @@ def test_unicode_system_character(self): self.assertEqual(len(parsed['Contents']), 1) self.assertEqual(parsed['Contents'][0]['Key'], 'foo%08') + def test_unicode_system_character_with_list_v2(self): + # Verify we can use a unicode system character which would normally + # break the xml parser + key_name = 'foo\x08' + self.create_object(key_name) + self.addCleanup(self.delete_object, key_name, self.bucket_name) + parsed = self.client.list_objects_v2(Bucket=self.bucket_name) + self.assertEqual(len(parsed['Contents']), 1) + self.assertEqual(parsed['Contents'][0]['Key'], key_name) + + parsed = self.client.list_objects_v2(Bucket=self.bucket_name, + EncodingType='url') + self.assertEqual(len(parsed['Contents']), 1) + self.assertEqual(parsed['Contents'][0]['Key'], 'foo%08') + def test_thread_safe_auth(self): self.auth_paths = [] self.session.register('before-sign', self.increment_auth) From 9d31414b4ac68324b6c469e49346f7e23984556f Mon Sep 17 00:00:00 2001 From: Dave Rocamora Date: Wed, 12 Sep 2018 12:25:08 -0700 Subject: [PATCH 6/6] updating changelog --- .changes/next-release/enhancement-s3-12341.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changes/next-release/enhancement-s3-12341.json diff --git a/.changes/next-release/enhancement-s3-12341.json b/.changes/next-release/enhancement-s3-12341.json new file mode 100644 index 0000000000..f232fa603f --- /dev/null +++ b/.changes/next-release/enhancement-s3-12341.json @@ -0,0 +1,5 @@ +{ + "category": "s3", + "type": "enhancement", + "description": "Adds encoding and decoding handlers for ListObjectsV2 `#1552 `__" +}