more refactoring

dlm6693 · Jun 26, 2023 · db1beb7 · db1beb7
1 parent 7f2a51f
commit db1beb7
Show file tree

Hide file tree

Showing 11 changed files with 218 additions and 83 deletions.
diff --git a/botocore/args.py b/botocore/args.py
@@ -520,19 +520,20 @@ def _compute_request_compression_config(self, config_kwargs):
         config_kwargs['disable_request_compression'] = disabled
 
     def _validate_min_compression_size(self, min_size):
-        try:
-            min_size = int(min_size)
-        except ValueError:
-            raise botocore.exceptions.InvalidConfigError(
-                f"Invalid value '{min_size}' for "
-                "request_min_compression_size_bytes. Value must be an integer."
-            )
-        if not 0 <= min_size <= 1048576:
-            raise botocore.exceptions.InvalidConfigError(
-                f"Invalid value '{min_size}' for "
-                "request_min_compression_size_bytes. Value must be between 0 "
-                "and 1048576."
-            )
+        if min_size is not None:
+            try:
+                min_size = int(min_size)
+            except ValueError:
+                raise botocore.exceptions.InvalidConfigError(
+                    error_msg=f"Invalid value '{min_size}' for "
+                    "request_min_compression_size_bytes. Value must be an integer."
+                )
+            if not 0 <= min_size <= 1048576:
+                raise botocore.exceptions.InvalidConfigError(
+                    error_msg=f"Invalid value '{min_size}' for "
+                    "request_min_compression_size_bytes. Value must be between 0 "
+                    "and 1048576."
+                )
 
     def _ensure_boolean(self, val):
         if isinstance(val, bool):

diff --git a/botocore/awsrequest.py b/botocore/awsrequest.py
@@ -540,8 +540,7 @@ class RequestCompressor:
     """A class that can compress the body of an ``AWSRequest``."""
 
     def compress(self, config, request_dict, operation_model):
-        """Compresses the request body using the specified encodings if conditions
-        are met.
+        """Compresses the request body using the specified encodings.
 
         Check if the request should be compressed based on the contents of its
         body and config settings. Set or append the `Content-Encoding` header
@@ -554,14 +553,11 @@ def compress(self, config, request_dict, operation_model):
             for encoding in encodings:
                 encoder = getattr(self, f'_{encoding}_compress_body', None)
                 if encoder is not None:
-                    if 'Content-Encoding' not in headers:
+                    ce_header = headers.get('Content-Encoding')
+                    if ce_header is None:
                         headers['Content-Encoding'] = encoding
-                    elif encoding not in headers['Content-Encoding'].split(
-                        ','
-                    ):
-                        headers[
-                            'Content-Encoding'
-                        ] = f'{headers["Content-Encoding"]},{encoding}'
+                    elif encoding not in ce_header.split(','):
+                        headers['Content-Encoding'] = f'{ce_header},{encoding}'
                     request_dict['body'] = encoder(body)
                     if 'headers' not in request_dict:
                         request_dict['headers'] = headers

diff --git a/botocore/endpoint.py b/botocore/endpoint.py
@@ -75,8 +75,14 @@ def convert_to_response_dict(http_response, operation_model):
 
 
 class Endpoint:
-    """Represents an endpoint for a particular service in a specific region.
-    Only an endpoint can make requests.
+    """
+    Represents an endpoint for a particular service in a specific
+    region.  Only an endpoint can make requests.
+
+    :ivar service: The Service object that describes this endpoints
+        service.
+    :ivar host: The fully qualified endpoint hostname.
+    :ivar session: The session object.
     """
 
     def __init__(
@@ -115,12 +121,6 @@ def make_request(self, operation_model, request_dict):
     def create_request(self, params, operation_model=None):
         request = create_request_object(params)
         if operation_model:
-            if operation_model.request_compression:
-                self._request_compressor.compress(
-                    params['context']['client_config'],
-                    request,
-                    operation_model,
-                )
             request.stream_output = any(
                 [
                     operation_model.has_streaming_output,

diff --git a/botocore/handlers.py b/botocore/handlers.py
@@ -1147,10 +1147,10 @@ def remove_content_type_header_for_presigning(request, **kwargs):
 
 
 def urlencode_body(model, params, context, **kwargs):
-    """Urlencode the request body if it is a dictionary.
+    """Url encode the request body if it is a dictionary.
 
-    This is used for services like S3 that require the body to be urlencoded
-    when the body is a dictionary.
+    This is used for services using the query protocol. The body must be serialized
+    as a urlencoded string before it can be compressed.
     """
     body = params.get('body')
     if (
@@ -1222,25 +1222,13 @@ def urlencode_body(model, params, context, **kwargs):
     ('before-call.s3', add_expect_header),
     ('before-call.glacier', add_glacier_version),
     ('before-call.apigateway', add_accept_header),
-    ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2),
-    ('before-call.rds.CopyDBClusterSnapshot', inject_presigned_url_rds),
-    ('before-call.rds.CreateDBCluster', inject_presigned_url_rds),
-    ('before-call.rds.CopyDBSnapshot', inject_presigned_url_rds),
-    ('before-call.rds.CreateDBInstanceReadReplica', inject_presigned_url_rds),
-    (
-        'before-call.rds.StartDBInstanceAutomatedBackupsReplication',
-        inject_presigned_url_rds,
-    ),
-    ('before-call.neptune.CopyDBClusterSnapshot', inject_presigned_url_rds),
-    ('before-call.neptune.CreateDBCluster', inject_presigned_url_rds),
-    ('before-call.docdb.CopyDBClusterSnapshot', inject_presigned_url_rds),
-    ('before-call.docdb.CreateDBCluster', inject_presigned_url_rds),
     ('before-call.s3.PutObject', conditionally_calculate_md5),
     ('before-call.s3.UploadPart', conditionally_calculate_md5),
     ('before-call.s3.DeleteObjects', escape_xml_payload),
     ('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload),
     ('before-call.glacier.UploadArchive', add_glacier_checksums),
     ('before-call.glacier.UploadMultipartPart', add_glacier_checksums),
+    ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2),
     ('request-created', add_retry_headers),
     ('request-created.machinelearning.Predict', switch_host_machinelearning),
     ('needs-retry.s3.UploadPartCopy', check_for_200_error, REGISTER_FIRST),
@@ -1375,6 +1363,14 @@ def urlencode_body(model, params, context, **kwargs):
     # RDS
     #############
     ('creating-client-class.rds', add_generate_db_auth_token),
+    ('before-call.rds.CopyDBClusterSnapshot', inject_presigned_url_rds),
+    ('before-call.rds.CreateDBCluster', inject_presigned_url_rds),
+    ('before-call.rds.CopyDBSnapshot', inject_presigned_url_rds),
+    ('before-call.rds.CreateDBInstanceReadReplica', inject_presigned_url_rds),
+    (
+        'before-call.rds.StartDBInstanceAutomatedBackupsReplication',
+        inject_presigned_url_rds,
+    ),
     # RDS PresignedUrl documentation customizations
     (
         'docs.*.rds.CopyDBClusterSnapshot.complete-section',
@@ -1399,6 +1395,8 @@ def urlencode_body(model, params, context, **kwargs):
     #############
     # Neptune
     #############
+    ('before-call.neptune.CopyDBClusterSnapshot', inject_presigned_url_rds),
+    ('before-call.neptune.CreateDBCluster', inject_presigned_url_rds),
     # Neptune PresignedUrl documentation customizations
     (
         'docs.*.neptune.CopyDBClusterSnapshot.complete-section',
@@ -1411,6 +1409,8 @@ def urlencode_body(model, params, context, **kwargs):
     #############
     # DocDB
     #############
+    ('before-call.docdb.CopyDBClusterSnapshot', inject_presigned_url_rds),
+    ('before-call.docdb.CreateDBCluster', inject_presigned_url_rds),
     # DocDB PresignedUrl documentation customizations
     (
         'docs.*.docdb.CopyDBClusterSnapshot.complete-section',

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -40,6 +40,16 @@
 _LOADER = botocore.loaders.Loader()
 
 
+def _all_services():
+    session = botocore.session.Session()
+    service_names = session.get_available_services()
+    return [session.get_service_model(name) for name in service_names]
+
+
+# Only compute our service models once
+ALL_SERVICES = _all_services()
+
+
 def skip_unless_has_memory_collection(cls):
     """Class decorator to skip tests that require memory collection.
 
@@ -575,3 +585,16 @@ def __enter__(self, *args, **kwargs):
 
     def __exit__(self, *args, **kwargs):
         self.datetime_patcher.stop()
+
+
+def patch_load_service_model(
+    session, monkeypatch, service_model_json, ruleset_json
+):
+    def mock_load_service_model(service_name, type_name, api_version=None):
+        if type_name == 'service-2':
+            return service_model_json
+        if type_name == 'endpoint-rule-set-1':
+            return ruleset_json
+
+    loader = session.get_component('data_loader')
+    monkeypatch.setattr(loader, 'load_service_model', mock_load_service_model)
diff --git a/tests/functional/test_compression.py b/tests/functional/test_compression.py
@@ -0,0 +1,132 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import gzip
+
+import pytest
+
+from botocore.config import Config
+from tests import ALL_SERVICES, ClientHTTPStubber, patch_load_service_model
+
+KNOWN_COMPRESSION_ENCODINGS = 'gzip'
+
+FAKE_MODEL = {
+    "version": "2.0",
+    "documentation": "",
+    "metadata": {
+        "apiVersion": "2020-02-02",
+        "endpointPrefix": "otherservice",
+        "protocol": "query",
+        "serviceFullName": "Other Service",
+        "serviceId": "Other Service",
+        "signatureVersion": "v4",
+        "signingName": "otherservice",
+        "uid": "otherservice-2020-02-02",
+    },
+    "operations": {
+        "MockOperation": {
+            "name": "MockOperation",
+            "http": {"method": "POST", "requestUri": "/"},
+            "input": {"shape": "MockOperationRequest"},
+            "documentation": "",
+            "requestcompression": {
+                "encodings": ["gzip"],
+            },
+        },
+    },
+    "shapes": {
+        "MockOpParamList": {
+            "type": "list",
+            "member": {"shape": "MockOpParam"},
+        },
+        "MockOpParam": {
+            "type": "structure",
+            "members": {"MockOpParam": {"shape": "MockOpParamValue"}},
+        },
+        "MockOpParamValue": {
+            "type": "string",
+        },
+        "MockOperationRequest": {
+            "type": "structure",
+            "required": ["MockOpParamList"],
+            "members": {
+                "MockOpParamList": {
+                    "shape": "MockOpParamList",
+                    "documentation": "",
+                },
+            },
+        },
+    },
+}
+
+FAKE_RULESET = {
+    "version": "1.0",
+    "parameters": {},
+    "rules": [
+        {
+            "conditions": [],
+            "type": "endpoint",
+            "endpoint": {
+                "url": "https://foo.bar",
+                "properties": {},
+                "headers": {},
+            },
+        }
+    ],
+}
+
+
+def _all_compression_operations():
+    for service_model in ALL_SERVICES:
+        for operation_name in service_model.operation_names:
+            operation_model = service_model.operation_model(operation_name)
+            if operation_model.request_compression is not None:
+                yield operation_model
+
+
+@pytest.mark.parametrize("operation_model", _all_compression_operations())
+def test_no_unknown_compression_encodings(operation_model):
+    for encoding in operation_model.request_compression["encodings"]:
+        assert (
+            encoding in KNOWN_COMPRESSION_ENCODINGS
+        ), f"Found unknown compression encoding '{encoding}' in operation {operation_model.name}"
+
+
+def test_compression(patched_session, monkeypatch):
+    patch_load_service_model(
+        patched_session, monkeypatch, FAKE_MODEL, FAKE_RULESET
+    )
+    client = patched_session.create_client(
+        'otherservice',
+        region_name='us-west-2',
+        config=Config(request_min_compression_size_bytes=100),
+    )
+    with ClientHTTPStubber(client, strict=True) as http_stubber:
+        http_stubber.add_response(
+            status=200,
+            body=b'<response><status>success</status><message>Request processed successfully</message></response>',
+        )
+        client.mock_operation(
+            MockOpParamList=[
+                {"MockOpParam": f"MockOpParamValue{i}"} for i in range(1, 21)
+            ]
+        )
+        serialized_body = b"&".join(
+            [
+                f"MockOpParamList.member.{i}.MockOpParam=MockOpParamValue{i}".encode()
+                for i in range(1, 21)
+            ]
+        )
+        assert serialized_body in gzip.decompress(
+            http_stubber.requests[0].body
+        )
diff --git a/tests/functional/test_context_params.py b/tests/functional/test_context_params.py
@@ -14,7 +14,7 @@
 import pytest
 
 from botocore.config import Config
-from tests import ClientHTTPStubber, mock
+from tests import ClientHTTPStubber, mock, patch_load_service_model
 
 # fake rulesets compatible with all fake service models below
 FAKE_RULESET_TEMPLATE = {
@@ -226,19 +226,6 @@
 }
 
 
-def patch_load_service_model(
-    session, monkeypatch, service_model_json, ruleset_json
-):
-    def mock_load_service_model(service_name, type_name, api_version=None):
-        if type_name == 'service-2':
-            return service_model_json
-        if type_name == 'endpoint-rule-set-1':
-            return ruleset_json
-
-    loader = session.get_component('data_loader')
-    monkeypatch.setattr(loader, 'load_service_model', mock_load_service_model)
-
-
 @pytest.mark.parametrize(
     'service_name,service_model,ruleset,call_should_include_ctx_param',
     [
@@ -411,7 +398,6 @@ def test_static_context_param_sent_to_endpoint_resolver(
     client = patched_session.create_client(
         service_name, region_name='us-east-1'
     )
-
     with ClientHTTPStubber(client, strict=True) as http_stubber:
         http_stubber.add_response(status=200)
         with mock.patch.object(

diff --git a/tests/functional/test_response_shadowing.py b/tests/functional/test_response_shadowing.py
@@ -12,17 +12,7 @@
 # language governing permissions and limitations under the License.
 import pytest
 
-from botocore.session import Session
-
-
-def _all_services():
-    session = Session()
-    service_names = session.get_available_services()
-    return [session.get_service_model(name) for name in service_names]
-
-
-# Only compute our service models once
-ALL_SERVICES = _all_services()
+from tests import ALL_SERVICES
 
 
 def _all_service_error_shapes():