diff --git a/botocore/args.py b/botocore/args.py index 2cd503a06f..d1f3ed81f4 100644 --- a/botocore/args.py +++ b/botocore/args.py @@ -520,19 +520,20 @@ def _compute_request_compression_config(self, config_kwargs): config_kwargs['disable_request_compression'] = disabled def _validate_min_compression_size(self, min_size): - try: - min_size = int(min_size) - except ValueError: - raise botocore.exceptions.InvalidConfigError( - f"Invalid value '{min_size}' for " - "request_min_compression_size_bytes. Value must be an integer." - ) - if not 0 <= min_size <= 1048576: - raise botocore.exceptions.InvalidConfigError( - f"Invalid value '{min_size}' for " - "request_min_compression_size_bytes. Value must be between 0 " - "and 1048576." - ) + if min_size is not None: + try: + min_size = int(min_size) + except ValueError: + raise botocore.exceptions.InvalidConfigError( + error_msg=f"Invalid value '{min_size}' for " + "request_min_compression_size_bytes. Value must be an integer." + ) + if not 0 <= min_size <= 1048576: + raise botocore.exceptions.InvalidConfigError( + error_msg=f"Invalid value '{min_size}' for " + "request_min_compression_size_bytes. Value must be between 0 " + "and 1048576." + ) def _ensure_boolean(self, val): if isinstance(val, bool): diff --git a/botocore/awsrequest.py b/botocore/awsrequest.py index ef93bdd4d8..f8b973f1af 100644 --- a/botocore/awsrequest.py +++ b/botocore/awsrequest.py @@ -540,8 +540,7 @@ class RequestCompressor: """A class that can compress the body of an ``AWSRequest``.""" def compress(self, config, request_dict, operation_model): - """Compresses the request body using the specified encodings if conditions - are met. + """Compresses the request body using the specified encodings. Check if the request should be compressed based on the contents of its body and config settings. Set or append the `Content-Encoding` header @@ -554,14 +553,11 @@ def compress(self, config, request_dict, operation_model): for encoding in encodings: encoder = getattr(self, f'_{encoding}_compress_body', None) if encoder is not None: - if 'Content-Encoding' not in headers: + ce_header = headers.get('Content-Encoding') + if ce_header is None: headers['Content-Encoding'] = encoding - elif encoding not in headers['Content-Encoding'].split( - ',' - ): - headers[ - 'Content-Encoding' - ] = f'{headers["Content-Encoding"]},{encoding}' + elif encoding not in ce_header.split(','): + headers['Content-Encoding'] = f'{ce_header},{encoding}' request_dict['body'] = encoder(body) if 'headers' not in request_dict: request_dict['headers'] = headers diff --git a/botocore/endpoint.py b/botocore/endpoint.py index 5b228de175..adc622c25a 100644 --- a/botocore/endpoint.py +++ b/botocore/endpoint.py @@ -75,8 +75,14 @@ def convert_to_response_dict(http_response, operation_model): class Endpoint: - """Represents an endpoint for a particular service in a specific region. - Only an endpoint can make requests. + """ + Represents an endpoint for a particular service in a specific + region. Only an endpoint can make requests. + + :ivar service: The Service object that describes this endpoints + service. + :ivar host: The fully qualified endpoint hostname. + :ivar session: The session object. """ def __init__( @@ -115,12 +121,6 @@ def make_request(self, operation_model, request_dict): def create_request(self, params, operation_model=None): request = create_request_object(params) if operation_model: - if operation_model.request_compression: - self._request_compressor.compress( - params['context']['client_config'], - request, - operation_model, - ) request.stream_output = any( [ operation_model.has_streaming_output, diff --git a/botocore/handlers.py b/botocore/handlers.py index fa5654cbff..6644a2810c 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -1147,10 +1147,10 @@ def remove_content_type_header_for_presigning(request, **kwargs): def urlencode_body(model, params, context, **kwargs): - """Urlencode the request body if it is a dictionary. + """Url encode the request body if it is a dictionary. - This is used for services like S3 that require the body to be urlencoded - when the body is a dictionary. + This is used for services using the query protocol. The body must be serialized + as a urlencoded string before it can be compressed. """ body = params.get('body') if ( @@ -1222,25 +1222,13 @@ def urlencode_body(model, params, context, **kwargs): ('before-call.s3', add_expect_header), ('before-call.glacier', add_glacier_version), ('before-call.apigateway', add_accept_header), - ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2), - ('before-call.rds.CopyDBClusterSnapshot', inject_presigned_url_rds), - ('before-call.rds.CreateDBCluster', inject_presigned_url_rds), - ('before-call.rds.CopyDBSnapshot', inject_presigned_url_rds), - ('before-call.rds.CreateDBInstanceReadReplica', inject_presigned_url_rds), - ( - 'before-call.rds.StartDBInstanceAutomatedBackupsReplication', - inject_presigned_url_rds, - ), - ('before-call.neptune.CopyDBClusterSnapshot', inject_presigned_url_rds), - ('before-call.neptune.CreateDBCluster', inject_presigned_url_rds), - ('before-call.docdb.CopyDBClusterSnapshot', inject_presigned_url_rds), - ('before-call.docdb.CreateDBCluster', inject_presigned_url_rds), ('before-call.s3.PutObject', conditionally_calculate_md5), ('before-call.s3.UploadPart', conditionally_calculate_md5), ('before-call.s3.DeleteObjects', escape_xml_payload), ('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload), ('before-call.glacier.UploadArchive', add_glacier_checksums), ('before-call.glacier.UploadMultipartPart', add_glacier_checksums), + ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2), ('request-created', add_retry_headers), ('request-created.machinelearning.Predict', switch_host_machinelearning), ('needs-retry.s3.UploadPartCopy', check_for_200_error, REGISTER_FIRST), @@ -1375,6 +1363,14 @@ def urlencode_body(model, params, context, **kwargs): # RDS ############# ('creating-client-class.rds', add_generate_db_auth_token), + ('before-call.rds.CopyDBClusterSnapshot', inject_presigned_url_rds), + ('before-call.rds.CreateDBCluster', inject_presigned_url_rds), + ('before-call.rds.CopyDBSnapshot', inject_presigned_url_rds), + ('before-call.rds.CreateDBInstanceReadReplica', inject_presigned_url_rds), + ( + 'before-call.rds.StartDBInstanceAutomatedBackupsReplication', + inject_presigned_url_rds, + ), # RDS PresignedUrl documentation customizations ( 'docs.*.rds.CopyDBClusterSnapshot.complete-section', @@ -1399,6 +1395,8 @@ def urlencode_body(model, params, context, **kwargs): ############# # Neptune ############# + ('before-call.neptune.CopyDBClusterSnapshot', inject_presigned_url_rds), + ('before-call.neptune.CreateDBCluster', inject_presigned_url_rds), # Neptune PresignedUrl documentation customizations ( 'docs.*.neptune.CopyDBClusterSnapshot.complete-section', @@ -1411,6 +1409,8 @@ def urlencode_body(model, params, context, **kwargs): ############# # DocDB ############# + ('before-call.docdb.CopyDBClusterSnapshot', inject_presigned_url_rds), + ('before-call.docdb.CreateDBCluster', inject_presigned_url_rds), # DocDB PresignedUrl documentation customizations ( 'docs.*.docdb.CopyDBClusterSnapshot.complete-section', diff --git a/tests/__init__.py b/tests/__init__.py index 3bd5a22a28..33307c0081 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -40,6 +40,16 @@ _LOADER = botocore.loaders.Loader() +def _all_services(): + session = botocore.session.Session() + service_names = session.get_available_services() + return [session.get_service_model(name) for name in service_names] + + +# Only compute our service models once +ALL_SERVICES = _all_services() + + def skip_unless_has_memory_collection(cls): """Class decorator to skip tests that require memory collection. @@ -575,3 +585,16 @@ def __enter__(self, *args, **kwargs): def __exit__(self, *args, **kwargs): self.datetime_patcher.stop() + + +def patch_load_service_model( + session, monkeypatch, service_model_json, ruleset_json +): + def mock_load_service_model(service_name, type_name, api_version=None): + if type_name == 'service-2': + return service_model_json + if type_name == 'endpoint-rule-set-1': + return ruleset_json + + loader = session.get_component('data_loader') + monkeypatch.setattr(loader, 'load_service_model', mock_load_service_model) diff --git a/tests/functional/test_compression.py b/tests/functional/test_compression.py new file mode 100644 index 0000000000..97f02ba4ae --- /dev/null +++ b/tests/functional/test_compression.py @@ -0,0 +1,132 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import gzip + +import pytest + +from botocore.config import Config +from tests import ALL_SERVICES, ClientHTTPStubber, patch_load_service_model + +KNOWN_COMPRESSION_ENCODINGS = 'gzip' + +FAKE_MODEL = { + "version": "2.0", + "documentation": "", + "metadata": { + "apiVersion": "2020-02-02", + "endpointPrefix": "otherservice", + "protocol": "query", + "serviceFullName": "Other Service", + "serviceId": "Other Service", + "signatureVersion": "v4", + "signingName": "otherservice", + "uid": "otherservice-2020-02-02", + }, + "operations": { + "MockOperation": { + "name": "MockOperation", + "http": {"method": "POST", "requestUri": "/"}, + "input": {"shape": "MockOperationRequest"}, + "documentation": "", + "requestcompression": { + "encodings": ["gzip"], + }, + }, + }, + "shapes": { + "MockOpParamList": { + "type": "list", + "member": {"shape": "MockOpParam"}, + }, + "MockOpParam": { + "type": "structure", + "members": {"MockOpParam": {"shape": "MockOpParamValue"}}, + }, + "MockOpParamValue": { + "type": "string", + }, + "MockOperationRequest": { + "type": "structure", + "required": ["MockOpParamList"], + "members": { + "MockOpParamList": { + "shape": "MockOpParamList", + "documentation": "", + }, + }, + }, + }, +} + +FAKE_RULESET = { + "version": "1.0", + "parameters": {}, + "rules": [ + { + "conditions": [], + "type": "endpoint", + "endpoint": { + "url": "https://foo.bar", + "properties": {}, + "headers": {}, + }, + } + ], +} + + +def _all_compression_operations(): + for service_model in ALL_SERVICES: + for operation_name in service_model.operation_names: + operation_model = service_model.operation_model(operation_name) + if operation_model.request_compression is not None: + yield operation_model + + +@pytest.mark.parametrize("operation_model", _all_compression_operations()) +def test_no_unknown_compression_encodings(operation_model): + for encoding in operation_model.request_compression["encodings"]: + assert ( + encoding in KNOWN_COMPRESSION_ENCODINGS + ), f"Found unknown compression encoding '{encoding}' in operation {operation_model.name}" + + +def test_compression(patched_session, monkeypatch): + patch_load_service_model( + patched_session, monkeypatch, FAKE_MODEL, FAKE_RULESET + ) + client = patched_session.create_client( + 'otherservice', + region_name='us-west-2', + config=Config(request_min_compression_size_bytes=100), + ) + with ClientHTTPStubber(client, strict=True) as http_stubber: + http_stubber.add_response( + status=200, + body=b'successRequest processed successfully', + ) + client.mock_operation( + MockOpParamList=[ + {"MockOpParam": f"MockOpParamValue{i}"} for i in range(1, 21) + ] + ) + serialized_body = b"&".join( + [ + f"MockOpParamList.member.{i}.MockOpParam=MockOpParamValue{i}".encode() + for i in range(1, 21) + ] + ) + assert serialized_body in gzip.decompress( + http_stubber.requests[0].body + ) diff --git a/tests/functional/test_context_params.py b/tests/functional/test_context_params.py index d72b65b265..e3c501ead9 100644 --- a/tests/functional/test_context_params.py +++ b/tests/functional/test_context_params.py @@ -14,7 +14,7 @@ import pytest from botocore.config import Config -from tests import ClientHTTPStubber, mock +from tests import ClientHTTPStubber, mock, patch_load_service_model # fake rulesets compatible with all fake service models below FAKE_RULESET_TEMPLATE = { @@ -226,19 +226,6 @@ } -def patch_load_service_model( - session, monkeypatch, service_model_json, ruleset_json -): - def mock_load_service_model(service_name, type_name, api_version=None): - if type_name == 'service-2': - return service_model_json - if type_name == 'endpoint-rule-set-1': - return ruleset_json - - loader = session.get_component('data_loader') - monkeypatch.setattr(loader, 'load_service_model', mock_load_service_model) - - @pytest.mark.parametrize( 'service_name,service_model,ruleset,call_should_include_ctx_param', [ @@ -411,7 +398,6 @@ def test_static_context_param_sent_to_endpoint_resolver( client = patched_session.create_client( service_name, region_name='us-east-1' ) - with ClientHTTPStubber(client, strict=True) as http_stubber: http_stubber.add_response(status=200) with mock.patch.object( diff --git a/tests/functional/test_response_shadowing.py b/tests/functional/test_response_shadowing.py index 708e989231..a5cbaac047 100644 --- a/tests/functional/test_response_shadowing.py +++ b/tests/functional/test_response_shadowing.py @@ -12,17 +12,7 @@ # language governing permissions and limitations under the License. import pytest -from botocore.session import Session - - -def _all_services(): - session = Session() - service_names = session.get_available_services() - return [session.get_service_model(name) for name in service_names] - - -# Only compute our service models once -ALL_SERVICES = _all_services() +from tests import ALL_SERVICES def _all_service_error_shapes(): diff --git a/tests/unit/test_awsrequest.py b/tests/unit/test_awsrequest.py index 45c1552b6d..53979b6b70 100644 --- a/tests/unit/test_awsrequest.py +++ b/tests/unit/test_awsrequest.py @@ -31,7 +31,7 @@ create_request_object, prepare_request_dict, ) -from botocore.compat import HTTPHeaders, file_type +from botocore.compat import file_type from botocore.config import Config from botocore.exceptions import UnseekableStreamError from tests import mock, unittest @@ -168,7 +168,6 @@ def _streaming_op_with_compression_requires_length(): disable_request_compression=False, request_min_compression_size_bytes=0, ) -GZIP_ENCODINGS = ['gzip'] def request_dict(): @@ -190,17 +189,15 @@ def request_compressor(): return RequestCompressor() -@pytest.fixture(scope="module") -def headers(): - return HTTPHeaders() +COMPRESSION_HEADERS = {'gzip': b'\x1f\x8b'} -def _assert_compression(is_compressed, body): +def _assert_compression(is_compressed, body, encoding): if hasattr(body, 'read'): header = body.read(2) else: header = body[:2] - assert is_compressed == (header == b'\x1f\x8b') + assert is_compressed == (header == COMPRESSION_HEADERS.get(encoding)) class TestAWSRequest(unittest.TestCase): @@ -1012,7 +1009,7 @@ def test_compress( encoding, ): request_compressor.compress(config, request_dict, operation_model) - _assert_compression(is_compressed, request_dict['body']) + _assert_compression(is_compressed, request_dict['body'], encoding) assert ( 'headers' in request_dict and 'Content-Encoding' in request_dict['headers'] diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index f3f3eb7216..b966358b56 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -27,6 +27,7 @@ from botocore.endpoint import DEFAULT_TIMEOUT from botocore.errorfactory import ClientExceptionsFactory from botocore.exceptions import ( + InvalidConfigError, InvalidMaxRetryAttemptsError, InvalidRetryConfigurationError, InvalidRetryModeError, @@ -1739,6 +1740,18 @@ def test_request_compression_client_config_overrides_config_store(self): service_client.meta.config.disable_request_compression, False ) + def test_bad_request_min_compression_size_bytes(self): + self.config_store.set_config_variable( + 'request_min_compression_size_bytes', 'foo' + ) + creator = self.create_client_creator() + with self.assertRaises(InvalidConfigError): + creator.create_client( + 'myservice', + 'us-west-2', + credentials=self.credentials, + ) + class TestClientErrors(TestAutoGeneratedClient): def add_error_response(self, error_response): diff --git a/tests/unit/test_endpoint.py b/tests/unit/test_endpoint.py index 33930b7746..a2dc45755a 100644 --- a/tests/unit/test_endpoint.py +++ b/tests/unit/test_endpoint.py @@ -59,7 +59,6 @@ def setUp(self): self.op = mock.Mock() self.op.has_streaming_output = False self.op.has_event_stream_output = False - self.op.request_compression = None self.op.metadata = {'protocol': 'json'} self.event_emitter = mock.Mock() self.event_emitter.emit.return_value = [] @@ -217,7 +216,6 @@ def setUp(self): self._operation.service_model.service_id = ServiceId('EC2') self._operation.has_streaming_output = False self._operation.has_event_stream_output = False - self._operation.request_compression = None def assert_events_emitted(self, event_emitter, expected_events): self.assertEqual( @@ -308,7 +306,6 @@ def test_reset_stream_on_retry(self): op.name = 'PutObject' op.has_streaming_output = True op.has_event_stream_output = False - op.request_compression = None op.metadata = {'protocol': 'rest-xml'} request = request_dict() request['body'] = body