From 54a0be45ef6199605de3f9d6ed6802472403528d Mon Sep 17 00:00:00 2001 From: Ilya Konstantinov Date: Tue, 25 Oct 2022 11:26:28 -0400 Subject: [PATCH] Workaround for _convert_to_request_dict change (#1083) botocore 1.28 changed the signature of private method botocore.client.BaseClient._convert_to_request_dict adding an endpoint_url parameter. We are updating pynamodb to inspect the signature and add this parameter as needed. --- docs/release_notes.rst | 10 ++++++ pynamodb/__init__.py | 2 +- pynamodb/connection/_botocore_private.py | 46 ++++++++++++++++++++++++ pynamodb/connection/base.py | 40 ++++++++++++++++----- tests/test_base_connection.py | 19 ++++++++++ 5 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 pynamodb/connection/_botocore_private.py diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 5317786fc..47bdfd7e2 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,6 +1,13 @@ Release Notes ============= +v4.4.0 +---------- +* Update for botocore 1.28 private API change (#1130) which caused the following exception:: + + TypeError: _convert_to_request_dict() missing 1 required positional argument: 'endpoint_url' + + v4.3.3 ---------- @@ -14,6 +21,9 @@ v4.3.3 MyModel.query(..., condition=MyModel.my_list[0] == 42) +* Fix a warning about ``collections.abc`` deprecation (#782) + + v4.3.2 ---------- diff --git a/pynamodb/__init__.py b/pynamodb/__init__.py index 99e639646..806321a91 100644 --- a/pynamodb/__init__.py +++ b/pynamodb/__init__.py @@ -7,4 +7,4 @@ """ __author__ = 'Jharrod LaFon' __license__ = 'MIT' -__version__ = '4.3.3' +__version__ = '4.4.0' diff --git a/pynamodb/connection/_botocore_private.py b/pynamodb/connection/_botocore_private.py new file mode 100644 index 000000000..ac4e8aab5 --- /dev/null +++ b/pynamodb/connection/_botocore_private.py @@ -0,0 +1,46 @@ +""" +Type-annotates the private botocore APIs that we're currently relying on. +""" +from typing import Any, Dict, Optional + +import botocore.client +import botocore.credentials +import botocore.endpoint +import botocore.hooks +import botocore.model +import botocore.signers + + +class BotocoreEndpointPrivate(botocore.endpoint.Endpoint): + _event_emitter: botocore.hooks.HierarchicalEmitter + + +class BotocoreRequestSignerPrivate(botocore.signers.RequestSigner): + _credentials: botocore.credentials.Credentials + + +class BotocoreBaseClientPrivate(botocore.client.BaseClient): + _endpoint: BotocoreEndpointPrivate + _request_signer: BotocoreRequestSignerPrivate + _service_model: botocore.model.ServiceModel + + def _resolve_endpoint_ruleset( + self, + operation_model: botocore.model.OperationModel, + params: Dict[str, Any], + request_context: Dict[str, Any], + ignore_signing_region: bool = ..., + ): + ... + + def _convert_to_request_dict( + self, + api_params: Dict[str, Any], + operation_model: botocore.model.OperationModel, + *, + endpoint_url: str = ..., # added in botocore 1.28 + context: Optional[Dict[str, Any]] = ..., + headers: Optional[Dict[str, Any]] = ..., + set_user_agent_header: bool = ..., + ) -> Dict[str, Any]: + ... diff --git a/pynamodb/connection/base.py b/pynamodb/connection/base.py index b6b1ab8ec..2ef176751 100644 --- a/pynamodb/connection/base.py +++ b/pynamodb/connection/base.py @@ -3,6 +3,7 @@ """ from __future__ import division +import inspect import json import logging import random @@ -11,6 +12,7 @@ import uuid from base64 import b64decode from threading import local +from typing import Any, Dict, List, Mapping, Optional, Sequence, cast import six import botocore.client @@ -22,6 +24,7 @@ from botocore.session import get_session from six.moves import range +from pynamodb.connection._botocore_private import BotocoreBaseClientPrivate from pynamodb.connection.util import pythonic from pynamodb.constants import ( RETURN_CONSUMED_CAPACITY_VALUES, RETURN_ITEM_COLL_METRICS_VALUES, @@ -247,7 +250,8 @@ def __init__(self, region=None, host=None, self._tables = {} self.host = host self._local = local() - self._client = None + self._client: Optional[BotocoreBaseClientPrivate] = None + self._convert_to_request_dict__endpoint_url = False if region: self.region = region else: @@ -364,10 +368,28 @@ def _make_api_call(self, operation_name, operation_kwargs): 2. It provides a place to monkey patch HTTP requests for unit testing """ operation_model = self.client._service_model.operation_model(operation_name) - request_dict = self.client._convert_to_request_dict( - operation_kwargs, - operation_model, - ) + if self._convert_to_request_dict__endpoint_url: + request_context = { + 'client_region': self.region, + 'client_config': self.client.meta.config, + 'has_streaming_input': operation_model.has_streaming_input, + 'auth_type': operation_model.auth_type, + } + endpoint_url, additional_headers = self.client._resolve_endpoint_ruleset( + operation_model, operation_kwargs, request_context + ) + request_dict = self.client._convert_to_request_dict( + api_params=operation_kwargs, + operation_model=operation_model, + endpoint_url=endpoint_url, + context=request_context, + headers=additional_headers, + ) + else: + request_dict = self.client._convert_to_request_dict( + operation_kwargs, + operation_model, + ) for i in range(0, self._max_retry_attempts_exception + 1): attempt_number = i + 1 @@ -518,7 +540,7 @@ def session(self): return self._local.session @property - def client(self): + def client(self) -> BotocoreBaseClientPrivate: """ Returns a botocore dynamodb client """ @@ -531,8 +553,10 @@ def client(self): parameter_validation=False, # Disable unnecessary validation for performance connect_timeout=self._connect_timeout_seconds, read_timeout=self._read_timeout_seconds, - max_pool_connections=self._max_pool_connections) - self._client = self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config) + max_pool_connections=self._max_pool_connections, + ) + self._client = cast(BotocoreBaseClientPrivate, self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config)) + self._convert_to_request_dict__endpoint_url = 'endpoint_url' in inspect.signature(self._client._convert_to_request_dict).parameters return self._client def get_meta_table(self, table_name, refresh=False): diff --git a/tests/test_base_connection.py b/tests/test_base_connection.py index 224c08407..af9738c92 100644 --- a/tests/test_base_connection.py +++ b/tests/test_base_connection.py @@ -2,11 +2,14 @@ Tests for the base connection class """ import base64 +import io import json import six from unittest import TestCase import botocore.exceptions +import botocore.httpsession +import urllib3 from botocore.awsrequest import AWSPreparedRequest, AWSRequest, AWSResponse from botocore.client import ClientError from botocore.exceptions import BotoCoreError @@ -1449,6 +1452,22 @@ def test_scan(self): conn.scan, table_name) + def test_make_api_call__happy_path(self): + response = AWSResponse( + url='https://www.example.com', + status_code=200, + raw=urllib3.HTTPResponse( + body=io.BytesIO(json.dumps({}).encode('utf-8')), + preload_content=False, + ), + headers={'x-amzn-RequestId': 'abcdef'}, + ) + + c = Connection() + + with patch.object(botocore.httpsession.URLLib3Session, 'send', return_value=response): + c._make_api_call('CreateTable', {'TableName': 'MyTable'}) + @mock.patch('pynamodb.connection.Connection.client') def test_make_api_call_throws_verbose_error_after_backoff(self, client_mock): response = AWSResponse(