Workaround for _convert_to_request_dict change (#1083)

botocore 1.28 changed the signature of private method botocore.client.BaseClient._convert_to_request_dict adding an endpoint_url parameter. We are updating pynamodb to inspect the signature and add this parameter as needed.

docs/release_notes.rst

@@ -1,6 +1,13 @@
 Release Notes
 =============
 
+v4.4.0
+----------
+* Update for botocore 1.28 private API change (#1130) which caused the following exception::
+
+    TypeError: _convert_to_request_dict() missing 1 required positional argument: 'endpoint_url'
+
+
 v4.3.3
 ----------
 
@@ -14,6 +21,9 @@ v4.3.3
 
     MyModel.query(..., condition=MyModel.my_list[0] == 42)
 
+* Fix a warning about ``collections.abc`` deprecation (#782)
+
+
 v4.3.2
 ----------
 

pynamodb/__init__.py

@@ -7,4 +7,4 @@
 """
 __author__ = 'Jharrod LaFon'
 __license__ = 'MIT'
-__version__ = '4.3.3'
+__version__ = '4.4.0'

pynamodb/connection/_botocore_private.py

@@ -0,0 +1,46 @@
+"""
+Type-annotates the private botocore APIs that we're currently relying on.
+"""
+from typing import Any, Dict, Optional
+
+import botocore.client
+import botocore.credentials
+import botocore.endpoint
+import botocore.hooks
+import botocore.model
+import botocore.signers
+
+
+class BotocoreEndpointPrivate(botocore.endpoint.Endpoint):
+    _event_emitter: botocore.hooks.HierarchicalEmitter
+
+
+class BotocoreRequestSignerPrivate(botocore.signers.RequestSigner):
+    _credentials: botocore.credentials.Credentials
+
+
+class BotocoreBaseClientPrivate(botocore.client.BaseClient):
+    _endpoint: BotocoreEndpointPrivate
+    _request_signer: BotocoreRequestSignerPrivate
+    _service_model: botocore.model.ServiceModel
+
+    def _resolve_endpoint_ruleset(
+        self,
+        operation_model: botocore.model.OperationModel,
+        params: Dict[str, Any],
+        request_context: Dict[str, Any],
+        ignore_signing_region: bool = ...,
+    ):
+        ...
+
+    def _convert_to_request_dict(
+        self,
+        api_params: Dict[str, Any],
+        operation_model: botocore.model.OperationModel,
+        *,
+        endpoint_url: str = ...,  # added in botocore 1.28
+        context: Optional[Dict[str, Any]] = ...,
+        headers: Optional[Dict[str, Any]] = ...,
+        set_user_agent_header: bool = ...,
+    ) -> Dict[str, Any]:
+        ...

pynamodb/connection/base.py

@@ -3,6 +3,7 @@
 """
 from __future__ import division
 
+import inspect
 import json
 import logging
 import random
@@ -11,6 +12,7 @@
 import uuid
 from base64 import b64decode
 from threading import local
+from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
 
 import six
 import botocore.client
@@ -22,6 +24,7 @@
 from botocore.session import get_session
 from six.moves import range
 
+from pynamodb.connection._botocore_private import BotocoreBaseClientPrivate
 from pynamodb.connection.util import pythonic
 from pynamodb.constants import (
     RETURN_CONSUMED_CAPACITY_VALUES, RETURN_ITEM_COLL_METRICS_VALUES,
@@ -247,7 +250,8 @@ def __init__(self, region=None, host=None,
         self._tables = {}
         self.host = host
         self._local = local()
-        self._client = None
+        self._client: Optional[BotocoreBaseClientPrivate] = None
+        self._convert_to_request_dict__endpoint_url = False
         if region:
             self.region = region
         else:
@@ -364,10 +368,28 @@ def _make_api_call(self, operation_name, operation_kwargs):
         2. It provides a place to monkey patch HTTP requests for unit testing
         """
         operation_model = self.client._service_model.operation_model(operation_name)
-        request_dict = self.client._convert_to_request_dict(
-            operation_kwargs,
-            operation_model,
-        )
+        if self._convert_to_request_dict__endpoint_url:
+            request_context = {
+                'client_region': self.region,
+                'client_config': self.client.meta.config,
+                'has_streaming_input': operation_model.has_streaming_input,
+                'auth_type': operation_model.auth_type,
+            }
+            endpoint_url, additional_headers = self.client._resolve_endpoint_ruleset(
+                operation_model, operation_kwargs, request_context
+            )
+            request_dict = self.client._convert_to_request_dict(
+                api_params=operation_kwargs,
+                operation_model=operation_model,
+                endpoint_url=endpoint_url,
+                context=request_context,
+                headers=additional_headers,
+            )
+        else:
+            request_dict = self.client._convert_to_request_dict(
+                operation_kwargs,
+                operation_model,
+            )
 
         for i in range(0, self._max_retry_attempts_exception + 1):
             attempt_number = i + 1
@@ -518,7 +540,7 @@ def session(self):
         return self._local.session
 
     @property
-    def client(self):
+    def client(self) -> BotocoreBaseClientPrivate:
         """
         Returns a botocore dynamodb client
         """
@@ -531,8 +553,10 @@ def client(self):
                 parameter_validation=False,  # Disable unnecessary validation for performance
                 connect_timeout=self._connect_timeout_seconds,
                 read_timeout=self._read_timeout_seconds,
-                max_pool_connections=self._max_pool_connections)
-            self._client = self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config)
+                max_pool_connections=self._max_pool_connections,
+            )
+            self._client = cast(BotocoreBaseClientPrivate, self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config))
+            self._convert_to_request_dict__endpoint_url = 'endpoint_url' in inspect.signature(self._client._convert_to_request_dict).parameters
         return self._client
 
     def get_meta_table(self, table_name, refresh=False):

tests/test_base_connection.py

@@ -2,11 +2,14 @@
 Tests for the base connection class
 """
 import base64
+import io
 import json
 import six
 from unittest import TestCase
 
 import botocore.exceptions
+import botocore.httpsession
+import urllib3
 from botocore.awsrequest import AWSPreparedRequest, AWSRequest, AWSResponse
 from botocore.client import ClientError
 from botocore.exceptions import BotoCoreError
@@ -1449,6 +1452,22 @@ def test_scan(self):
                 conn.scan,
                 table_name)
 
+    def test_make_api_call__happy_path(self):
+        response = AWSResponse(
+            url='https://www.example.com',
+            status_code=200,
+            raw=urllib3.HTTPResponse(
+                body=io.BytesIO(json.dumps({}).encode('utf-8')),
+                preload_content=False,
+            ),
+            headers={'x-amzn-RequestId': 'abcdef'},
+        )
+
+        c = Connection()
+
+        with patch.object(botocore.httpsession.URLLib3Session, 'send', return_value=response):
+            c._make_api_call('CreateTable', {'TableName': 'MyTable'})
+
     @mock.patch('pynamodb.connection.Connection.client')
     def test_make_api_call_throws_verbose_error_after_backoff(self, client_mock):
         response = AWSResponse(