bump botocore dependency specification

CHANGES.rst

@@ -1,6 +1,10 @@
 Changes
 -------
 
+2.9.0 (2023-12-05)
+^^^^^^^^^^^^^^^^^^
+* bump botocore dependency specification
+
 2.8.0 (2023-11-28)
 ^^^^^^^^^^^^^^^^^^
 * add AioStubber that returns AioAWSResponse()

aiobotocore/__init__.py

@@ -1 +1 @@
-__version__ = '2.8.0'
+__version__ = '2.9.0'

aiobotocore/client.py

@@ -17,11 +17,12 @@
 
 from . import waiter
 from .args import AioClientArgsCreator
+from .credentials import AioRefreshableCredentials
 from .discovery import AioEndpointDiscoveryHandler, AioEndpointDiscoveryManager
 from .httpchecksum import apply_request_checksum
 from .paginate import AioPaginator
 from .retries import adaptive, standard
-from .utils import AioS3RegionRedirectorv2
+from .utils import AioS3ExpressIdentityResolver, AioS3RegionRedirectorv2
 
 history_recorder = get_global_history_recorder()
 
@@ -96,6 +97,7 @@ async def create_client(
             client_config=client_config,
             scoped_config=scoped_config,
         )
+        self._register_s3express_events(client=service_client)
         self._register_s3_control_events(client=service_client)
         self._register_endpoint_discovery(
             service_client, endpoint_url, client_config
@@ -223,6 +225,20 @@ def _register_endpoint_discovery(self, client, endpoint_url, config):
                 block_endpoint_discovery_required_operations,
             )
 
+    def _register_s3express_events(
+        self,
+        client,
+        endpoint_bridge=None,
+        endpoint_url=None,
+        client_config=None,
+        scoped_config=None,
+    ):
+        if client.meta.service_model.service_name != 's3':
+            return
+        AioS3ExpressIdentityResolver(
+            client, AioRefreshableCredentials
+        ).register()
+
     def _register_s3_events(
         self,
         client,
@@ -331,11 +347,17 @@ async def _make_api_call(self, operation_name, api_params):
             operation_model=operation_model,
             context=request_context,
         )
-        # fmt: off
-        endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
+        (
+            endpoint_url,
+            additional_headers,
+            properties,
+        ) = await self._resolve_endpoint_ruleset(
             operation_model, api_params, request_context
         )
-        # fmt: on
+        if properties:
+            # Pass arbitrary endpoint info with the Request
+            # for use during construction.
+            request_context['endpoint_properties'] = properties
         request_dict = await self._convert_to_request_dict(
             api_params=api_params,
             operation_model=operation_model,
@@ -482,6 +504,7 @@ async def _resolve_endpoint_ruleset(
         if self._ruleset_resolver is None:
             endpoint_url = self.meta.endpoint_url
             additional_headers = {}
+            endpoint_properties = {}
         else:
             endpoint_info = await self._ruleset_resolver.construct_endpoint(
                 operation_model=operation_model,
@@ -490,6 +513,7 @@ async def _resolve_endpoint_ruleset(
             )
             endpoint_url = endpoint_info.url
             additional_headers = endpoint_info.headers
+            endpoint_properties = endpoint_info.properties
             # If authSchemes is present, overwrite default auth type and
             # signing context derived from service model.
             auth_schemes = endpoint_info.properties.get('authSchemes')
@@ -506,7 +530,7 @@ async def _resolve_endpoint_ruleset(
                 else:
                     request_context['signing'] = signing_context
 
-        return endpoint_url, additional_headers
+        return endpoint_url, additional_headers, endpoint_properties
 
     def get_paginator(self, operation_name):
         """Create a paginator for an operation.

aiobotocore/signers.py

@@ -69,6 +69,12 @@ async def sign(
                 kwargs['region_name'] = signing_context['region']
             if signing_context.get('signing_name'):
                 kwargs['signing_name'] = signing_context['signing_name']
+            if signing_context.get('identity_cache') is not None:
+                self._resolve_identity_cache(
+                    kwargs,
+                    signing_context['identity_cache'],
+                    signing_context['cache_key'],
+                )
             try:
                 auth = await self.get_auth_instance(**kwargs)
             except UnknownSignatureVersionError as e:
@@ -141,11 +147,16 @@ async def get_auth_instance(
             auth = cls(frozen_token)
             return auth
 
+        credentials = self._credentials
+        if getattr(cls, "REQUIRES_IDENTITY_CACHE", None) is True:
+            cache = kwargs["identity_cache"]
+            key = kwargs["cache_key"]
+            credentials = await cache.get_credentials(key)
+            del kwargs["cache_key"]
+
         frozen_credentials = None
-        if self._credentials is not None:
-            frozen_credentials = (
-                await self._credentials.get_frozen_credentials()
-            )
+        if credentials is not None:
+            frozen_credentials = await credentials.get_frozen_credentials()
         kwargs['credentials'] = frozen_credentials
         if cls.REQUIRES_REGION:
             if self._region_name is None:
@@ -331,7 +342,11 @@ async def generate_presigned_url(
         context=context,
     )
     bucket_is_arn = ArnParser.is_arn(params.get('Bucket', ''))
-    endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
+    (
+        endpoint_url,
+        additional_headers,
+        properties,
+    ) = await self._resolve_endpoint_ruleset(
         operation_model,
         params,
         context,
@@ -396,7 +411,11 @@ async def generate_presigned_post(
         context=context,
     )
     bucket_is_arn = ArnParser.is_arn(params.get('Bucket', ''))
-    endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
+    (
+        endpoint_url,
+        additional_headers,
+        properties,
+    ) = await self._resolve_endpoint_ruleset(
         operation_model,
         params,
         context,

aiobotocore/utils.py

@@ -17,11 +17,14 @@
     ClientError,
     ContainerMetadataFetcher,
     HTTPClientError,
+    IdentityCache,
     IMDSFetcher,
     IMDSRegionProvider,
     InstanceMetadataFetcher,
     InstanceMetadataRegionFetcher,
     ReadTimeoutError,
+    S3ExpressIdentityCache,
+    S3ExpressIdentityResolver,
     S3RegionRedirector,
     S3RegionRedirectorv2,
     get_environ_proxies,
@@ -348,6 +351,51 @@ async def _get_region(self):
         return region
 
 
+class AioIdentityCache(IdentityCache):
+    async def get_credentials(self, **kwargs):
+        callback = self.build_refresh_callback(**kwargs)
+        metadata = await callback()
+        credential_entry = self._credential_cls.create_from_metadata(
+            metadata=metadata,
+            refresh_using=callback,
+            method=self.METHOD,
+            advisory_timeout=45,
+            mandatory_timeout=10,
+        )
+        return credential_entry
+
+
+class AioS3ExpressIdentityCache(AioIdentityCache, S3ExpressIdentityCache):
+    # @functools.lru_cache(maxsize=100)  # TODO: support caching
+    async def get_credentials(self, bucket):
+        return await super().get_credentials(bucket=bucket)
+
+    def build_refresh_callback(self, bucket):
+        async def refresher():
+            response = await self._client.create_session(Bucket=bucket)  # TODO
+            creds = response['Credentials']
+            expiration = self._serialize_if_needed(
+                creds['Expiration'], iso=True
+            )
+            return {
+                "access_key": creds['AccessKeyId'],
+                "secret_key": creds['SecretAccessKey'],
+                "token": creds['SessionToken'],
+                "expiry_time": expiration,
+            }
+
+        return refresher
+
+
+class AioS3ExpressIdentityResolver(S3ExpressIdentityResolver):
+    def __init__(self, client, credential_cls, cache=None):
+        super().__init__(client, credential_cls, cache)
+
+        if cache is None:
+            cache = AioS3ExpressIdentityCache(self._client, credential_cls)
+        self._cache = cache
+
+
 class AioS3RegionRedirectorv2(S3RegionRedirectorv2):
     async def redirect_from_error(
         self,

setup.py

@@ -7,15 +7,15 @@
 # NOTE: When updating botocore make sure to update awscli/boto3 versions below
 install_requires = [
     # pegged to also match items in `extras_require`
-    'botocore>=1.32.4,<1.33.2',
+    'botocore>=1.33.2,<1.33.3',
     'aiohttp>=3.7.4.post0,<4.0.0',
     'wrapt>=1.10.10, <2.0.0',
     'aioitertools>=0.5.1,<1.0.0',
 ]
 
 extras_require = {
-    'awscli': ['awscli>=1.30.4,<1.31.2'],
-    'boto3': ['boto3>=1.29.4,<1.33.2'],
+    'awscli': ['awscli>=1.31.2,<1.31.3'],
+    'boto3': ['boto3>=1.33.2,<1.33.3'],
 }
 
 

tests/test_patches.py

@@ -91,10 +91,13 @@
 )
 from botocore.utils import (
     ContainerMetadataFetcher,
+    IdentityCache,
     IMDSFetcher,
     IMDSRegionProvider,
     InstanceMetadataFetcher,
     InstanceMetadataRegionFetcher,
+    S3ExpressIdentityCache,
+    S3ExpressIdentityResolver,
     S3RegionRedirector,
     S3RegionRedirectorv2,
 )
@@ -140,7 +143,9 @@
         '0f80192233321ae4a55d95b68f5b8a68f3ad18e6',
     },
     # client.py
-    ClientCreator.create_client: {'ef5bef8f4b2887143165e72554fd85c36af7e822'},
+    ClientCreator.create_client: {
+        'eeb7c4730ac86aec37de53b2be0779490b05f50b',
+    },
     ClientCreator._create_client_class: {
         'fcecaf8d4f2c1ac3c5d0eb50c573233ef86d641d'
     },
@@ -150,6 +155,9 @@
     ClientCreator._get_client_args: {
         'd5e19b1e62f64a745de842963c2472825a66e854'
     },
+    ClientCreator._register_s3express_events: {
+        '716c1549989eef6bbd048bf4f134c1b4659e124a',
+    },
     ClientCreator._register_s3_events: {
         '5659a5312caeb3ea97d663d837d6d201f08824f2'
     },
@@ -166,15 +174,15 @@
         '000b2f2a122602e2e741ec2e89308dc2e2b67329'
     },
     BaseClient._make_api_call: {
-        '1ac2e166cc8e5020224a808d2ccdfda18e6bdbf2',
+        '2cb11088d36a89cf9f5c41508bce908acbde24c4',
     },
     BaseClient._make_request: {'cfd8bbf19ea132134717cdf9c460694ddacdbf58'},
     BaseClient._convert_to_request_dict: {
         '5e0a374926b6ee1a8715963ab551e126506e7fc9'
     },
     BaseClient._emit_api_params: {'abd67874dae8d5cd2788412e4699398cb915a119'},
     BaseClient._resolve_endpoint_ruleset: {
-        '3206a73ae79601c42f8a5ae1d7e0e903a2495acb',
+        'f09731451ff6ba0645dc82e5c7948dfbf781e025',
     },
     BaseClient.get_paginator: {'3531d5988aaaf0fbb3885044ccee1a693ec2608b'},
     BaseClient.get_waiter: {'44f0473d993d49ac7502984a7ccee3240b088404'},
@@ -190,7 +198,7 @@
         'eb247f2884aee311bdabba3435e749c3b8589100'
     },
     RefreshableCredentials.__init__: {
-        '1a6b83fc845f05feab117ce4fab73b13baed6e3b'
+        '25ee814f47e5ce617f57e893ae158e5fd6d358ea',
     },
     # We've overridden some properties
     RefreshableCredentials.__dict__['access_key'].fset: {
@@ -433,22 +441,28 @@
     },
     # signers.py
     RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'},
-    RequestSigner.sign: {'d90346d5e066e89cd902c5c936f59b644ecde275'},
+    RequestSigner.sign: {
+        '8b6ca96055e5546a6572ad790d5af74a23bc0b52',
+    },
     RequestSigner.get_auth: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'},
     RequestSigner.get_auth_instance: {
-        '4f9be5feafd6c08ffd7bb8de3c9bc36bc02cbfc8'
+        'dcd41ea686506dcf056d8252ccf73acd501efd2b',
     },
     RequestSigner._choose_signer: {'bd0e9784029b8aa182b5aec73910d94cb67c36b0'},
     RequestSigner.generate_presigned_url: {
         '417682868eacc10bf4c65f3dfbdba7d20d9250db'
     },
     add_generate_presigned_url: {'5820f74ac46b004eb79e00eea1adc467bcf4defe'},
-    generate_presigned_url: {'48f6745f8a37cfba04b3b2f6fb3910210b4a7201'},
+    generate_presigned_url: {
+        'd03631d6810e2453b8874bc76619927b694a4207',
+    },
     S3PostPresigner.generate_presigned_post: {
         '269efc9af054a2fd2728d5b0a27db82c48053d7f'
     },
     add_generate_presigned_post: {'e30360f2bd893fabf47f5cdb04b0de420ccd414d'},
-    generate_presigned_post: {'eedf40b48c63f6772ed05e3f335c8193d187f503'},
+    generate_presigned_post: {
+        'a3a834a08be2cf76c20ea137ba6b28e7a12f58ed',
+    },
     add_generate_db_auth_token: {'f61014e6fac4b5c7ee7ac2d2bec15fb16fa9fbe5'},
     generate_db_auth_token: {'1f37e1e5982d8528841ce6b79f229b3e23a18959'},
     # tokens.py
@@ -549,6 +563,18 @@
     IMDSRegionProvider._create_fetcher: {
         '18da52c786a20d91615258a8127b566688ecbb39',
     },
+    IdentityCache.get_credentials: {
+        'baf98c4caaddfa0594745eb490c327c65cff8920',
+    },
+    # S3ExpressIdentityCache.get_credentials: {
+    #     'tbd',  # TODO: hash of lru_cache decorated method?
+    # },
+    S3ExpressIdentityCache.build_refresh_callback: {
+        '0e833cc5e30b76fa13e8caf5c024fe2a21c10f22',
+    },
+    S3ExpressIdentityResolver.__init__: {
+        '148a10274d3268dd42df05d3bcfb98c668f01086',
+    },
     # waiter.py
     NormalizedOperationMethod.__call__: {
         '79723632d023739aa19c8a899bc2b814b8ab12ff'