Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support up-to-date botocore #1063

Merged
merged 2 commits into from
Dec 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,25 +1,25 @@
repos:
- repo: 'https://github.com/pre-commit/pre-commit-hooks'
rev: v4.4.0
rev: v4.5.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: 'https://github.com/PyCQA/isort'
rev: 5.12.0
hooks:
- id: isort
- repo: 'https://github.com/pycqa/flake8'
rev: 6.0.0
hooks:
- id: flake8
- repo: 'https://github.com/asottile/pyupgrade'
rev: v3.3.2
rev: v3.15.0
hooks:
- id: pyupgrade
args:
- '--py36-plus'
- repo: 'https://github.com/PyCQA/isort'
rev: 5.12.0
hooks:
- id: isort
- repo: 'https://github.com/psf/black'
rev: 23.3.0
rev: 23.11.0
hooks:
- id: black
- repo: 'https://github.com/pycqa/flake8'
rev: 6.1.0
hooks:
- id: flake8
4 changes: 4 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
Changes
-------

2.9.0 (2023-12-12)
^^^^^^^^^^^^^^^^^^
* bump botocore dependency specification

2.8.0 (2023-11-28)
^^^^^^^^^^^^^^^^^^
* add AioStubber that returns AioAWSResponse()
Expand Down
2 changes: 1 addition & 1 deletion aiobotocore/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '2.8.0'
__version__ = '2.9.0'
34 changes: 29 additions & 5 deletions aiobotocore/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@

from . import waiter
from .args import AioClientArgsCreator
from .credentials import AioRefreshableCredentials
from .discovery import AioEndpointDiscoveryHandler, AioEndpointDiscoveryManager
from .httpchecksum import apply_request_checksum
from .paginate import AioPaginator
from .retries import adaptive, standard
from .utils import AioS3RegionRedirectorv2
from .utils import AioS3ExpressIdentityResolver, AioS3RegionRedirectorv2

history_recorder = get_global_history_recorder()

Expand Down Expand Up @@ -96,6 +97,7 @@
client_config=client_config,
scoped_config=scoped_config,
)
self._register_s3express_events(client=service_client)
self._register_s3_control_events(client=service_client)
self._register_endpoint_discovery(
service_client, endpoint_url, client_config
Expand Down Expand Up @@ -223,6 +225,20 @@
block_endpoint_discovery_required_operations,
)

def _register_s3express_events(
self,
client,
endpoint_bridge=None,
endpoint_url=None,
client_config=None,
scoped_config=None,
):
if client.meta.service_model.service_name != 's3':
return
AioS3ExpressIdentityResolver(
client, AioRefreshableCredentials
).register()

def _register_s3_events(
self,
client,
Expand Down Expand Up @@ -331,11 +347,17 @@
operation_model=operation_model,
context=request_context,
)
# fmt: off
endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
(
endpoint_url,
additional_headers,
properties,
) = await self._resolve_endpoint_ruleset(
operation_model, api_params, request_context
)
# fmt: on
if properties:
# Pass arbitrary endpoint info with the Request
# for use during construction.
request_context['endpoint_properties'] = properties
request_dict = await self._convert_to_request_dict(
api_params=api_params,
operation_model=operation_model,
Expand Down Expand Up @@ -482,6 +504,7 @@
if self._ruleset_resolver is None:
endpoint_url = self.meta.endpoint_url
additional_headers = {}
endpoint_properties = {}

Check warning on line 507 in aiobotocore/client.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/client.py#L507

Added line #L507 was not covered by tests
else:
endpoint_info = await self._ruleset_resolver.construct_endpoint(
operation_model=operation_model,
Expand All @@ -490,6 +513,7 @@
)
endpoint_url = endpoint_info.url
additional_headers = endpoint_info.headers
endpoint_properties = endpoint_info.properties
# If authSchemes is present, overwrite default auth type and
# signing context derived from service model.
auth_schemes = endpoint_info.properties.get('authSchemes')
Expand All @@ -506,7 +530,7 @@
else:
request_context['signing'] = signing_context

return endpoint_url, additional_headers
return endpoint_url, additional_headers, endpoint_properties

def get_paginator(self, operation_name):
"""Create a paginator for an operation.
Expand Down
31 changes: 25 additions & 6 deletions aiobotocore/signers.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@
kwargs['region_name'] = signing_context['region']
if signing_context.get('signing_name'):
kwargs['signing_name'] = signing_context['signing_name']
if signing_context.get('identity_cache') is not None:
self._resolve_identity_cache(

Check warning on line 73 in aiobotocore/signers.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/signers.py#L73

Added line #L73 was not covered by tests
kwargs,
signing_context['identity_cache'],
signing_context['cache_key'],
)
try:
auth = await self.get_auth_instance(**kwargs)
except UnknownSignatureVersionError as e:
Expand Down Expand Up @@ -141,11 +147,16 @@
auth = cls(frozen_token)
return auth

credentials = self._credentials
if getattr(cls, "REQUIRES_IDENTITY_CACHE", None) is True:
cache = kwargs["identity_cache"]
key = kwargs["cache_key"]
credentials = await cache.get_credentials(key)
del kwargs["cache_key"]

Check warning on line 155 in aiobotocore/signers.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/signers.py#L152-L155

Added lines #L152 - L155 were not covered by tests

frozen_credentials = None
if self._credentials is not None:
frozen_credentials = (
await self._credentials.get_frozen_credentials()
)
if credentials is not None:
frozen_credentials = await credentials.get_frozen_credentials()
kwargs['credentials'] = frozen_credentials
if cls.REQUIRES_REGION:
if self._region_name is None:
Expand Down Expand Up @@ -331,7 +342,11 @@
context=context,
)
bucket_is_arn = ArnParser.is_arn(params.get('Bucket', ''))
endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
(
endpoint_url,
additional_headers,
properties,
) = await self._resolve_endpoint_ruleset(
operation_model,
params,
context,
Expand Down Expand Up @@ -396,7 +411,11 @@
context=context,
)
bucket_is_arn = ArnParser.is_arn(params.get('Bucket', ''))
endpoint_url, additional_headers = await self._resolve_endpoint_ruleset(
(
endpoint_url,
additional_headers,
properties,
) = await self._resolve_endpoint_ruleset(
operation_model,
params,
context,
Expand Down
70 changes: 70 additions & 0 deletions aiobotocore/utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import asyncio
import functools
import inspect
import json
import logging
Expand All @@ -17,11 +18,14 @@
ClientError,
ContainerMetadataFetcher,
HTTPClientError,
IdentityCache,
IMDSFetcher,
IMDSRegionProvider,
InstanceMetadataFetcher,
InstanceMetadataRegionFetcher,
ReadTimeoutError,
S3ExpressIdentityCache,
S3ExpressIdentityResolver,
S3RegionRedirector,
S3RegionRedirectorv2,
get_environ_proxies,
Expand Down Expand Up @@ -348,6 +352,72 @@
return region


class AioIdentityCache(IdentityCache):
async def get_credentials(self, **kwargs):
callback = self.build_refresh_callback(**kwargs)
metadata = await callback()
credential_entry = self._credential_cls.create_from_metadata(

Check warning on line 359 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L357-L359

Added lines #L357 - L359 were not covered by tests
metadata=metadata,
refresh_using=callback,
method=self.METHOD,
advisory_timeout=45,
mandatory_timeout=10,
)
return credential_entry

Check warning on line 366 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L366

Added line #L366 was not covered by tests


class AioS3ExpressIdentityCache(AioIdentityCache, S3ExpressIdentityCache):
@functools.cached_property
def _aio_credential_cache(self):
"""Substitutes upstream credential cache."""
return {}

Check warning on line 373 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L373

Added line #L373 was not covered by tests

async def get_credentials(self, bucket):
# upstream uses `@functools.lru_cache(maxsize=100)` to cache credentials.
# This is incompatible with async code.
# We need to implement custom caching logic.

if (credentials := self._aio_credential_cache.get(bucket)) is None:

Check warning on line 380 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L380

Added line #L380 was not covered by tests
# cache miss -> get credentials asynchronously
credentials = await super().get_credentials(bucket=bucket)

Check warning on line 382 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L382

Added line #L382 was not covered by tests

# upstream cache is bounded at 100 entries
if len(self._aio_credential_cache) >= 100:

Check warning on line 385 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L385

Added line #L385 was not covered by tests
# drop oldest entry from cache (deviates from lru_cache logic)
self._aio_credential_cache.pop(

Check warning on line 387 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L387

Added line #L387 was not covered by tests
next(iter(self._aio_credential_cache)),
)

self._aio_credential_cache[bucket] = credentials

Check warning on line 391 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L391

Added line #L391 was not covered by tests

return credentials

Check warning on line 393 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L393

Added line #L393 was not covered by tests

def build_refresh_callback(self, bucket):
async def refresher():
response = await self._client.create_session(Bucket=bucket)
jakob-keller marked this conversation as resolved.
Show resolved Hide resolved
creds = response['Credentials']
expiration = self._serialize_if_needed(

Check warning on line 399 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L396-L399

Added lines #L396 - L399 were not covered by tests
creds['Expiration'], iso=True
)
return {

Check warning on line 402 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L402

Added line #L402 was not covered by tests
"access_key": creds['AccessKeyId'],
"secret_key": creds['SecretAccessKey'],
"token": creds['SessionToken'],
"expiry_time": expiration,
}

return refresher

Check warning on line 409 in aiobotocore/utils.py

View check run for this annotation

Codecov / codecov/patch

aiobotocore/utils.py#L409

Added line #L409 was not covered by tests


class AioS3ExpressIdentityResolver(S3ExpressIdentityResolver):
def __init__(self, client, credential_cls, cache=None):
super().__init__(client, credential_cls, cache)

if cache is None:
cache = AioS3ExpressIdentityCache(self._client, credential_cls)
self._cache = cache


class AioS3RegionRedirectorv2(S3RegionRedirectorv2):
async def redirect_from_error(
self,
Expand Down
6 changes: 3 additions & 3 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@
# NOTE: When updating botocore make sure to update awscli/boto3 versions below
install_requires = [
# pegged to also match items in `extras_require`
'botocore>=1.32.4,<1.33.2',
jakob-keller marked this conversation as resolved.
Show resolved Hide resolved
'botocore>=1.33.2,<1.33.14',
thehesiod marked this conversation as resolved.
Show resolved Hide resolved
'aiohttp>=3.7.4.post0,<4.0.0',
'wrapt>=1.10.10, <2.0.0',
'aioitertools>=0.5.1,<1.0.0',
]

extras_require = {
'awscli': ['awscli>=1.30.4,<1.31.2'],
'boto3': ['boto3>=1.29.4,<1.33.2'],
'awscli': ['awscli>=1.31.2,<1.31.14'],
'boto3': ['boto3>=1.33.2,<1.33.14'],
}


Expand Down
Loading
Loading