From ba4264f42bea0b76865b6346774ef9c731f12519 Mon Sep 17 00:00:00 2001 From: Kenneth Daily Date: Fri, 26 May 2023 10:17:42 -0700 Subject: [PATCH] Implement configured endpoint URL resolution Adds a config provider to resolve the endpoint URL provided in an environment variable or shared configuration file. It resolves the endpoint in the following manner: 1. The value provided through the `endpoint_url` parameter provided to the client constructor. 2. The value provided by a service-specific environment variable. 3. The value provided by the global endpoint environment variable (`AWS_ENDPOINT_URL`). 4. The value provided by a service-specific parameter from a services definition section in the shared configuration file. 5. The value provided by the global parameter from a services definition section in the shared configuration file. 6. The value resolved when no configured endpoint URL is provided. The endpoint config provider uses the client name (name used to instantiate a client object) for construction and add to the config value store. This uses multiple lookups to handle service name changes for backwards compatibility. --- .../feature-configuration-3829.json | 5 + botocore/args.py | 32 ++- botocore/config.py | 8 + botocore/configloader.py | 29 +-- botocore/configprovider.py | 163 ++++++++++++++ botocore/handlers.py | 3 +- botocore/session.py | 18 ++ botocore/utils.py | 76 +++++++ tests/functional/test_endpoints.py | 70 +----- tests/unit/cfg/aws_services_config | 9 + tests/unit/test_config_provider.py | 204 ++++++++++++++++++ tests/unit/test_configloader.py | 18 ++ 12 files changed, 553 insertions(+), 82 deletions(-) create mode 100644 .changes/next-release/feature-configuration-3829.json create mode 100644 tests/unit/cfg/aws_services_config diff --git a/.changes/next-release/feature-configuration-3829.json b/.changes/next-release/feature-configuration-3829.json new file mode 100644 index 0000000000..ee690b9d29 --- /dev/null +++ b/.changes/next-release/feature-configuration-3829.json @@ -0,0 +1,5 @@ +{ + "type": "feature", + "category": "configuration", + "description": "Configure the endpoint URL in the shared configuration file or via an environment variable for a specific AWS service or all AWS services." +} diff --git a/botocore/args.py b/botocore/args.py index b844fce1d9..08dec16025 100644 --- a/botocore/args.py +++ b/botocore/args.py @@ -107,7 +107,7 @@ def get_client_args( s3_config = final_args['s3_config'] partition = endpoint_config['metadata'].get('partition', None) socket_options = final_args['socket_options'] - + configured_endpoint_url = final_args['configured_endpoint_url'] signing_region = endpoint_config['signing_region'] endpoint_region_name = endpoint_config['region_name'] @@ -152,7 +152,7 @@ def get_client_args( service_model, endpoint_region_name, region_name, - endpoint_url, + configured_endpoint_url, endpoint, is_secure, endpoint_bridge, @@ -202,10 +202,16 @@ def compute_client_args( user_agent += ' %s' % client_config.user_agent_extra s3_config = self.compute_s3_config(client_config) + + configured_endpoint_url = self._compute_configured_endpoint_url( + client_config=client_config, + endpoint_url=endpoint_url, + ) + endpoint_config = self._compute_endpoint_config( service_name=service_name, region_name=region_name, - endpoint_url=endpoint_url, + endpoint_url=configured_endpoint_url, is_secure=is_secure, endpoint_bridge=endpoint_bridge, s3_config=s3_config, @@ -250,6 +256,7 @@ def compute_client_args( 'service_name': service_name, 'parameter_validation': parameter_validation, 'user_agent': user_agent, + 'configured_endpoint_url': configured_endpoint_url, 'endpoint_config': endpoint_config, 'protocol': protocol, 'config_kwargs': config_kwargs, @@ -259,6 +266,25 @@ def compute_client_args( ), } + def _compute_configured_endpoint_url(self, client_config, endpoint_url): + if endpoint_url is not None or self._ignore_configured_endpoint_urls( + client_config + ): + return endpoint_url + + return self._config_store.get_config_variable('endpoint_url') + + def _ignore_configured_endpoint_urls(self, client_config): + if ( + client_config + and client_config.ignore_configured_endpoint_urls is not None + ): + return client_config.ignore_configured_endpoint_urls + + return self._config_store.get_config_variable( + 'ignore_configured_endpoint_urls' + ) + def compute_s3_config(self, client_config): s3_configuration = self._config_store.get_config_variable('s3') diff --git a/botocore/config.py b/botocore/config.py index 049ad47535..49573b7c00 100644 --- a/botocore/config.py +++ b/botocore/config.py @@ -188,6 +188,13 @@ class Config: Defaults to None. + :type ignore_configured_endpoint_urls: bool + :param ignore_configured_endpoint_urls: Setting to True disables use + of endpoint URLs provided via environment variables and + the shared configuration file. + + Defaults to None. + :type tcp_keepalive: bool :param tcp_keepalive: Enables the TCP Keep-Alive socket option used when creating new connections if set to True. @@ -214,6 +221,7 @@ class Config: ('endpoint_discovery_enabled', None), ('use_dualstack_endpoint', None), ('use_fips_endpoint', None), + ('ignore_configured_endpoint_urls', None), ('defaults_mode', None), ('tcp_keepalive', None), ] diff --git a/botocore/configloader.py b/botocore/configloader.py index 245d9d8eb7..0b6c82bcad 100644 --- a/botocore/configloader.py +++ b/botocore/configloader.py @@ -200,6 +200,17 @@ def _parse_nested(config_value): return parsed +def _parse_section(key, values): + result = {} + try: + parts = shlex.split(key) + except ValueError: + return result + if len(parts) == 2: + result[parts[1]] = values + return result + + def build_profile_map(parsed_ini_config): """Convert the parsed INI config into a profile map. @@ -254,22 +265,15 @@ def build_profile_map(parsed_ini_config): parsed_config = copy.deepcopy(parsed_ini_config) profiles = {} sso_sessions = {} + services = {} final_config = {} for key, values in parsed_config.items(): if key.startswith("profile"): - try: - parts = shlex.split(key) - except ValueError: - continue - if len(parts) == 2: - profiles[parts[1]] = values + profiles.update(_parse_section(key, values)) elif key.startswith("sso-session"): - try: - parts = shlex.split(key) - except ValueError: - continue - if len(parts) == 2: - sso_sessions[parts[1]] = values + sso_sessions.update(_parse_section(key, values)) + elif key.startswith("services"): + services.update(_parse_section(key, values)) elif key == 'default': # default section is special and is considered a profile # name but we don't require you use 'profile "default"' @@ -279,4 +283,5 @@ def build_profile_map(parsed_ini_config): final_config[key] = values final_config['profiles'] = profiles final_config['sso_sessions'] = sso_sessions + final_config['services'] = services return final_config diff --git a/botocore/configprovider.py b/botocore/configprovider.py index 0d0465fcea..85f79d974c 100644 --- a/botocore/configprovider.py +++ b/botocore/configprovider.py @@ -18,6 +18,7 @@ import os from botocore import utils +from botocore.exceptions import InvalidConfigError logger = logging.getLogger(__name__) @@ -108,6 +109,12 @@ None, utils.ensure_boolean, ), + 'ignore_configured_endpoint_urls': ( + 'ignore_configured_endpoint_urls', + 'AWS_IGNORE_CONFIGURED_ENDPOINT_URLS', + None, + utils.ensure_boolean, + ), 'parameter_validation': ('parameter_validation', None, True, None), # Client side monitoring configurations. # Note: These configurations are considered internal to botocore. @@ -851,3 +858,159 @@ def provide(self): def __repr__(self): return 'ConstantProvider(value=%s)' % self._value + + +class ConfiguredEndpointProvider(BaseProvider): + """Lookup an endpoint URL from environment variable or shared cong file. + + NOTE: This class is considered private and is subject to abrupt breaking + changes or removal without prior announcement. Please do not use it + directly. + """ + + _ENDPOINT_URL_LOOKUP_ORDER = [ + 'environment_service', + 'environment_global', + 'config_service', + 'config_global', + ] + + def __init__( + self, + full_config, + scoped_config, + client_name, + environ=None, + ): + """Initialize a ConfiguredEndpointProviderChain. + + :type full_config: dict + :param full_config: This is the dict representing the full + configuration file. + + :type scoped_config: dict + :param scoped_config: This is the dict representing the configuration + for the current profile for the session. + + :type client_name: str + :param client_name: The name used to instantiate a client using + botocore.session.Session.create_client. + + :type environ: dict + :param environ: A mapping to use for environment variables. If this + is not provided it will default to use os.environ. + """ + self._full_config = full_config + self._scoped_config = scoped_config + self._client_name = client_name + + if environ is None: + environ = os.environ + self._environ = environ + + def provide(self): + """Lookup the configured endpoint URL. + + The order is: + + 1. The value provided by a service-specific environment variable. + 2. The value provided by the global endpoint environment variable + (AWS_ENDPOINT_URL). + 3. The value provided by a service-specific parameter from a services + definition section in the shared configuration file. + 4. The value provided by the global parameter from a services + definition section in the shared configuration file. + """ + for location in self._ENDPOINT_URL_LOOKUP_ORDER: + logger.debug( + 'Looking for endpoint for %s via: %s', + self._client_name, + location, + ) + + endpoint_url = getattr(self, f'_get_endpoint_url_{location}')() + + if endpoint_url: + logger.info( + 'Found endpoint for %s via: %s.', + self._client_name, + location, + ) + return endpoint_url + + logger.debug('No configured endpoint found.') + return None + + def _get_snake_case_service_id(self, client_name): + # Use lookups to get the serice ID without loading the service data + # file. `client_name`` refers to the name used to instantiate a client + # through botocore.session.Session.create_client (parameter name there + # is `service_name`). This client name is generally the hyphenized + # service ID. We need to look up services that have been renamed + # (SERVICE_NAME_ALIASES) as well as services that do not use + # the service ID as their data directory name + # (CLIENT_NAME_TO_HYPHENIZED_SERVICE_ID_OVERRIDES). + client_name = utils.SERVICE_NAME_ALIASES.get(client_name, client_name) + hyphenized_service_id = ( + utils.CLIENT_NAME_TO_HYPHENIZED_SERVICE_ID_OVERRIDES.get( + client_name, client_name + ) + ) + return hyphenized_service_id.replace('-', '_') + + def _get_service_env_var_name(self): + transformed_service_id_env = self._get_snake_case_service_id( + self._client_name + ).upper() + return f'AWS_ENDPOINT_URL_{transformed_service_id_env}' + + def _get_services_config(self): + if 'services' not in self._scoped_config: + return {} + + section_name = self._scoped_config['services'] + services_section = self._full_config.get('services', {}).get( + section_name + ) + + if not services_section: + error_msg = ( + f'The profile is configured to use the services ' + f'section but the "{section_name}" services ' + f'configuration does not exist.' + ) + raise InvalidConfigError(error_msg=error_msg) + + return services_section + + def _get_endpoint_url_config_service(self): + snakecase_service_id = self._get_snake_case_service_id( + self._client_name + ).lower() + return ( + self._get_services_config() + .get(snakecase_service_id, {}) + .get('endpoint_url') + ) + + def _get_endpoint_url_config_global(self): + return self._scoped_config.get('endpoint_url') + + def _get_endpoint_url_environment_service(self): + return EnvironmentProvider( + name=self._get_service_env_var_name(), env=self._environ + ).provide() + + def _get_endpoint_url_environment_global(self): + return EnvironmentProvider( + name='AWS_ENDPOINT_URL', env=self._environ + ).provide() + + def __repr__(self): + return ( + f'ConfiguredEndpointProvider(' + f'full_config={self._full_config}, ' + f'scoped_config={self._scoped_config}, ' + f'client_name={self._client_name}, ' + f'environ={self._environ})' + ) diff --git a/botocore/handlers.py b/botocore/handlers.py index 3014123f9b..05258f21b1 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -73,6 +73,7 @@ from botocore.exceptions import MissingServiceIdError # noqa from botocore.utils import hyphenize_service_id # noqa from botocore.utils import is_global_accesspoint # noqa +from botocore.utils import SERVICE_NAME_ALIASES # noqa logger = logging.getLogger(__name__) @@ -99,8 +100,6 @@ S3_SIGNING_NAMES = ('s3', 's3-outposts', 's3-object-lambda') VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$') -SERVICE_NAME_ALIASES = {'runtime.sagemaker': 'sagemaker-runtime'} - def handle_service_name_alias(service_name, **kwargs): return SERVICE_NAME_ALIASES.get(service_name, service_name) diff --git a/botocore/session.py b/botocore/session.py index 316c6a68b5..5c18b1b6a9 100644 --- a/botocore/session.py +++ b/botocore/session.py @@ -42,6 +42,7 @@ from botocore.configprovider import ( BOTOCORE_DEFAUT_SESSION_VARIABLES, ConfigChainFactory, + ConfiguredEndpointProvider, ConfigValueStore, DefaultConfigResolver, SmartDefaultsConfigStoreFactory, @@ -962,6 +963,12 @@ def create_client( smart_defaults_factory.merge_smart_defaults( config_store, defaults_mode, region_name ) + + self._add_configured_endpoint_provider( + client_name=service_name, + config_store=config_store, + ) + client_creator = botocore.client.ClientCreator( loader, endpoint_resolver, @@ -1030,6 +1037,17 @@ def _resolve_defaults_mode(self, client_config, config_store): return lmode + def _add_configured_endpoint_provider(self, client_name, config_store): + chain = ConfiguredEndpointProvider( + full_config=self.full_config, + scoped_config=self.get_scoped_config(), + client_name=client_name, + ) + config_store.set_config_provider( + logical_name='endpoint_url', + provider=chain, + ) + def _missing_cred_vars(self, access_key, secret_key): if access_key is not None and secret_key is None: return 'aws_secret_access_key' diff --git a/botocore/utils.py b/botocore/utils.py index 484dd0f8f6..86f75f3d8a 100644 --- a/botocore/utils.py +++ b/botocore/utils.py @@ -3315,3 +3315,79 @@ def _serialize_if_needed(self, value, iso=False): return value.isoformat() return value.strftime('%Y-%m-%dT%H:%M:%S%Z') return value + + +# This parameter is not part of the public interface and is subject to abrupt +# breaking changes or removal without prior announcement. +# Mapping of services that have been renamed for backwards compatibility reasons. +# Keys are the previous name that should be allowed, values are the documented +# and preferred client name. +SERVICE_NAME_ALIASES = {'runtime.sagemaker': 'sagemaker-runtime'} + + +# This parameter is not part of the public interface and is subject to abrupt +# breaking changes or removal without prior announcement. +# Mapping to determine the service ID for services that do not use it as the +# model data directory name. The keys are the data directory name and the +# values are the transformed service IDs (lower case and hyphenated). +CLIENT_NAME_TO_HYPHENIZED_SERVICE_ID_OVERRIDES = { + # Actual service name we use -> Allowed computed service name. + 'alexaforbusiness': 'alexa-for-business', + 'apigateway': 'api-gateway', + 'application-autoscaling': 'application-auto-scaling', + 'appmesh': 'app-mesh', + 'autoscaling': 'auto-scaling', + 'autoscaling-plans': 'auto-scaling-plans', + 'ce': 'cost-explorer', + 'cloudhsmv2': 'cloudhsm-v2', + 'cloudsearchdomain': 'cloudsearch-domain', + 'cognito-idp': 'cognito-identity-provider', + 'config': 'config-service', + 'cur': 'cost-and-usage-report-service', + 'datapipeline': 'data-pipeline', + 'directconnect': 'direct-connect', + 'devicefarm': 'device-farm', + 'discovery': 'application-discovery-service', + 'dms': 'database-migration-service', + 'ds': 'directory-service', + 'dynamodbstreams': 'dynamodb-streams', + 'elasticbeanstalk': 'elastic-beanstalk', + 'elastictranscoder': 'elastic-transcoder', + 'elb': 'elastic-load-balancing', + 'elbv2': 'elastic-load-balancing-v2', + 'es': 'elasticsearch-service', + 'events': 'eventbridge', + 'globalaccelerator': 'global-accelerator', + 'iot-data': 'iot-data-plane', + 'iot-jobs-data': 'iot-jobs-data-plane', + 'iot1click-devices': 'iot-1click-devices-service', + 'iot1click-projects': 'iot-1click-projects', + 'iotevents-data': 'iot-events-data', + 'iotevents': 'iot-events', + 'iotwireless': 'iot-wireless', + 'kinesisanalytics': 'kinesis-analytics', + 'kinesisanalyticsv2': 'kinesis-analytics-v2', + 'kinesisvideo': 'kinesis-video', + 'lex-models': 'lex-model-building-service', + 'lexv2-models': 'lex-models-v2', + 'lex-runtime': 'lex-runtime-service', + 'lexv2-runtime': 'lex-runtime-v2', + 'logs': 'cloudwatch-logs', + 'machinelearning': 'machine-learning', + 'marketplacecommerceanalytics': 'marketplace-commerce-analytics', + 'marketplace-entitlement': 'marketplace-entitlement-service', + 'meteringmarketplace': 'marketplace-metering', + 'mgh': 'migration-hub', + 'sms-voice': 'pinpoint-sms-voice', + 'resourcegroupstaggingapi': 'resource-groups-tagging-api', + 'route53': 'route-53', + 'route53domains': 'route-53-domains', + 's3control': 's3-control', + 'sdb': 'simpledb', + 'secretsmanager': 'secrets-manager', + 'serverlessrepo': 'serverlessapplicationrepository', + 'servicecatalog': 'service-catalog', + 'servicecatalog-appregistry': 'service-catalog-appregistry', + 'stepfunctions': 'sfn', + 'storagegateway': 'storage-gateway', +} diff --git a/tests/functional/test_endpoints.py b/tests/functional/test_endpoints.py index 59ce8f0438..1372f8ff89 100644 --- a/tests/functional/test_endpoints.py +++ b/tests/functional/test_endpoints.py @@ -13,69 +13,7 @@ import pytest from botocore.session import get_session - -SERVICE_RENAMES = { - # Actual service name we use -> Allowed computed service name. - 'alexaforbusiness': 'alexa-for-business', - 'apigateway': 'api-gateway', - 'application-autoscaling': 'application-auto-scaling', - 'appmesh': 'app-mesh', - 'autoscaling': 'auto-scaling', - 'autoscaling-plans': 'auto-scaling-plans', - 'ce': 'cost-explorer', - 'cloudhsmv2': 'cloudhsm-v2', - 'cloudsearchdomain': 'cloudsearch-domain', - 'cognito-idp': 'cognito-identity-provider', - 'config': 'config-service', - 'cur': 'cost-and-usage-report-service', - 'datapipeline': 'data-pipeline', - 'directconnect': 'direct-connect', - 'devicefarm': 'device-farm', - 'discovery': 'application-discovery-service', - 'dms': 'database-migration-service', - 'ds': 'directory-service', - 'dynamodbstreams': 'dynamodb-streams', - 'elasticbeanstalk': 'elastic-beanstalk', - 'elastictranscoder': 'elastic-transcoder', - 'elb': 'elastic-load-balancing', - 'elbv2': 'elastic-load-balancing-v2', - 'es': 'elasticsearch-service', - 'events': 'eventbridge', - 'globalaccelerator': 'global-accelerator', - 'iot-data': 'iot-data-plane', - 'iot-jobs-data': 'iot-jobs-data-plane', - 'iot1click-devices': 'iot-1click-devices-service', - 'iot1click-projects': 'iot-1click-projects', - 'iotevents-data': 'iot-events-data', - 'iotevents': 'iot-events', - 'iotwireless': 'iot-wireless', - 'kinesisanalytics': 'kinesis-analytics', - 'kinesisanalyticsv2': 'kinesis-analytics-v2', - 'kinesisvideo': 'kinesis-video', - 'lex-models': 'lex-model-building-service', - 'lexv2-models': 'lex-models-v2', - 'lex-runtime': 'lex-runtime-service', - 'lexv2-runtime': 'lex-runtime-v2', - 'logs': 'cloudwatch-logs', - 'machinelearning': 'machine-learning', - 'marketplacecommerceanalytics': 'marketplace-commerce-analytics', - 'marketplace-entitlement': 'marketplace-entitlement-service', - 'meteringmarketplace': 'marketplace-metering', - 'mgh': 'migration-hub', - 'sms-voice': 'pinpoint-sms-voice', - 'resourcegroupstaggingapi': 'resource-groups-tagging-api', - 'route53': 'route-53', - 'route53domains': 'route-53-domains', - 's3control': 's3-control', - 'sdb': 'simpledb', - 'secretsmanager': 'secrets-manager', - 'serverlessrepo': 'serverlessapplicationrepository', - 'servicecatalog': 'service-catalog', - 'servicecatalog-appregistry': 'service-catalog-appregistry', - 'stepfunctions': 'sfn', - 'storagegateway': 'storage-gateway', -} - +from botocore.utils import CLIENT_NAME_TO_HYPHENIZED_SERVICE_ID_OVERRIDES ENDPOINT_PREFIX_OVERRIDE = { # entry in endpoints.json -> actual endpoint prefix. @@ -163,7 +101,7 @@ def test_endpoint_matches_service(endpoint_prefix): @pytest.mark.parametrize("service_name", AVAILABLE_SERVICES) -def test_service_name_matches_endpoint_prefix(service_name): +def test_client_name_matches_hyphenized_service_id(service_name): """Generates tests for each service to verify that the computed service named based on the service id matches the service name used to create a client (i.e the directory name in botocore/data) @@ -174,7 +112,9 @@ def test_service_name_matches_endpoint_prefix(service_name): # Handle known exceptions where we have renamed the service directory # for one reason or another. - actual_service_name = SERVICE_RENAMES.get(service_name, service_name) + actual_service_name = CLIENT_NAME_TO_HYPHENIZED_SERVICE_ID_OVERRIDES.get( + service_name, service_name + ) err_msg = ( f"Actual service name `{actual_service_name}` does not match " diff --git a/tests/unit/cfg/aws_services_config b/tests/unit/cfg/aws_services_config new file mode 100644 index 0000000000..cf9040c2a4 --- /dev/null +++ b/tests/unit/cfg/aws_services_config @@ -0,0 +1,9 @@ +[default] +endpoint_url = https://localhost:1234/ +services = my-services + +[services my-services] +s3 = + endpoint_url = https://localhost:5678/ +dynamodb = + endpoint_url = https://localhost:8888/ diff --git a/tests/unit/test_config_provider.py b/tests/unit/test_config_provider.py index 931448dff6..4a7d91b0d6 100644 --- a/tests/unit/test_config_provider.py +++ b/tests/unit/test_config_provider.py @@ -21,6 +21,7 @@ BaseProvider, ChainProvider, ConfigChainFactory, + ConfiguredEndpointProvider, ConfigValueStore, ConstantProvider, DefaultConfigResolver, @@ -942,3 +943,206 @@ def test_resolve_auto_mode_imds_region_provider_connect_timeout(self): ) mode = smart_defaults_factory.resolve_auto_mode('us-west-2') assert mode == 'standard' + + +def create_cases(): + service = 'batch' + + return [ + dict( + service=service, + environ_map={}, + full_config_map={}, + expected_value=None, + ), + dict( + service=service, + environ_map={'AWS_ENDPOINT_URL': 'global-from-env'}, + full_config_map={}, + expected_value='global-from-env', + ), + dict( + service=service, + environ_map={ + f'AWS_ENDPOINT_URL_{service.upper()}': 'service-from-env', + 'AWS_ENDPOINT_URL': 'global-from-env', + }, + full_config_map={}, + expected_value='service-from-env', + ), + dict( + service=service, + environ_map={ + 'AWS_ENDPOINT_URL': 'global-from-env', + 'AWS_ENDPOINT_URL_S3': 's3-endpoint-url', + }, + full_config_map={}, + expected_value='global-from-env', + ), + dict( + service=service, + environ_map={}, + full_config_map={ + 'profiles': {'default': {'endpoint_url': 'global-from-config'}} + }, + expected_value='global-from-config', + ), + dict( + service=service, + environ_map={}, + full_config_map={ + 'profiles': { + 'default': { + 'services': 'my-services', + } + }, + 'services': { + 'my-services': { + service: {'endpoint_url': "service-from-config"} + } + }, + }, + expected_value='service-from-config', + ), + dict( + service=service, + environ_map={}, + full_config_map={ + 'profiles': { + 'default': { + 'services': 'my-services', + 'endpoint_url': 'global-from-config', + } + }, + 'services': { + 'my-services': { + service: {'endpoint_url': "service-from-config"} + } + }, + }, + expected_value='service-from-config', + ), + dict( + service=service, + environ_map={ + 'AWS_ENDPOINT_URL': 'global-from-env', + }, + full_config_map={ + 'profiles': { + 'default': { + 'endpoint_url': 'global-from-config', + } + }, + }, + expected_value='global-from-env', + ), + dict( + service=service, + environ_map={ + f'AWS_ENDPOINT_URL_{service.upper()}': 'service-from-env', + }, + full_config_map={ + 'profiles': { + 'default': { + 'endpoint_url': 'global-from-config', + } + }, + }, + expected_value='service-from-env', + ), + dict( + service='s3', + environ_map={}, + full_config_map={ + 'profiles': { + 'default': { + 'services': 'my-services', + 'endpoint_url': 'global-from-config', + } + }, + 'services': { + 'my-services': { + service: {'endpoint_url': "service-from-config"} + } + }, + }, + expected_value='global-from-config', + ), + dict( + service='runtime.sagemaker', + environ_map={}, + full_config_map={ + 'profiles': { + 'default': { + 'services': 'my-services', + } + }, + 'services': { + 'my-services': { + 'sagemaker_runtime': { + 'endpoint_url': "service-from-config" + } + } + }, + }, + expected_value='service-from-config', + ), + dict( + service='apigateway', + environ_map={}, + full_config_map={ + 'profiles': { + 'default': { + 'services': 'my-services', + } + }, + 'services': { + 'my-services': { + 'api_gateway': {'endpoint_url': "service-from-config"} + } + }, + }, + expected_value='service-from-config', + ), + ] + + +class TestConfiguredEndpointProvider: + def assert_does_provide( + self, + service, + environ_map, + full_config_map, + expected_value, + ): + scoped_config_map = full_config_map.get('profiles', {}).get( + 'default', {} + ) + + chain = ConfiguredEndpointProvider( + scoped_config=scoped_config_map, + full_config=full_config_map, + client_name=service, + environ=environ_map, + ) + value = chain.provide() + assert value == expected_value + + @pytest.mark.parametrize('test_case', create_cases()) + def test_does_provide(self, test_case): + self.assert_does_provide(**test_case) + + def test_is_deepcopyable(self): + env = {'AWS_ENDPOINT_URL_BATCH': 'https://endpoint-override'} + provider = ConfiguredEndpointProvider( + full_config={}, scoped_config={}, client_name='batch', environ=env + ) + + provider_deepcopy = copy.deepcopy(provider) + assert provider is not provider_deepcopy + assert provider.provide() == 'https://endpoint-override' + assert provider_deepcopy.provide() == 'https://endpoint-override' + + env['AWS_ENDPOINT_URL_BATCH'] = 'https://another-new-endpoint-override' + assert provider.provide() == 'https://another-new-endpoint-override' + assert provider_deepcopy.provide() == 'https://endpoint-override' diff --git a/tests/unit/test_configloader.py b/tests/unit/test_configloader.py index 4a331c9e2d..a6e02f7d4f 100644 --- a/tests/unit/test_configloader.py +++ b/tests/unit/test_configloader.py @@ -188,6 +188,24 @@ def test_sso_session_config(self): self.assertEqual(sso_config['sso_region'], 'us-east-1') self.assertEqual(sso_config['sso_start_url'], 'https://example.com') + def test_services_config(self): + filename = path('aws_services_config') + loaded_config = load_config(filename) + self.assertIn('profiles', loaded_config) + self.assertIn('default', loaded_config['profiles']) + self.assertIn('services', loaded_config) + self.assertIn('my-services', loaded_config['services']) + services_config = loaded_config['services']['my-services'] + self.assertIn('s3', services_config) + self.assertIn('dynamodb', services_config) + self.assertEqual( + services_config['s3']['endpoint_url'], 'https://localhost:5678/' + ) + self.assertEqual( + services_config['dynamodb']['endpoint_url'], + 'https://localhost:8888/', + ) + if __name__ == "__main__": unittest.main()