diff --git a/tests/functional/configured_endpoint_urls/__init__.py b/tests/functional/configured_endpoint_urls/__init__.py new file mode 100644 index 0000000000..c5c740907d --- /dev/null +++ b/tests/functional/configured_endpoint_urls/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/tests/functional/configured_endpoint_urls/profile-tests.json b/tests/functional/configured_endpoint_urls/profile-tests.json new file mode 100644 index 0000000000..153df73848 --- /dev/null +++ b/tests/functional/configured_endpoint_urls/profile-tests.json @@ -0,0 +1,478 @@ +{ + "description": [ + "These are test descriptions that describe how specific data should be loaded from a profile file based on a ", + "profile name." + ], + + "testSuites": [ + { + "profiles": { + "default": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10" + }, + "service_localhost_global_only": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10", + "endpoint_url": "http://localhost:1234" + }, + "service_global_only": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10", + "endpoint_url": "https://global.endpoint.aws" + }, + "service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "services": "service_specific_s3", + "region": "fake-region-10" + }, + "global_and_service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "endpoint_url": "https://global.endpoint.aws", + "services": "service_specific_s3", + "region": "fake-region-10" + }, + "ignore_global_and_service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "endpoint_url": "https://global.endpoint.aws", + "services": "service_specific_s3", + "region": "fake-region-10", + "ignore_configured_endpoint_urls": "true" + }, + "service_specific_dynamodb_and_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "services": "service_specific_dynamodb_and_s3", + "region": "fake-region-10" + } + }, + + "services": { + "service_specific_s3": { + "s3": { + "endpoint_url": "https://s3.endpoint.aws" + } + }, + "service_specific_dynamodb_and_s3": { + "dynamodb": { + "endpoint_url": "https://dynamodb.endpoint.aws" + }, + "s3": { + "endpoint_url": "https://s3.endpoint.aws" + } + } + }, + + "client_configs": { + "default": {}, + "endpoint_url_provided":{ + "endpoint_url": "https://client-config.endpoint.aws" + } + }, + + "environments": { + "default": {}, + "global_only": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws" + }, + "service_specific_s3": { + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + }, + "global_and_service_specific_s3": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + + }, + "ignore_global_and_service_specific_s3": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws", + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS": "true" + }, + "service_specific_dynamodb_and_s3": { + "AWS_ENDPOINT_URL_DYNAMODB": "https://dynamodb-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + } + }, + + "endpointUrlTests": [ + { + "name": "Global endpoint url is read from services section and used for an S3 client.", + "profile": "service_global_only", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://global.endpoint.aws" + } + }, + { + "name": "Service specific endpoint url is read from services section and used for an S3 client.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://s3.endpoint.aws" + } + }, + { + "name": "S3 Service-specific endpoint URL from configuration file takes precedence over global endpoint URL from configuration file.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://s3.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precedence over the value resolved by the SDK.", + "profile": "default", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over global endpoint configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over service-specific endpoint configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over global endpoint configuration option and service-specific endpoint configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the value resolved by the SDK.", + "profile": "default", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the services-specific endpoint url configuration option and the global endpoint url configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable.", + "profile": "default", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable and the the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable, the service-specific endpoint URL configuration option, and the global endpoint URL configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over value provided by the SDK.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url from services section and used for an S3 client.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service specific endpoint url from services section and used for an S3 client.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over S3 Service-specific endpoint URL from configuration file and global endpoint URL from configuration file.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable and global endpoint configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable and service-specific endpoint configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable, global endpoint configuration option, and service-specific endpoint configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the services-specific endpoint url configuration option, and the global endpoint url configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the global endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, and the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, the service-specific endpoint URL configuration option, and the global endpoint URL configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "All configured endpoints ignored due to environment variable.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "ignore_global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3.fake-region-10.amazonaws.com" + } + }, + { + "name": "All configured endpoints ignored due to shared config variable.", + "profile": "ignore_global_and_service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3.fake-region-10.amazonaws.com" + } + }, + { + "name": "Environment variable and shared config file configured endpoints ignored due to ignore shared config variable and client configured endpoint is used.", + "profile": "ignore_global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Environment variable and shared config file configured endpoints ignored due to ignore environment variable and client configured endpoint is used.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "ignore_global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "DynamoDB service-specific endpoint url shared config variable is used when service-specific S3 shared config variable is also present.", + "profile": "service_specific_dynamodb_and_s3", + "client_config": "default", + "environment": "default", + "service": "dynamodb", + "output": { + "endpointUrl": "https://dynamodb.endpoint.aws" + } + }, + { + "name": "DynamoDB service-specific endpoint url environment variable is used when service-specific S3 environment variable is also present.", + "profile": "default", + "client_config": "default", + "environment": "service_specific_dynamodb_and_s3", + "service": "dynamodb", + "output": { + "endpointUrl": "https://dynamodb-from-envvar.endpoint.aws" + } + } + + ] + } + ] +} diff --git a/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py b/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py new file mode 100644 index 0000000000..c29a8f9357 --- /dev/null +++ b/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py @@ -0,0 +1,228 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import json +from pathlib import Path +from unittest import mock + +import pytest + +import botocore.configprovider +import botocore.utils +from botocore.compat import urlsplit +from tests import ClientHTTPStubber + +ENDPOINT_TESTDATA_FILE = Path(__file__).parent / "profile-tests.json" + + +def dict_to_ini_section(ini_dict, section_header): + section_str = f'[{section_header}]\n' + for key, value in ini_dict.items(): + if isinstance(value, dict): + section_str += f"{key} =\n" + for new_key, new_value in value.items(): + section_str += f" {new_key}={new_value}\n" + else: + section_str += f"{key}={value}\n" + return section_str + "\n" + + +def create_cases(): + with open(ENDPOINT_TESTDATA_FILE) as f: + test_suite = json.load(f)['testSuites'][0] + + for test_case_data in test_suite['endpointUrlTests']: + yield pytest.param( + { + 'service': test_case_data['service'], + 'profile': test_case_data['profile'], + 'expected_endpoint_url': test_case_data['output'][ + 'endpointUrl' + ], + 'client_args': test_suite['client_configs'].get( + test_case_data['client_config'], {} + ), + 'config_file_contents': get_config_file_contents( + test_case_data['profile'], test_suite + ), + 'environment': test_suite['environments'].get( + test_case_data['environment'], {} + ), + }, + id=test_case_data['name'], + ) + + +def get_config_file_contents(profile_name, test_suite): + profile = test_suite['profiles'][profile_name] + + profile_str = dict_to_ini_section( + profile, + section_header=f"profile {profile_name}", + ) + + services_section_name = profile.get('services', None) + + if services_section_name is None: + return profile_str + + services_section = test_suite['services'][services_section_name] + + service_section_str = dict_to_ini_section( + services_section, + section_header=f'services {services_section_name}', + ) + + return profile_str + service_section_str + + +@pytest.fixture +def client_creator(tmp_path): + tmp_config_file_path = tmp_path / 'config' + environ = {'AWS_CONFIG_FILE': str(tmp_config_file_path)} + + def _do_create_client( + service, + profile, + client_args=None, + config_file_contents=None, + environment=None, + ): + environ.update(environment) + with open(tmp_config_file_path, 'w') as f: + f.write(config_file_contents) + f.flush() + + return botocore.session.Session(profile=profile).create_client( + service, **client_args + ) + + with mock.patch('os.environ', environ): + yield _do_create_client + + +def _normalize_endpoint(url): + split_endpoint = urlsplit(url) + actual_endpoint = f"{split_endpoint.scheme}://{split_endpoint.netloc}" + return actual_endpoint + + +def assert_client_endpoint_url(client, expected_endpoint_url): + assert client.meta.endpoint_url == expected_endpoint_url + + +def assert_endpoint_url_used_for_operation( + client, expected_endpoint_url, operation, params +): + http_stubber = ClientHTTPStubber(client) + http_stubber.start() + http_stubber.add_response() + + # Call an operation on the client + getattr(client, operation)(**params) + + assert ( + _normalize_endpoint(http_stubber.requests[0].url) + == expected_endpoint_url + ) + + +def _known_service_names_and_models(): + my_session = botocore.session.get_session() + loader = my_session.get_component('data_loader') + available_services = loader.list_available_services('service-2') + + result = [] + for service_name in available_services: + model = my_session.get_service_model(service_name) + result.append((model.service_name, model)) + return sorted(result) + + +SERVICE_TO_OPERATION = {'s3': 'list_buckets', 'dynamodb': 'list_tables'} + + +@pytest.mark.parametrize("test_case", create_cases()) +def test_resolve_configured_endpoint_url(test_case, client_creator): + client = client_creator( + service=test_case['service'], + profile=test_case['profile'], + client_args=test_case['client_args'], + config_file_contents=test_case['config_file_contents'], + environment=test_case['environment'], + ) + + assert_endpoint_url_used_for_operation( + client=client, + expected_endpoint_url=test_case['expected_endpoint_url'], + operation=SERVICE_TO_OPERATION[test_case['service']], + params={}, + ) + + +@pytest.mark.parametrize( + 'service_name,service_model', _known_service_names_and_models() +) +def test_expected_service_env_var_name_is_respected( + service_name, service_model, client_creator +): + transformed_service_id = service_model.service_id.replace(' ', '_').upper() + + client = client_creator( + service=service_name, + profile='default', + client_args={}, + config_file_contents=( + '[profile default]\n' + 'aws_access_key_id=123\n' + 'aws_secret_access_key=456\n' + 'region=fake-region-10\n' + ), + environment={ + f'AWS_ENDPOINT_URL_{transformed_service_id}': 'https://endpoint-override' + }, + ) + + assert_client_endpoint_url( + client=client, expected_endpoint_url='https://endpoint-override' + ) + + +@pytest.mark.parametrize( + 'service_name,service_model', _known_service_names_and_models() +) +def test_expected_service_config_section_name_is_respected( + service_name, service_model, client_creator +): + transformed_service_id = service_model.service_id.replace(' ', '_').lower() + + client = client_creator( + service=service_name, + profile='default', + client_args={}, + config_file_contents=( + f'[profile default]\n' + f'services=my-services\n' + f'aws_access_key_id=123\n' + f'aws_secret_access_key=456\n' + f'region=fake-region-10\n\n' + f'[services my-services]\n' + f'{transformed_service_id} = \n' + f' endpoint_url = https://endpoint-override\n\n' + ), + environment={}, + ) + + assert_client_endpoint_url( + client=client, expected_endpoint_url='https://endpoint-override' + )