From d9a7dd9c61c5541d87703e6f33efc9c82ff2d8d9 Mon Sep 17 00:00:00 2001 From: Kenneth Daily Date: Mon, 5 Jun 2023 13:15:54 -0700 Subject: [PATCH] Add functional tests for configured endpoint url resolution The tests use a data file to load tests that enumerate the ways that the endpoint URL can be defined and asserts that the correct endpoint is used in a request and that it is ignored correctly if the appropriate shared config file property or environment variable is supplied. They also assert that the correct config section name and environment variable name are used to configure and override the endpoint URL for every AWS service. --- .../configured_endpoint_urls/__init__.py | 12 + .../profile-tests.json | 478 ++++++++++++++++++ .../test_configured_endpoint_url.py | 228 +++++++++ 3 files changed, 718 insertions(+) create mode 100644 tests/functional/configured_endpoint_urls/__init__.py create mode 100644 tests/functional/configured_endpoint_urls/profile-tests.json create mode 100644 tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py diff --git a/tests/functional/configured_endpoint_urls/__init__.py b/tests/functional/configured_endpoint_urls/__init__.py new file mode 100644 index 0000000000..c5c740907d --- /dev/null +++ b/tests/functional/configured_endpoint_urls/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/tests/functional/configured_endpoint_urls/profile-tests.json b/tests/functional/configured_endpoint_urls/profile-tests.json new file mode 100644 index 0000000000..153df73848 --- /dev/null +++ b/tests/functional/configured_endpoint_urls/profile-tests.json @@ -0,0 +1,478 @@ +{ + "description": [ + "These are test descriptions that describe how specific data should be loaded from a profile file based on a ", + "profile name." + ], + + "testSuites": [ + { + "profiles": { + "default": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10" + }, + "service_localhost_global_only": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10", + "endpoint_url": "http://localhost:1234" + }, + "service_global_only": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "region": "fake-region-10", + "endpoint_url": "https://global.endpoint.aws" + }, + "service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "services": "service_specific_s3", + "region": "fake-region-10" + }, + "global_and_service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "endpoint_url": "https://global.endpoint.aws", + "services": "service_specific_s3", + "region": "fake-region-10" + }, + "ignore_global_and_service_specific_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "endpoint_url": "https://global.endpoint.aws", + "services": "service_specific_s3", + "region": "fake-region-10", + "ignore_configured_endpoint_urls": "true" + }, + "service_specific_dynamodb_and_s3": { + "aws_access_key_id": "123", + "aws_secret_access_key": "456", + "services": "service_specific_dynamodb_and_s3", + "region": "fake-region-10" + } + }, + + "services": { + "service_specific_s3": { + "s3": { + "endpoint_url": "https://s3.endpoint.aws" + } + }, + "service_specific_dynamodb_and_s3": { + "dynamodb": { + "endpoint_url": "https://dynamodb.endpoint.aws" + }, + "s3": { + "endpoint_url": "https://s3.endpoint.aws" + } + } + }, + + "client_configs": { + "default": {}, + "endpoint_url_provided":{ + "endpoint_url": "https://client-config.endpoint.aws" + } + }, + + "environments": { + "default": {}, + "global_only": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws" + }, + "service_specific_s3": { + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + }, + "global_and_service_specific_s3": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + + }, + "ignore_global_and_service_specific_s3": { + "AWS_ENDPOINT_URL": "https://global-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws", + "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS": "true" + }, + "service_specific_dynamodb_and_s3": { + "AWS_ENDPOINT_URL_DYNAMODB": "https://dynamodb-from-envvar.endpoint.aws", + "AWS_ENDPOINT_URL_S3": "https://s3-from-envvar.endpoint.aws" + } + }, + + "endpointUrlTests": [ + { + "name": "Global endpoint url is read from services section and used for an S3 client.", + "profile": "service_global_only", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://global.endpoint.aws" + } + }, + { + "name": "Service specific endpoint url is read from services section and used for an S3 client.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://s3.endpoint.aws" + } + }, + { + "name": "S3 Service-specific endpoint URL from configuration file takes precedence over global endpoint URL from configuration file.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://s3.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precedence over the value resolved by the SDK.", + "profile": "default", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over global endpoint configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over service-specific endpoint configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Global endpoint url environment variable takes precendence over global endpoint configuration option and service-specific endpoint configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://global-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the value resolved by the SDK.", + "profile": "default", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the services-specific endpoint url configuration option and the global endpoint url configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable.", + "profile": "default", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable and the the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Service-specific endpoint url environment variable takes precedence over the global endpoint url environment variable, the service-specific endpoint URL configuration option, and the global endpoint URL configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3-from-envvar.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over value provided by the SDK.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url from services section and used for an S3 client.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service specific endpoint url from services section and used for an S3 client.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over S3 Service-specific endpoint URL from configuration file and global endpoint URL from configuration file.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "default", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable and global endpoint configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable and service-specific endpoint configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over global endpoint url environment variable, global endpoint configuration option, and service-specific endpoint configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_only", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the services-specific endpoint url configuration option, and the global endpoint url configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable and the global endpoint url environment variable.", + "profile": "default", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, and the global endpoint url configuration option.", + "profile": "service_global_only", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, and the service-specific endpoint url configuration option.", + "profile": "service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Client configuration takes precedence over service-specific endpoint url environment variable, the global endpoint url environment variable, the service-specific endpoint URL configuration option, and the global endpoint URL configuration option.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "All configured endpoints ignored due to environment variable.", + "profile": "global_and_service_specific_s3", + "client_config": "default", + "environment": "ignore_global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3.fake-region-10.amazonaws.com" + } + }, + { + "name": "All configured endpoints ignored due to shared config variable.", + "profile": "ignore_global_and_service_specific_s3", + "client_config": "default", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://s3.fake-region-10.amazonaws.com" + } + }, + { + "name": "Environment variable and shared config file configured endpoints ignored due to ignore shared config variable and client configured endpoint is used.", + "profile": "ignore_global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "Environment variable and shared config file configured endpoints ignored due to ignore environment variable and client configured endpoint is used.", + "profile": "global_and_service_specific_s3", + "client_config": "endpoint_url_provided", + "environment": "ignore_global_and_service_specific_s3", + "service": "s3", + "output": { + "endpointUrl": "https://client-config.endpoint.aws" + } + }, + { + "name": "DynamoDB service-specific endpoint url shared config variable is used when service-specific S3 shared config variable is also present.", + "profile": "service_specific_dynamodb_and_s3", + "client_config": "default", + "environment": "default", + "service": "dynamodb", + "output": { + "endpointUrl": "https://dynamodb.endpoint.aws" + } + }, + { + "name": "DynamoDB service-specific endpoint url environment variable is used when service-specific S3 environment variable is also present.", + "profile": "default", + "client_config": "default", + "environment": "service_specific_dynamodb_and_s3", + "service": "dynamodb", + "output": { + "endpointUrl": "https://dynamodb-from-envvar.endpoint.aws" + } + } + + ] + } + ] +} diff --git a/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py b/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py new file mode 100644 index 0000000000..c29a8f9357 --- /dev/null +++ b/tests/functional/configured_endpoint_urls/test_configured_endpoint_url.py @@ -0,0 +1,228 @@ +# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import json +from pathlib import Path +from unittest import mock + +import pytest + +import botocore.configprovider +import botocore.utils +from botocore.compat import urlsplit +from tests import ClientHTTPStubber + +ENDPOINT_TESTDATA_FILE = Path(__file__).parent / "profile-tests.json" + + +def dict_to_ini_section(ini_dict, section_header): + section_str = f'[{section_header}]\n' + for key, value in ini_dict.items(): + if isinstance(value, dict): + section_str += f"{key} =\n" + for new_key, new_value in value.items(): + section_str += f" {new_key}={new_value}\n" + else: + section_str += f"{key}={value}\n" + return section_str + "\n" + + +def create_cases(): + with open(ENDPOINT_TESTDATA_FILE) as f: + test_suite = json.load(f)['testSuites'][0] + + for test_case_data in test_suite['endpointUrlTests']: + yield pytest.param( + { + 'service': test_case_data['service'], + 'profile': test_case_data['profile'], + 'expected_endpoint_url': test_case_data['output'][ + 'endpointUrl' + ], + 'client_args': test_suite['client_configs'].get( + test_case_data['client_config'], {} + ), + 'config_file_contents': get_config_file_contents( + test_case_data['profile'], test_suite + ), + 'environment': test_suite['environments'].get( + test_case_data['environment'], {} + ), + }, + id=test_case_data['name'], + ) + + +def get_config_file_contents(profile_name, test_suite): + profile = test_suite['profiles'][profile_name] + + profile_str = dict_to_ini_section( + profile, + section_header=f"profile {profile_name}", + ) + + services_section_name = profile.get('services', None) + + if services_section_name is None: + return profile_str + + services_section = test_suite['services'][services_section_name] + + service_section_str = dict_to_ini_section( + services_section, + section_header=f'services {services_section_name}', + ) + + return profile_str + service_section_str + + +@pytest.fixture +def client_creator(tmp_path): + tmp_config_file_path = tmp_path / 'config' + environ = {'AWS_CONFIG_FILE': str(tmp_config_file_path)} + + def _do_create_client( + service, + profile, + client_args=None, + config_file_contents=None, + environment=None, + ): + environ.update(environment) + with open(tmp_config_file_path, 'w') as f: + f.write(config_file_contents) + f.flush() + + return botocore.session.Session(profile=profile).create_client( + service, **client_args + ) + + with mock.patch('os.environ', environ): + yield _do_create_client + + +def _normalize_endpoint(url): + split_endpoint = urlsplit(url) + actual_endpoint = f"{split_endpoint.scheme}://{split_endpoint.netloc}" + return actual_endpoint + + +def assert_client_endpoint_url(client, expected_endpoint_url): + assert client.meta.endpoint_url == expected_endpoint_url + + +def assert_endpoint_url_used_for_operation( + client, expected_endpoint_url, operation, params +): + http_stubber = ClientHTTPStubber(client) + http_stubber.start() + http_stubber.add_response() + + # Call an operation on the client + getattr(client, operation)(**params) + + assert ( + _normalize_endpoint(http_stubber.requests[0].url) + == expected_endpoint_url + ) + + +def _known_service_names_and_models(): + my_session = botocore.session.get_session() + loader = my_session.get_component('data_loader') + available_services = loader.list_available_services('service-2') + + result = [] + for service_name in available_services: + model = my_session.get_service_model(service_name) + result.append((model.service_name, model)) + return sorted(result) + + +SERVICE_TO_OPERATION = {'s3': 'list_buckets', 'dynamodb': 'list_tables'} + + +@pytest.mark.parametrize("test_case", create_cases()) +def test_resolve_configured_endpoint_url(test_case, client_creator): + client = client_creator( + service=test_case['service'], + profile=test_case['profile'], + client_args=test_case['client_args'], + config_file_contents=test_case['config_file_contents'], + environment=test_case['environment'], + ) + + assert_endpoint_url_used_for_operation( + client=client, + expected_endpoint_url=test_case['expected_endpoint_url'], + operation=SERVICE_TO_OPERATION[test_case['service']], + params={}, + ) + + +@pytest.mark.parametrize( + 'service_name,service_model', _known_service_names_and_models() +) +def test_expected_service_env_var_name_is_respected( + service_name, service_model, client_creator +): + transformed_service_id = service_model.service_id.replace(' ', '_').upper() + + client = client_creator( + service=service_name, + profile='default', + client_args={}, + config_file_contents=( + '[profile default]\n' + 'aws_access_key_id=123\n' + 'aws_secret_access_key=456\n' + 'region=fake-region-10\n' + ), + environment={ + f'AWS_ENDPOINT_URL_{transformed_service_id}': 'https://endpoint-override' + }, + ) + + assert_client_endpoint_url( + client=client, expected_endpoint_url='https://endpoint-override' + ) + + +@pytest.mark.parametrize( + 'service_name,service_model', _known_service_names_and_models() +) +def test_expected_service_config_section_name_is_respected( + service_name, service_model, client_creator +): + transformed_service_id = service_model.service_id.replace(' ', '_').lower() + + client = client_creator( + service=service_name, + profile='default', + client_args={}, + config_file_contents=( + f'[profile default]\n' + f'services=my-services\n' + f'aws_access_key_id=123\n' + f'aws_secret_access_key=456\n' + f'region=fake-region-10\n\n' + f'[services my-services]\n' + f'{transformed_service_id} = \n' + f' endpoint_url = https://endpoint-override\n\n' + ), + environment={}, + ) + + assert_client_endpoint_url( + client=client, expected_endpoint_url='https://endpoint-override' + )