From b673285e73398d86bd90122b349dc484d5705271 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 21 Oct 2013 22:20:41 -0700 Subject: [PATCH 1/2] Specifying endpoint_url should not require metadata dict If a service has not metadata dict that contains regions/endpoint/protocol info, we should still be able to construct endpoints by passing endpoint_url. I've also updated the code to not require region_name unless it's a sigv4 service (which is the only reason we need to require a region name if we're given an explicit endpoint_url). With this commit, we can consume the unprocessed model as it exists in services/*.json. --- botocore/auth.py | 45 +++++++++++++++------------- botocore/endpoint.py | 17 +++++++---- botocore/service.py | 50 ++++++++++++++++++++++++++----- tests/__init__.py | 1 + tests/unit/auth/test_hmacv3.py | 2 +- tests/unit/test_service.py | 54 +++++++++++++++++++++++++++++----- 6 files changed, 128 insertions(+), 41 deletions(-) diff --git a/botocore/auth.py b/botocore/auth.py index 7b6c2aabf2..c3954deb24 100644 --- a/botocore/auth.py +++ b/botocore/auth.py @@ -40,16 +40,22 @@ 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') -class SigV2Auth(object): +class BaseSigner(object): + REQUIRES_REGION = False + + def add_auth(self, request): + raise NotImplementedError("add_auth") + + +class SigV2Auth(BaseSigner): """ Sign a request with Signature V2. """ - def __init__(self, credentials, service_name=None, region_name=None): + + def __init__(self, credentials): self.credentials = credentials if self.credentials is None: raise NoCredentialsError - self.service_name = service_name - self.region_name = region_name def calc_signature(self, request, params): logger.debug("Calculating signature using v2 auth.") @@ -97,13 +103,11 @@ def add_auth(self, request): return request -class SigV3Auth(object): - def __init__(self, credentials, service_name=None, region_name=None): +class SigV3Auth(BaseSigner): + def __init__(self, credentials): self.credentials = credentials if self.credentials is None: raise NoCredentialsError - self.service_name = service_name - self.region_name = region_name def add_auth(self, request): if 'Date' not in request.headers: @@ -120,19 +124,20 @@ def add_auth(self, request): request.headers['X-Amzn-Authorization'] = signature -class SigV4Auth(object): +class SigV4Auth(BaseSigner): """ Sign a request with Signature V4. """ + REQUIRES_REGION = True - def __init__(self, credentials, service_name=None, region_name=None): + def __init__(self, credentials, service_name, region_name): self.credentials = credentials if self.credentials is None: raise NoCredentialsError self.now = datetime.datetime.utcnow() self.timestamp = self.now.strftime('%Y%m%dT%H%M%SZ') - self.region_name = region_name - self.service_name = service_name + self._region_name = region_name + self._service_name = service_name def _sign(self, key, msg, hex=False): if hex: @@ -208,16 +213,16 @@ def canonical_request(self, request): def scope(self, args): scope = [self.credentials.access_key] scope.append(self.timestamp[0:8]) - scope.append(self.region_name) - scope.append(self.service_name) + scope.append(self._region_name) + scope.append(self._service_name) scope.append('aws4_request') return '/'.join(scope) def credential_scope(self, args): scope = [] scope.append(self.timestamp[0:8]) - scope.append(self.region_name) - scope.append(self.service_name) + scope.append(self._region_name) + scope.append(self._service_name) scope.append('aws4_request') return '/'.join(scope) @@ -237,8 +242,8 @@ def signature(self, string_to_sign): key = self.credentials.secret_key k_date = self._sign(('AWS4' + key).encode('utf-8'), self.timestamp[0:8]) - k_region = self._sign(k_date, self.region_name) - k_service = self._sign(k_region, self.service_name) + k_region = self._sign(k_date, self._region_name) + k_service = self._sign(k_region, self._service_name) k_signing = self._sign(k_service, 'aws4_request') return self._sign(k_signing, string_to_sign, hex=True) @@ -267,7 +272,7 @@ def add_auth(self, request): return request -class HmacV1Auth(object): +class HmacV1Auth(BaseSigner): # List of Query String Arguments of Interest QSAOfInterest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging', @@ -283,8 +288,6 @@ def __init__(self, credentials, service_name=None, region_name=None): self.credentials = credentials if self.credentials is None: raise NoCredentialsError - self.service_name = service_name - self.region_name = region_name self.auth_path = None # see comment in canonical_resource below def sign_string(self, string_to_sign): diff --git a/botocore/endpoint.py b/botocore/endpoint.py index d54b3a6b44..71f88d2637 100644 --- a/botocore/endpoint.py +++ b/botocore/endpoint.py @@ -253,19 +253,26 @@ def get_endpoint(service, region_name, endpoint_url): auth = _get_auth(service.signature_version, credentials=service.session.get_credentials(), service_name=service_name, - region_name=region_name) + region_name=region_name, + service_object=service) proxies = _get_proxies(endpoint_url) return cls(service, region_name, endpoint_url, auth=auth, proxies=proxies) -def _get_auth(signature_version, credentials, service_name, region_name): +def _get_auth(signature_version, credentials, service_name, region_name, + service_object): cls = AUTH_TYPE_MAPS.get(signature_version) if cls is None: raise UnknownSignatureVersionError(signature_version=signature_version) else: - return cls(credentials=credentials, - service_name=service_name, - region_name=region_name) + kwargs = {'credentials': credentials} + if cls.REQUIRES_REGION: + if region_name is None: + envvar_name = service_object.session.env_vars['region'][1] + raise botocore.exceptions.NoRegionError(env_var=envvar_name) + kwargs['region_name'] = region_name + kwargs['service_name'] = service_name + return cls(**kwargs) SERVICE_TO_ENDPOINT = { diff --git a/botocore/service.py b/botocore/service.py index 98591174fe..6f504bddc6 100644 --- a/botocore/service.py +++ b/botocore/service.py @@ -56,6 +56,10 @@ def __init__(self, session, provider, service_name, self.path = path self.port = port self.cli_name = service_name + if not hasattr(self, 'metadata'): + # metadata is an option thing that comes from .extra.json + # so if it's not there we just default to an empty dict. + self.metadata = {} def _create_operation_objects(self): logger.debug("Creating operation objects for: %s", self) @@ -78,7 +82,7 @@ def operations(self): @property def region_names(self): - return self.metadata['regions'].keys() + return self.metadata.get('regions', {}).keys() def _build_endpoint_url(self, host, is_secure): if is_secure: @@ -106,23 +110,55 @@ def get_endpoint(self, region_name=None, is_secure=True, :type endpoint_url: str :param endpoint_url: You can explicitly override the default - computed endpoint name with this parameter. + computed endpoint name with this parameter. If this arg is + provided then neither ``region_name`` nor ``is_secure`` + is used in building the final ``endpoint_url``. + ``region_name`` can still be useful for services that require + a region name independent of the endpoint_url (for example services + that use Signature Version 4, which require a region name for + use in the signature calculation). + """ - if not region_name: + if region_name is None: region_name = self.session.get_variable('region') - if region_name is None and not self.global_endpoint: - envvar_name = self.session.env_vars['region'][1] - raise NoRegionError(env_var=envvar_name) - if region_name not in self.metadata['regions']: + if endpoint_url is not None: + # Before getting into any of the region/endpoint + # logic, if an endpoint_url is explicitly + # provided, just use what's been explicitly passed in. + return get_endpoint(self, region_name, endpoint_url) + if region_name is None and not self.global_endpoint: + # The only time it's ok to *not* provide a region is + # if the service is a global_endpoint (e.g. IAM). + envvar_name = self.session.env_vars['region'][1] + raise NoRegionError(env_var=envvar_name) + if region_name not in self.region_names: if self.global_endpoint: + # If we haven't provided a region_name and this is a global + # endpoint, we can just use the global_endpoint (which is a + # string of the hostname of the global endpoint) to construct + # the full endpoint_url. endpoint_url = self._build_endpoint_url(self.global_endpoint, is_secure) region_name = 'us-east-1' else: + # Otherwise we've specified a region name that is + # not supported by the service so we raise + # an exception. raise ServiceNotInRegionError(service_name=self.endpoint_prefix, region_name=region_name) + # The 'regions' dict can call out the specific hostname + # to use for a particular region. If this is the case, + # this will have precedence. + # TODO: It looks like the region_name overrides shouldn't have the + # protocol prefix. Otherwise, it doesn't seem possible to override + # the hostname for a region *and* support both http/https. Should + # be an easy change but it will be backwards incompatible to anyone + # creating their own service descriptions with region overrides. endpoint_url = endpoint_url or self.metadata['regions'][region_name] if endpoint_url is None: + # If the entry in the 'regions' dict is None, + # then we fall back to the patter of + # endpoint_prefix.region.amazonaws.com. host = '%s.%s.amazonaws.com' % (self.endpoint_prefix, region_name) endpoint_url = self._build_endpoint_url(host, is_secure) return get_endpoint(self, region_name, endpoint_url) diff --git a/tests/__init__.py b/tests/__init__.py index 76b71db402..02538a54af 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -66,6 +66,7 @@ def setUp(self): super(BaseSessionTest, self).setUp() self.environ['AWS_ACCESS_KEY_ID'] = 'access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'secret_key' + self.session = botocore.session.get_session() class TestParamSerialization(BaseSessionTest): diff --git a/tests/unit/auth/test_hmacv3.py b/tests/unit/auth/test_hmacv3.py index ff22ee8019..c93803ebd6 100644 --- a/tests/unit/auth/test_hmacv3.py +++ b/tests/unit/auth/test_hmacv3.py @@ -40,7 +40,7 @@ def setUp(self): self.secret_key = 'secret_key' self.credentials = botocore.credentials.Credentials(self.access_key, self.secret_key) - self.auth = botocore.auth.SigV3Auth(self.credentials, None, None) + self.auth = botocore.auth.SigV3Auth(self.credentials) def test_signature_with_date_headers(self): request = AWSRequest() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 8831e8af98..9a9aa623ef 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -20,21 +20,61 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # -from tests import BaseEnvVar -import botocore.session +from tests import BaseSessionTest +import botocore.exceptions -class TestService(BaseEnvVar): + +class TestService(BaseSessionTest): def test_get_endpoint_with_no_region(self): - self.environ['AWS_ACCESS_KEY_ID'] = 'access_key' - self.environ['AWS_SECRET_ACCESS_KEY'] = 'secret_key' - session = botocore.session.get_session() # Test global endpoint service such as iam. - service = session.get_service('iam') + service = self.session.get_service('iam') endpoint = service.get_endpoint() self.assertEqual(endpoint.host, 'https://iam.amazonaws.com/') + def test_endpoint_arg_overrides_everything(self): + service = self.session.get_service('iam') + endpoint = service.get_endpoint( + region_name='us-east-1', + endpoint_url='https://wherever.i.want.com') + self.assertEqual(endpoint.host, 'https://wherever.i.want.com') + self.assertEqual(endpoint.region_name, 'us-east-1') + + def test_service_metadata_not_required_for_region(self): + service = self.session.get_service('iam') + # Empty out the service metadata. This contains info + # about supported protocools and region/endpoints. + # Even if this info is not present, if the user + # passes in an endpoint_url, we should be able to use + # this value. + service.metadata = {} + endpoint = service.get_endpoint( + region_name='us-east-1', + endpoint_url='https://wherever.i.want.com') + self.assertEqual(endpoint.host, 'https://wherever.i.want.com') + self.assertEqual(endpoint.region_name, 'us-east-1') + + def test_region_not_required_if_endpoint_url_given(self): + # Only services that require the region_name (sigv4) + # should require this param. If we're talking to + # a service that doesn't need this info, there's no + # reason to require this param in botocore. + service = self.session.get_service('ec2') + service.metadata = {} + endpoint = service.get_endpoint( + endpoint_url='https://wherever.i.want.com') + self.assertEqual(endpoint.host, 'https://wherever.i.want.com') + self.assertIsNone(endpoint.region_name) + + def test_region_required_for_sigv4(self): + # However, if the service uses siv4 auth, then an exception + # is raised if we call get_endpoint without a region name. + service = self.session.get_service('cloudformation') + service.metadata = {} + with self.assertRaises(botocore.exceptions.NoRegionError): + service.get_endpoint(endpoint_url='https://wherever.i.want.com') + if __name__ == "__main__": unittest.main() From 87f6d4421884050d091bd255cb756f6c334778e9 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Tue, 22 Oct 2013 10:51:32 -0700 Subject: [PATCH 2/2] Add test that verifies a region is required with no endpoint_url --- tests/unit/test_service.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 9a9aa623ef..0c28be87ae 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -75,6 +75,13 @@ def test_region_required_for_sigv4(self): with self.assertRaises(botocore.exceptions.NoRegionError): service.get_endpoint(endpoint_url='https://wherever.i.want.com') + def test_region_required_for_non_global_endpoint(self): + # If you don't provide an endpoint_url, than you need to + # provide a region_name. + service = self.session.get_service('ec2') + with self.assertRaises(botocore.exceptions.NoRegionError): + service.get_endpoint() + if __name__ == "__main__": unittest.main()