Specifying endpoint_url should not require metadata dict

botocore/auth.py

@@ -40,16 +40,22 @@
     'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
 
 
-class SigV2Auth(object):
+class BaseSigner(object):
+    REQUIRES_REGION = False
+
+    def add_auth(self, request):
+        raise NotImplementedError("add_auth")
+
+
+class SigV2Auth(BaseSigner):
     """
     Sign a request with Signature V2.
     """
-    def __init__(self, credentials, service_name=None, region_name=None):
+
+    def __init__(self, credentials):
         self.credentials = credentials
         if self.credentials is None:
             raise NoCredentialsError
-        self.service_name = service_name
-        self.region_name = region_name
 
     def calc_signature(self, request, params):
         logger.debug("Calculating signature using v2 auth.")
@@ -97,13 +103,11 @@ def add_auth(self, request):
         return request
 
 
-class SigV3Auth(object):
-    def __init__(self, credentials, service_name=None, region_name=None):
+class SigV3Auth(BaseSigner):
+    def __init__(self, credentials):
         self.credentials = credentials
         if self.credentials is None:
             raise NoCredentialsError
-        self.service_name = service_name
-        self.region_name = region_name
 
     def add_auth(self, request):
         if 'Date' not in request.headers:
@@ -120,19 +124,20 @@ def add_auth(self, request):
         request.headers['X-Amzn-Authorization'] = signature
 
 
-class SigV4Auth(object):
+class SigV4Auth(BaseSigner):
     """
     Sign a request with Signature V4.
     """
+    REQUIRES_REGION = True
 
-    def __init__(self, credentials, service_name=None, region_name=None):
+    def __init__(self, credentials, service_name, region_name):
         self.credentials = credentials
         if self.credentials is None:
             raise NoCredentialsError
         self.now = datetime.datetime.utcnow()
         self.timestamp = self.now.strftime('%Y%m%dT%H%M%SZ')
-        self.region_name = region_name
-        self.service_name = service_name
+        self._region_name = region_name
+        self._service_name = service_name
 
     def _sign(self, key, msg, hex=False):
         if hex:
@@ -208,16 +213,16 @@ def canonical_request(self, request):
     def scope(self, args):
         scope = [self.credentials.access_key]
         scope.append(self.timestamp[0:8])
-        scope.append(self.region_name)
-        scope.append(self.service_name)
+        scope.append(self._region_name)
+        scope.append(self._service_name)
         scope.append('aws4_request')
         return '/'.join(scope)
 
     def credential_scope(self, args):
         scope = []
         scope.append(self.timestamp[0:8])
-        scope.append(self.region_name)
-        scope.append(self.service_name)
+        scope.append(self._region_name)
+        scope.append(self._service_name)
         scope.append('aws4_request')
         return '/'.join(scope)
 
@@ -237,8 +242,8 @@ def signature(self, string_to_sign):
         key = self.credentials.secret_key
         k_date = self._sign(('AWS4' + key).encode('utf-8'),
                             self.timestamp[0:8])
-        k_region = self._sign(k_date, self.region_name)
-        k_service = self._sign(k_region, self.service_name)
+        k_region = self._sign(k_date, self._region_name)
+        k_service = self._sign(k_region, self._service_name)
         k_signing = self._sign(k_service, 'aws4_request')
         return self._sign(k_signing, string_to_sign, hex=True)
 
@@ -267,7 +272,7 @@ def add_auth(self, request):
         return request
 
 
-class HmacV1Auth(object):
+class HmacV1Auth(BaseSigner):
 
     # List of Query String Arguments of Interest
     QSAOfInterest = ['acl', 'cors', 'defaultObjectAcl', 'location', 'logging',
@@ -283,8 +288,6 @@ def __init__(self, credentials, service_name=None, region_name=None):
         self.credentials = credentials
         if self.credentials is None:
             raise NoCredentialsError
-        self.service_name = service_name
-        self.region_name = region_name
         self.auth_path = None  # see comment in canonical_resource below
 
     def sign_string(self, string_to_sign):

botocore/endpoint.py

@@ -253,19 +253,26 @@ def get_endpoint(service, region_name, endpoint_url):
         auth = _get_auth(service.signature_version,
                          credentials=service.session.get_credentials(),
                          service_name=service_name,
-                         region_name=region_name)
+                         region_name=region_name,
+                         service_object=service)
     proxies = _get_proxies(endpoint_url)
     return cls(service, region_name, endpoint_url, auth=auth, proxies=proxies)
 
 
-def _get_auth(signature_version, credentials, service_name, region_name):
+def _get_auth(signature_version, credentials, service_name, region_name,
+              service_object):
     cls = AUTH_TYPE_MAPS.get(signature_version)
     if cls is None:
         raise UnknownSignatureVersionError(signature_version=signature_version)
     else:
-        return cls(credentials=credentials,
-                   service_name=service_name,
-                   region_name=region_name)
+        kwargs = {'credentials': credentials}
+        if cls.REQUIRES_REGION:
+            if region_name is None:
+                envvar_name = service_object.session.env_vars['region'][1]
+                raise botocore.exceptions.NoRegionError(env_var=envvar_name)
+            kwargs['region_name'] = region_name
+            kwargs['service_name'] = service_name
+        return cls(**kwargs)
 
 
 SERVICE_TO_ENDPOINT = {

botocore/service.py

@@ -56,6 +56,10 @@ def __init__(self, session, provider, service_name,
         self.path = path
         self.port = port
         self.cli_name = service_name
+        if not hasattr(self, 'metadata'):
+            # metadata is an option thing that comes from .extra.json
+            # so if it's not there we just default to an empty dict.
+            self.metadata = {}
 
     def _create_operation_objects(self):
         logger.debug("Creating operation objects for: %s", self)
@@ -78,7 +82,7 @@ def operations(self):
 
     @property
     def region_names(self):
-        return self.metadata['regions'].keys()
+        return self.metadata.get('regions', {}).keys()
 
     def _build_endpoint_url(self, host, is_secure):
         if is_secure:
@@ -106,23 +110,55 @@ def get_endpoint(self, region_name=None, is_secure=True,
 
         :type endpoint_url: str
         :param endpoint_url: You can explicitly override the default
-            computed endpoint name with this parameter.
+            computed endpoint name with this parameter.  If this arg is
+            provided then neither ``region_name`` nor ``is_secure``
+            is used in building the final ``endpoint_url``.
+            ``region_name`` can still be useful for services that require
+            a region name independent of the endpoint_url (for example services
+            that use Signature Version 4, which require a region name for
+            use in the signature calculation).
+
         """
-        if not region_name:
+        if region_name is None:
             region_name = self.session.get_variable('region')
-            if region_name is None and not self.global_endpoint:
-                envvar_name = self.session.env_vars['region'][1]
-                raise NoRegionError(env_var=envvar_name)
-        if region_name not in self.metadata['regions']:
+        if endpoint_url is not None:
+            # Before getting into any of the region/endpoint
+            # logic, if an endpoint_url is explicitly
+            # provided, just use what's been explicitly passed in.
+            return get_endpoint(self, region_name, endpoint_url)
+        if region_name is None and not self.global_endpoint:
+            # The only time it's ok to *not* provide a region is
+            # if the service is a global_endpoint (e.g. IAM).
+            envvar_name = self.session.env_vars['region'][1]
+            raise NoRegionError(env_var=envvar_name)
+        if region_name not in self.region_names:
             if self.global_endpoint:
+                # If we haven't provided a region_name and this is a global
+                # endpoint, we can just use the global_endpoint (which is a
+                # string of the hostname of the global endpoint) to construct
+                # the full endpoint_url.
                 endpoint_url = self._build_endpoint_url(self.global_endpoint,
                                                         is_secure)
                 region_name = 'us-east-1'
             else:
+                # Otherwise we've specified a region name that is
+                # not supported by the service so we raise
+                # an exception.
                 raise ServiceNotInRegionError(service_name=self.endpoint_prefix,
                                               region_name=region_name)
+        # The 'regions' dict can call out the specific hostname
+        # to use for a particular region.  If this is the case,
+        # this will have precedence.
+        # TODO: It looks like the region_name overrides shouldn't have the
+        # protocol prefix.  Otherwise, it doesn't seem possible to override
+        # the hostname for a region *and* support both http/https.  Should
+        # be an easy change but it will be backwards incompatible to anyone
+        # creating their own service descriptions with region overrides.
         endpoint_url = endpoint_url or self.metadata['regions'][region_name]
         if endpoint_url is None:
+            # If the entry in the 'regions' dict is None,
+            # then we fall back to the patter of
+            # endpoint_prefix.region.amazonaws.com.
             host = '%s.%s.amazonaws.com' % (self.endpoint_prefix, region_name)
             endpoint_url = self._build_endpoint_url(host, is_secure)
         return get_endpoint(self, region_name, endpoint_url)

tests/__init__.py

@@ -66,6 +66,7 @@ def setUp(self):
         super(BaseSessionTest, self).setUp()
         self.environ['AWS_ACCESS_KEY_ID'] = 'access_key'
         self.environ['AWS_SECRET_ACCESS_KEY'] = 'secret_key'
+        self.session = botocore.session.get_session()
 
 
 class TestParamSerialization(BaseSessionTest):

tests/unit/auth/test_hmacv3.py

@@ -40,7 +40,7 @@ def setUp(self):
         self.secret_key = 'secret_key'
         self.credentials = botocore.credentials.Credentials(self.access_key,
                                                             self.secret_key)
-        self.auth = botocore.auth.SigV3Auth(self.credentials, None, None)
+        self.auth = botocore.auth.SigV3Auth(self.credentials)
 
     def test_signature_with_date_headers(self):
         request = AWSRequest()

tests/unit/test_service.py

@@ -20,21 +20,68 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
-from tests import BaseEnvVar
-import botocore.session
+from tests import BaseSessionTest
 
+import botocore.exceptions
 
-class TestService(BaseEnvVar):
+
+class TestService(BaseSessionTest):
 
     def test_get_endpoint_with_no_region(self):
-        self.environ['AWS_ACCESS_KEY_ID'] = 'access_key'
-        self.environ['AWS_SECRET_ACCESS_KEY'] = 'secret_key'
-        session = botocore.session.get_session()
         # Test global endpoint service such as iam.
-        service = session.get_service('iam')
+        service = self.session.get_service('iam')
         endpoint = service.get_endpoint()
         self.assertEqual(endpoint.host, 'https://iam.amazonaws.com/')
 
+    def test_endpoint_arg_overrides_everything(self):
+        service = self.session.get_service('iam')
+        endpoint = service.get_endpoint(
+            region_name='us-east-1',
+            endpoint_url='https://wherever.i.want.com')
+        self.assertEqual(endpoint.host, 'https://wherever.i.want.com')
+        self.assertEqual(endpoint.region_name, 'us-east-1')
+
+    def test_service_metadata_not_required_for_region(self):
+        service = self.session.get_service('iam')
+        # Empty out the service metadata.  This contains info
+        # about supported protocools and region/endpoints.
+        # Even if this info is not present, if the user
+        # passes in an endpoint_url, we should be able to use
+        # this value.
+        service.metadata = {}
+        endpoint = service.get_endpoint(
+            region_name='us-east-1',
+            endpoint_url='https://wherever.i.want.com')
+        self.assertEqual(endpoint.host, 'https://wherever.i.want.com')
+        self.assertEqual(endpoint.region_name, 'us-east-1')
+
+    def test_region_not_required_if_endpoint_url_given(self):
+        # Only services that require the region_name (sigv4)
+        # should require this param.  If we're talking to
+        # a service that doesn't need this info, there's no
+        # reason to require this param in botocore.
+        service = self.session.get_service('ec2')
+        service.metadata = {}
+        endpoint = service.get_endpoint(
+            endpoint_url='https://wherever.i.want.com')
+        self.assertEqual(endpoint.host, 'https://wherever.i.want.com')
+        self.assertIsNone(endpoint.region_name)
+
+    def test_region_required_for_sigv4(self):
+        # However, if the service uses siv4 auth, then an exception
+        # is raised if we call get_endpoint without a region name.
+        service = self.session.get_service('cloudformation')
+        service.metadata = {}
+        with self.assertRaises(botocore.exceptions.NoRegionError):
+            service.get_endpoint(endpoint_url='https://wherever.i.want.com')
+
+    def test_region_required_for_non_global_endpoint(self):
+        # If you don't provide an endpoint_url, than you need to
+        # provide a region_name.
+        service = self.session.get_service('ec2')
+        with self.assertRaises(botocore.exceptions.NoRegionError):
+            service.get_endpoint()
+
 
 if __name__ == "__main__":
     unittest.main()