Merge branch 'endpoint-prefix-revamp' into develop

* endpoint-prefix-revamp:
  Update test to use service model class instead of dict
  Change transcribe serviceid to 'Transcribe'
  Add remaining serviceIds
  Update test to require serviceId
  Update endpoint test to use service ids

botocore/data/appsync/2017-07-25/service-2.json

@@ -7,6 +7,7 @@
     "protocol":"rest-json",
     "serviceAbbreviation":"AWSAppSync",
     "serviceFullName":"AWS AppSync",
+    "serviceId":"AppSync",
     "signatureVersion":"v4",
     "signingName":"appsync",
     "uid":"appsync-2017-07-25"

botocore/data/clouddirectory/2017-01-11/service-2.json

@@ -5,6 +5,7 @@
     "endpointPrefix":"clouddirectory",
     "protocol":"rest-json",
     "serviceFullName":"Amazon CloudDirectory",
+    "serviceId":"CloudDirectory",
     "signatureVersion":"v4",
     "signingName":"clouddirectory",
     "uid":"clouddirectory-2017-01-11"

botocore/data/greengrass/2017-06-07/service-2.json

@@ -4,6 +4,7 @@
     "endpointPrefix" : "greengrass",
     "signingName" : "greengrass",
     "serviceFullName" : "AWS Greengrass",
+    "serviceId" : "Greengrass",
     "protocol" : "rest-json",
     "jsonVersion" : "1.1",
     "uid" : "greengrass-2017-06-07",

botocore/data/iot-jobs-data/2017-09-29/service-2.json

@@ -5,6 +5,7 @@
     "endpointPrefix":"data.jobs.iot",
     "protocol":"rest-json",
     "serviceFullName":"AWS IoT Jobs Data Plane",
+    "serviceId":"IoT Jobs Data Plane",
     "signatureVersion":"v4",
     "signingName":"iot-jobs-data",
     "uid":"iot-jobs-data-2017-09-29"

botocore/data/pinpoint/2016-12-01/service-2.json

@@ -4,6 +4,7 @@
     "endpointPrefix" : "pinpoint",
     "signingName" : "mobiletargeting",
     "serviceFullName" : "Amazon Pinpoint",
+    "serviceId" : "Pinpoint",
     "protocol" : "rest-json",
     "jsonVersion" : "1.1",
     "uid" : "pinpoint-2016-12-01",

botocore/data/transcribe/2017-10-26/service-2.json

@@ -6,6 +6,7 @@
     "jsonVersion":"1.1",
     "protocol":"json",
     "serviceFullName":"Amazon Transcribe Service",
+    "serviceId":"Transcribe",
     "signatureVersion":"v4",
     "signingName":"transcribe",
     "targetPrefix":"Transcribe",

tests/functional/test_endpoints.py

@@ -10,76 +10,134 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-import os
-import json
 from nose.tools import assert_equal
 from botocore.session import get_session
 
 
-# Several services have names that don't match for one reason or another.
 SERVICE_RENAMES = {
-    'application-autoscaling': 'autoscaling',
-    'appstream': 'appstream2',
+    # Actual service name we use -> Allowed computed service name.
+    'alexaforbusiness': 'alexa-for-business',
+    'apigateway': 'api-gateway',
+    'application-autoscaling': 'application-auto-scaling',
+    'autoscaling': 'auto-scaling',
+    'autoscaling-plans': 'auto-scaling-plans',
+    'ce': 'cost-explorer',
+    'cloudhsmv2': 'cloudhsm-v2',
+    'cloudsearchdomain': 'cloudsearch-domain',
+    'cognito-idp': 'cognito-identity-provider',
+    'config': 'config-service',
+    'cur': 'cost-and-usage-report-service',
+    'datapipeline': 'data-pipeline',
+    'directconnect': 'direct-connect',
+    'devicefarm': 'device-farm',
+    'discovery': 'application-discovery-service',
+    'dms': 'database-migration-service',
+    'ds': 'directory-service',
+    'dynamodbstreams': 'dynamodb-streams',
+    'elasticbeanstalk': 'elastic-beanstalk',
+    'elastictranscoder': 'elastic-transcoder',
+    'elb': 'elastic-load-balancing',
+    'elbv2': 'elastic-load-balancing-v2',
+    'es': 'elasticsearch-service',
+    'events': 'cloudwatch-events',
+    'iot-data': 'iot-data-plane',
+    'iot-jobs-data': 'iot-jobs-data-plane',
+    'iot1click-devices': 'iot-1click-devices-service',
+    'iot1click-projects': 'iot-1click-projects',
+    'kinesisanalytics': 'kinesis-analytics',
+    'kinesisvideo': 'kinesis-video',
+    'lex-models': 'lex-model-building-service',
+    'lex-runtime': 'lex-runtime-service',
+    'logs': 'cloudwatch-logs',
+    'machinelearning': 'machine-learning',
+    'marketplacecommerceanalytics': 'marketplace-commerce-analytics',
+    'marketplace-entitlement': 'marketplace-entitlement-service',
+    'meteringmarketplace': 'marketplace-metering',
+    'mgh': 'migration-hub',
+    'resourcegroupstaggingapi': 'resource-groups-tagging-api',
+    'route53': 'route-53',
+    'route53domains': 'route-53-domains',
+    'sdb': 'simpledb',
+    'secretsmanager': 'secrets-manager',
+    'serverlessrepo': 'serverlessapplicationrepository',
+    'servicecatalog': 'service-catalog',
+    'stepfunctions': 'sfn',
+    'storagegateway': 'storage-gateway',
+}
+
+
+ENDPOINT_PREFIX_OVERRIDE = {
+    # entry in endpoints.json -> actual endpoint prefix.
+    # The autoscaling-* services actually send requests to the
+    # autoscaling service, but they're exposed as separate clients
+    # in botocore.
     'autoscaling-plans': 'autoscaling',
-    'dynamodbstreams': 'streams.dynamodb',
-    'cloudwatch': 'monitoring',
-    'efs': 'elasticfilesystem',
-    'elb': 'elasticloadbalancing',
-    'elbv2': 'elasticloadbalancing',
-    'emr': 'elasticmapreduce',
-    'iot1click-devices': 'devices.iot1click',
-    'iot1click-projects': 'projects.iot1click',
-    'iot-data': 'data.iot',
-    'meteringmarketplace': 'metering.marketplace',
-    'opsworkscm': 'opsworks-cm',
-    'ses': 'email',
-    'stepfunctions': 'states',
-    'lex-runtime': 'runtime.lex',
-    'mturk': 'mturk-requester',
-    'resourcegroupstaggingapi': 'tagging',
-    'sagemaker-runtime': 'runtime.sagemaker',
-    'lex-models': 'models.lex',
-    'marketplace-entitlement': 'entitlement.marketplace',
-    'pricing': 'api.pricing',
-    'mediastore-data': 'data.mediastore',
-    'iot-jobs-data': 'data.jobs.iot',
-    'kinesis-video-media': 'kinesisvideo',
-    'kinesis-video-archived-media': 'kinesisvideo',
-    'alexaforbusiness': 'a4b',
+    'application-autoscaling': 'autoscaling',
+    # For neptune, we send requests to the RDS endpoint.
     'neptune': 'rds',
-    'mediatailor': 'api.mediatailor',
 }
-
-BLACKLIST = [
+NOT_SUPPORTED_IN_SDK = [
     'mobileanalytics',
 ]
 
 
 def test_endpoint_matches_service():
-    backwards_renames = dict((v, k) for k, v in SERVICE_RENAMES.items())
+    # This verifies client names match up with data from the endpoints.json
+    # file.  We want to verify that every entry in the endpoints.json
+    # file corresponds to a client we can construct via
+    # session.create_client(...).
+    # So first we get a list of all the service names in the endpoints
+    # file.
     session = get_session()
     loader = session.get_component('data_loader')
-    expected_services = set(loader.list_available_services('service-2'))
+    endpoints = loader.load_data('endpoints')
+    # A service can be in multiple partitions so we're using
+    # a set here to remove dupes.
+    services_in_endpoints_file = set([])
+    for partition in endpoints['partitions']:
+        for service in partition['services']:
+            # There are some services we don't support in the SDK
+            # so we don't need to add them to the list of services
+            # we need to check.
+            if service not in NOT_SUPPORTED_IN_SDK:
+                services_in_endpoints_file.add(service)
+
+    # Now we need to cross check them against services we know about.
+    # The entries in endpoints.json are keyed off of the endpoint
+    # prefix.  We don't directly have that data, so we have to load
+    # every service model and look up its endpoint prefix in its
+    # ``metadata`` section.
+    known_services = loader.list_available_services('service-2')
+    known_endpoint_prefixes = [
+        session.get_service_model(service_name).endpoint_prefix
+        for service_name in known_services
+    ]
 
-    pdir = os.path.dirname
-    endpoints_path = os.path.join(pdir(pdir(pdir(__file__))),
-                                  'botocore', 'data', 'endpoints.json')
-    with open(endpoints_path, 'r') as f:
-        data = json.loads(f.read())
-    for partition in data['partitions']:
-        for service in partition['services'].keys():
-            service = backwards_renames.get(service, service)
-            if service not in BLACKLIST:
-                yield _assert_endpoint_is_service, service, expected_services
+    # Now we go through every known endpoint prefix in the endpoints.json
+    # file and ensure it maps to an endpoint prefix we've seen
+    # in a service model.
+    for endpoint_prefix in services_in_endpoints_file:
+        # Check for an override where we know that an entry
+        # in the endpoints.json actually maps to a different endpoint
+        # prefix.
+        endpoint_prefix = ENDPOINT_PREFIX_OVERRIDE.get(endpoint_prefix,
+                                                       endpoint_prefix)
+        yield (_assert_known_endpoint_prefix,
+               endpoint_prefix,
+               known_endpoint_prefixes)
 
 
-def _assert_endpoint_is_service(service, expected_services):
-    assert service in expected_services
+def _assert_known_endpoint_prefix(endpoint_prefix, known_endpoint_prefixes):
+    assert endpoint_prefix in known_endpoint_prefixes
 
 
 def test_service_name_matches_endpoint_prefix():
-    # Generates tests for each service to verify that the endpoint prefix
-    # matches the service name unless there is an explicit exception.
+    # Generates tests for each service to verify that the computed service
+    # named based on the service id matches the service name used to
+    # create a client (i.e the directory name in botocore/data)
+    # unless there is an explicit exception.
+    # If there model has no serviceId then we fall back to the endpoint
+    # prefix.
     session = get_session()
     loader = session.get_component('data_loader')
 
@@ -88,19 +146,18 @@ def test_service_name_matches_endpoint_prefix():
     services = loader.list_available_services('service-2')
 
     for service in services:
-        yield _assert_service_name_matches_endpoint_prefix, loader, service
+        yield _assert_service_name_matches_endpoint_prefix, session, service
 
 
-def _assert_service_name_matches_endpoint_prefix(loader, service_name):
-    # Load the service model and grab its endpoint prefix
-    service_model = loader.load_service_model(service_name, 'service-2')
-    endpoint_prefix = service_model['metadata']['endpointPrefix']
+def _assert_service_name_matches_endpoint_prefix(session, service_name):
+    service_model = session.get_service_model(service_name)
+    computed_name = service_model.service_id.replace(' ', '-').lower()
 
     # Handle known exceptions where we have renamed the service directory
     # for one reason or another.
-    expected_endpoint_prefix = SERVICE_RENAMES.get(service_name, service_name)
+    actual_service_name = SERVICE_RENAMES.get(service_name, service_name)
     assert_equal(
-        endpoint_prefix, expected_endpoint_prefix,
-        "Service name `%s` does not match expected endpoint "
-        "prefix `%s`, actual: `%s`" % (
-            service_name, expected_endpoint_prefix, endpoint_prefix))
+        computed_name, actual_service_name,
+        "Actual service name `%s` does not match expected service name "
+        "we computed: `%s`" % (
+            actual_service_name, computed_name))