From d2aaab844cb2f4fb527eb108dc23edc9f5f3285c Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Mon, 16 Feb 2015 20:33:03 -0800 Subject: [PATCH 1/2] Adding support for GCD dev server in datastore package. Enabling - set_default_dataset_id() support for DATASTORE_DATASET environment variable name - gcloud.datastore.Connection.API_BASE_URL override for DATASTORE_HOST environment variable name Fixes #650. --- gcloud/datastore/__init__.py | 4 +++ gcloud/datastore/connection.py | 9 +++++ gcloud/datastore/test___init__.py | 53 +++++++++++++++++++++++++++-- gcloud/datastore/test_connection.py | 41 ++++++++++++++++++++++ 4 files changed, 105 insertions(+), 2 deletions(-) diff --git a/gcloud/datastore/__init__.py b/gcloud/datastore/__init__.py index 6f77e05fba51..eb03a318cd82 100644 --- a/gcloud/datastore/__init__.py +++ b/gcloud/datastore/__init__.py @@ -67,6 +67,7 @@ """The scopes required for authenticating as a Cloud Datastore consumer.""" _DATASET_ENV_VAR_NAME = 'GCLOUD_DATASET_ID' +_GCD_DATASET_ENV_VAR_NAME = 'DATASTORE_DATASET' def set_default_dataset_id(dataset_id=None): @@ -86,6 +87,9 @@ def set_default_dataset_id(dataset_id=None): if dataset_id is None: dataset_id = os.getenv(_DATASET_ENV_VAR_NAME) + if dataset_id is None: + dataset_id = os.getenv(_GCD_DATASET_ENV_VAR_NAME) + if dataset_id is None: dataset_id = _implicit_environ.app_engine_id() diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 916599590d83..9c259556239b 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -14,12 +14,17 @@ """Connections to gcloud datastore API servers.""" +import os + from gcloud import connection from gcloud.exceptions import make_exception from gcloud.datastore import _datastore_v1_pb2 as datastore_pb from gcloud.datastore import helpers +_GCD_HOST_ENV_VAR_NAME = 'DATASTORE_HOST' + + class Connection(connection.Connection): """A connection to the Google Cloud Datastore via the Protobuf API. @@ -30,6 +35,10 @@ class Connection(connection.Connection): :param credentials: The OAuth2 Credentials to use for this connection. """ + API_BASE_URL = os.getenv(_GCD_HOST_ENV_VAR_NAME, + connection.Connection.API_BASE_URL) + """The base of the API call URL.""" + API_VERSION = 'v1beta2' """The version of the API, used in building the API call's URL.""" diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index e4907862a10f..140bbddc3a93 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -30,11 +30,11 @@ def _callFUT(self, dataset_id=None): from gcloud.datastore import set_default_dataset_id return set_default_dataset_id(dataset_id=dataset_id) - def _monkeyEnviron(self, implicit_dataset_id): + def _monkeyEnviron(self, implicit_dataset_id, environ=None): import os from gcloud._testing import _Monkey from gcloud.datastore import _DATASET_ENV_VAR_NAME - environ = {_DATASET_ENV_VAR_NAME: implicit_dataset_id} + environ = environ or {_DATASET_ENV_VAR_NAME: implicit_dataset_id} return _Monkey(os, getenv=environ.get) def _monkeyImplicit(self, connection=None, app_identity=None): @@ -112,6 +112,55 @@ def test_set_explicit_None_w_env_var_set(self): self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) + def test_set_from_gcd_env_var(self): + from gcloud.datastore import _GCD_DATASET_ENV_VAR_NAME + from gcloud.datastore import _implicit_environ + + GCD_DATASET_ID = 'GCD-IMPLICIT' + ENVIRON = {_GCD_DATASET_ENV_VAR_NAME: GCD_DATASET_ID} + + with self._monkeyEnviron(None, environ=ENVIRON): + with self._monkeyImplicit(): + self._callFUT() + + self.assertEqual(_implicit_environ.DATASET_ID, GCD_DATASET_ID) + + def test_set_gcd_and_production_env_vars(self): + from gcloud.datastore import _DATASET_ENV_VAR_NAME + from gcloud.datastore import _GCD_DATASET_ENV_VAR_NAME + from gcloud.datastore import _implicit_environ + + IMPLICIT_DATASET_ID = 'IMPLICIT' + GCD_DATASET_ID = 'GCD-IMPLICIT' + ENVIRON = { + _DATASET_ENV_VAR_NAME: IMPLICIT_DATASET_ID, + _GCD_DATASET_ENV_VAR_NAME: GCD_DATASET_ID, + } + + with self._monkeyEnviron(None, environ=ENVIRON): + with self._monkeyImplicit(): + self._callFUT() + + self.assertNotEqual(_implicit_environ.DATASET_ID, GCD_DATASET_ID) + self.assertEqual(_implicit_environ.DATASET_ID, IMPLICIT_DATASET_ID) + + def test_set_gcd_env_vars_and_appengine(self): + from gcloud.datastore import _GCD_DATASET_ENV_VAR_NAME + from gcloud.datastore import _implicit_environ + + GCD_DATASET_ID = 'GCD-IMPLICIT' + ENVIRON = {_GCD_DATASET_ENV_VAR_NAME: GCD_DATASET_ID} + + APP_ENGINE_ID = 'GAE' + APP_IDENTITY = _AppIdentity(APP_ENGINE_ID) + + with self._monkeyEnviron(None, environ=ENVIRON): + with self._monkeyImplicit(app_identity=APP_IDENTITY): + self._callFUT() + + self.assertNotEqual(_implicit_environ.DATASET_ID, APP_ENGINE_ID) + self.assertEqual(_implicit_environ.DATASET_ID, GCD_DATASET_ID) + def test_set_implicit_from_appengine(self): from gcloud.datastore import _implicit_environ diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index 487c7b7d6e24..30e202509699 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -46,6 +46,47 @@ def _verifyProtobufCall(self, called_with, URI, conn): self.assertEqual(called_with['headers']['User-Agent'], conn.USER_AGENT) + def test_default_url(self): + from gcloud.connection import Connection + + klass = self._getTargetClass() + self.assertEqual(klass.API_BASE_URL, Connection.API_BASE_URL) + + def test_custom_url(self): + import os + import sys + from gcloud._testing import _Monkey + from gcloud.connection import Connection as BaseConnection + from gcloud.datastore.connection import _GCD_HOST_ENV_VAR_NAME + + HOST = object() + fake_environ = {_GCD_HOST_ENV_VAR_NAME: HOST} + + def fake_getenv(key, default_val=None): + return fake_environ.get(key, default_val) + + # We want to temporarily all the gcloud.datastore modules except for + # the gcloud.datastore package itself. Then re-import. + gcloud_keys = [key for key in sys.modules + if 'gcloud.datastore.' in key] + modules_as_is = sys.modules.copy() + try: + for key in gcloud_keys: + sys.modules.pop(key) + + with _Monkey(os, getenv=fake_getenv): + from gcloud.datastore.connection import Connection + finally: + sys.modules.update(modules_as_is) + + self.assertNotEqual(Connection.API_BASE_URL, + BaseConnection.API_BASE_URL) + self.assertEqual(Connection.API_BASE_URL, HOST) + + # Make sure the restored import. + from gcloud.datastore.connection import Connection + self.assertEqual(Connection.API_BASE_URL, BaseConnection.API_BASE_URL) + def test_ctor_defaults(self): conn = self._makeOne() self.assertEqual(conn.credentials, None) From b9aaf101c122c2671b2ddd431900a55eb4730680 Mon Sep 17 00:00:00 2001 From: Danny Hermes Date: Tue, 17 Feb 2015 20:33:11 -0800 Subject: [PATCH 2/2] Making datastore api_base_url set in Connection constructor. This avoids setting anything at import time. --- gcloud/connection.py | 7 +- gcloud/datastore/connection.py | 24 +++--- gcloud/datastore/test_api.py | 2 +- gcloud/datastore/test_connection.py | 109 +++++++++++++++------------- gcloud/storage/connection.py | 7 +- 5 files changed, 82 insertions(+), 67 deletions(-) diff --git a/gcloud/connection.py b/gcloud/connection.py index b6ddea26ebfc..14e1006e5dc2 100644 --- a/gcloud/connection.py +++ b/gcloud/connection.py @@ -19,6 +19,10 @@ import httplib2 +API_BASE_URL = 'https://www.googleapis.com' +"""The base of the API call URL.""" + + class Connection(object): """A generic connection to Google Cloud Platform. @@ -54,9 +58,6 @@ class Connection(object): :param http: An optional HTTP object to make requests. """ - API_BASE_URL = 'https://www.googleapis.com' - """The base of the API call URL.""" - USER_AGENT = "gcloud-python/{0}".format(get_distribution('gcloud').version) """The user agent for gcloud-python requests.""" diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 9c259556239b..4e5d569c02f5 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -33,11 +33,11 @@ class Connection(connection.Connection): :type credentials: :class:`oauth2client.client.OAuth2Credentials` :param credentials: The OAuth2 Credentials to use for this connection. - """ - API_BASE_URL = os.getenv(_GCD_HOST_ENV_VAR_NAME, - connection.Connection.API_BASE_URL) - """The base of the API call URL.""" + :type api_base_url: string + :param api_base_url: The base of the API call URL. Defaults to the value + from :mod:`gcloud.connection`. + """ API_VERSION = 'v1beta2' """The version of the API, used in building the API call's URL.""" @@ -46,6 +46,13 @@ class Connection(connection.Connection): '/datasets/{dataset_id}/{method}') """A template for the URL of a particular API call.""" + def __init__(self, credentials=None, http=None, api_base_url=None): + super(Connection, self).__init__(credentials=credentials, http=http) + if api_base_url is None: + api_base_url = os.getenv(_GCD_HOST_ENV_VAR_NAME, + connection.API_BASE_URL) + self.api_base_url = api_base_url + def _request(self, dataset_id, method, data): """Make a request over the Http transport to the Cloud Datastore API. @@ -102,8 +109,7 @@ def _rpc(self, dataset_id, method, request_pb, response_pb_cls): data=request_pb.SerializeToString()) return response_pb_cls.FromString(response) - @classmethod - def build_api_url(cls, dataset_id, method, base_url=None, + def build_api_url(self, dataset_id, method, base_url=None, api_version=None): """Construct the URL for a particular API call. @@ -125,9 +131,9 @@ def build_api_url(cls, dataset_id, method, base_url=None, :param api_version: The version of the API to connect to. You shouldn't have to provide this. """ - return cls.API_URL_TEMPLATE.format( - api_base=(base_url or cls.API_BASE_URL), - api_version=(api_version or cls.API_VERSION), + return self.API_URL_TEMPLATE.format( + api_base=(base_url or self.api_base_url), + api_version=(api_version or self.API_VERSION), dataset_id=dataset_id, method=method) def lookup(self, dataset_id, key_pbs, diff --git a/gcloud/datastore/test_api.py b/gcloud/datastore/test_api.py index 587fb9448a0c..5edb03f6ae95 100644 --- a/gcloud/datastore/test_api.py +++ b/gcloud/datastore/test_api.py @@ -359,7 +359,7 @@ def test_w_deferred_from_backend_but_not_passed(self): # Make URI to check for requests. URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index 30e202509699..c748d258968e 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -47,45 +47,50 @@ def _verifyProtobufCall(self, called_with, URI, conn): conn.USER_AGENT) def test_default_url(self): - from gcloud.connection import Connection + from gcloud.connection import API_BASE_URL - klass = self._getTargetClass() - self.assertEqual(klass.API_BASE_URL, Connection.API_BASE_URL) + conn = self._makeOne() + self.assertEqual(conn.api_base_url, API_BASE_URL) - def test_custom_url(self): + def test_custom_url_from_env(self): import os - import sys from gcloud._testing import _Monkey - from gcloud.connection import Connection as BaseConnection + from gcloud.connection import API_BASE_URL from gcloud.datastore.connection import _GCD_HOST_ENV_VAR_NAME HOST = object() fake_environ = {_GCD_HOST_ENV_VAR_NAME: HOST} - def fake_getenv(key, default_val=None): - return fake_environ.get(key, default_val) + with _Monkey(os, getenv=fake_environ.get): + conn = self._makeOne() - # We want to temporarily all the gcloud.datastore modules except for - # the gcloud.datastore package itself. Then re-import. - gcloud_keys = [key for key in sys.modules - if 'gcloud.datastore.' in key] - modules_as_is = sys.modules.copy() - try: - for key in gcloud_keys: - sys.modules.pop(key) + self.assertNotEqual(conn.api_base_url, API_BASE_URL) + self.assertEqual(conn.api_base_url, HOST) - with _Monkey(os, getenv=fake_getenv): - from gcloud.datastore.connection import Connection - finally: - sys.modules.update(modules_as_is) + def test_custom_url_from_constructor(self): + from gcloud.connection import API_BASE_URL - self.assertNotEqual(Connection.API_BASE_URL, - BaseConnection.API_BASE_URL) - self.assertEqual(Connection.API_BASE_URL, HOST) + HOST = object() + conn = self._makeOne(api_base_url=HOST) + self.assertNotEqual(conn.api_base_url, API_BASE_URL) + self.assertEqual(conn.api_base_url, HOST) - # Make sure the restored import. - from gcloud.datastore.connection import Connection - self.assertEqual(Connection.API_BASE_URL, BaseConnection.API_BASE_URL) + def test_custom_url_constructor_and_env(self): + import os + from gcloud._testing import _Monkey + from gcloud.connection import API_BASE_URL + from gcloud.datastore.connection import _GCD_HOST_ENV_VAR_NAME + + HOST1 = object() + HOST2 = object() + fake_environ = {_GCD_HOST_ENV_VAR_NAME: HOST1} + + with _Monkey(os, getenv=fake_environ.get): + conn = self._makeOne(api_base_url=HOST2) + + self.assertNotEqual(conn.api_base_url, API_BASE_URL) + self.assertNotEqual(conn.api_base_url, HOST1) + self.assertEqual(conn.api_base_url, HOST2) def test_ctor_defaults(self): conn = self._makeOne() @@ -128,7 +133,7 @@ def test__request_w_200(self): DATA = b'DATA' conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -174,7 +179,7 @@ def FromString(cls, pb): METHOD = 'METHOD' conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -191,23 +196,23 @@ def FromString(cls, pb): def test_build_api_url_w_default_base_version(self): DATASET_ID = 'DATASET' METHOD = 'METHOD' - klass = self._getTargetClass() + conn = self._makeOne() URI = '/'.join([ - klass.API_BASE_URL, + conn.api_base_url, 'datastore', - klass.API_VERSION, + conn.API_VERSION, 'datasets', DATASET_ID, METHOD, ]) - self.assertEqual(klass.build_api_url(DATASET_ID, METHOD), URI) + self.assertEqual(conn.build_api_url(DATASET_ID, METHOD), URI) def test_build_api_url_w_explicit_base_version(self): BASE = 'http://example.com/' VER = '3.1415926' DATASET_ID = 'DATASET' METHOD = 'METHOD' - klass = self._getTargetClass() + conn = self._makeOne() URI = '/'.join([ BASE, 'datastore', @@ -216,7 +221,7 @@ def test_build_api_url_w_explicit_base_version(self): DATASET_ID, METHOD, ]) - self.assertEqual(klass.build_api_url(DATASET_ID, METHOD, BASE, VER), + self.assertEqual(conn.build_api_url(DATASET_ID, METHOD, BASE, VER), URI) def test_lookup_single_key_empty_response(self): @@ -227,7 +232,7 @@ def test_lookup_single_key_empty_response(self): rsp_pb = datastore_pb.LookupResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -256,7 +261,7 @@ def test_lookup_single_key_empty_response_w_eventual(self): rsp_pb = datastore_pb.LookupResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -298,7 +303,7 @@ def test_lookup_single_key_empty_response_w_transaction(self): rsp_pb = datastore_pb.LookupResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -332,7 +337,7 @@ def test_lookup_single_key_nonempty_response(self): rsp_pb.found.add(entity=entity) conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -363,7 +368,7 @@ def test_lookup_multiple_keys_empty_response(self): rsp_pb = datastore_pb.LookupResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -398,7 +403,7 @@ def test_lookup_multiple_keys_w_missing(self): er_2.entity.key.CopyFrom(key_pb2) conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -432,7 +437,7 @@ def test_lookup_multiple_keys_w_deferred(self): rsp_pb.deferred.add().CopyFrom(key_pb2) conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -473,7 +478,7 @@ def test_run_query_w_eventual_no_transaction(self): rsp_pb.batch.entity_result_type = datastore_pb.EntityResult.FULL conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -513,7 +518,7 @@ def test_run_query_wo_eventual_w_transaction(self): rsp_pb.batch.entity_result_type = datastore_pb.EntityResult.FULL conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -569,7 +574,7 @@ def test_run_query_wo_namespace_empty_result(self): rsp_pb.batch.entity_result_type = datastore_pb.EntityResult.FULL conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -603,7 +608,7 @@ def test_run_query_w_namespace_nonempty_result(self): rsp_pb.batch.more_results = 3 # NO_MORE_RESULTS conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -630,7 +635,7 @@ def test_begin_transaction_default_serialize(self): rsp_pb.transaction = TRANSACTION conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -655,7 +660,7 @@ def test_begin_transaction_explicit_serialize(self): rsp_pb.transaction = TRANSACTION conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -685,7 +690,7 @@ def test_commit_wo_transaction(self): prop.value.string_value = u'Foo' conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -719,7 +724,7 @@ def test_commit_w_transaction(self): prop.value.string_value = u'Foo' conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -747,7 +752,7 @@ def test_rollback_ok(self): rsp_pb = datastore_pb.RollbackResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -770,7 +775,7 @@ def test_allocate_ids_empty(self): rsp_pb = datastore_pb.AllocateIdsResponse() conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', @@ -803,7 +808,7 @@ def test_allocate_ids_non_empty(self): rsp_pb.key.add().CopyFrom(after_key_pbs[1]) conn = self._makeOne() URI = '/'.join([ - conn.API_BASE_URL, + conn.api_base_url, 'datastore', conn.API_VERSION, 'datasets', diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index 8a6b48b95508..fabb1edfb94d 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -18,13 +18,13 @@ from six.moves.urllib.parse import urlencode # pylint: disable=F0401 -from gcloud.connection import Connection as _Base +from gcloud import connection as base_connection from gcloud.exceptions import make_exception from gcloud.storage.bucket import Bucket from gcloud.storage.iterator import Iterator -class Connection(_Base): +class Connection(base_connection.Connection): """A connection to Google Cloud Storage via the JSON REST API. This defines :meth:`Connection.api_request` for making a generic JSON @@ -66,6 +66,9 @@ class Connection(_Base): """ + API_BASE_URL = base_connection.API_BASE_URL + """The base of the API call URL.""" + API_VERSION = 'v1' """The version of the API, used in building the API call's URL."""