diff --git a/gcloud/datastore/__init__.py b/gcloud/datastore/__init__.py index b8b8ba9b1344..7f68446ae2b4 100644 --- a/gcloud/datastore/__init__.py +++ b/gcloud/datastore/__init__.py @@ -48,7 +48,6 @@ when race conditions may occur. """ -from gcloud.datastore._implicit_environ import get_connection from gcloud.datastore._implicit_environ import get_default_connection from gcloud.datastore._implicit_environ import get_default_dataset_id from gcloud.datastore._implicit_environ import set_default_connection @@ -87,3 +86,23 @@ def set_defaults(dataset_id=None, connection=None): """ set_default_dataset_id(dataset_id=dataset_id) set_default_connection(connection=connection) + + +def get_connection(): + """Shortcut method to establish a connection to the Cloud Datastore. + + Use this if you are going to access several datasets + with the same set of credentials (unlikely): + + >>> from gcloud import datastore + + >>> connection = datastore.get_connection() + >>> key1 = datastore.Key('Kind', 1234, dataset_id='dataset1') + >>> key2 = datastore.Key('Kind', 1234, dataset_id='dataset2') + >>> entity1 = datastore.get(key1, connection=connection) + >>> entity2 = datastore.get(key2, connection=connection) + + :rtype: :class:`gcloud.datastore.connection.Connection` + :returns: A connection defined with the proper credentials. + """ + return Connection.from_environment() diff --git a/gcloud/datastore/_implicit_environ.py b/gcloud/datastore/_implicit_environ.py index fd5ba9e5b8f9..e263a6f6b635 100644 --- a/gcloud/datastore/_implicit_environ.py +++ b/gcloud/datastore/_implicit_environ.py @@ -24,6 +24,7 @@ from gcloud._helpers import _compute_engine_id from gcloud._helpers import _lazy_property_deco from gcloud.datastore.connection import Connection +from gcloud.datastore.connection import _CONNECTIONS _DATASET_ENV_VAR_NAME = 'GCLOUD_DATASET_ID' @@ -104,24 +105,26 @@ def get_default_dataset_id(): return _DEFAULTS.dataset_id -def get_connection(): - """Shortcut method to establish a connection to the Cloud Datastore. +def _require_connection(connection=None): + """Infer a connection from the environment, if not passed explicitly. - Use this if you are going to access several datasets - with the same set of credentials (unlikely): - - >>> from gcloud import datastore - - >>> connection = datastore.get_connection() - >>> key1 = datastore.Key('Kind', 1234, dataset_id='dataset1') - >>> key2 = datastore.Key('Kind', 1234, dataset_id='dataset2') - >>> entity1 = datastore.get(key1, connection=connection) - >>> entity2 = datastore.get(key2, connection=connection) + :type connection: :class:`gcloud.datastore.connection.Connection` + :param connection: Optional. :rtype: :class:`gcloud.datastore.connection.Connection` - :returns: A connection defined with the proper credentials. + :returns: A connection based on the current environment. + :raises: :class:`EnvironmentError` if ``connection`` is ``None``, and + cannot be inferred from the environment. """ - return Connection.from_environment() + if connection is None: + top = _CONNECTIONS.top + if top is not None: + connection = top.connection + else: + connection = get_default_connection() + if connection is None: + raise EnvironmentError('Connection could not be inferred.') + return connection def set_default_connection(connection=None): @@ -130,8 +133,7 @@ def set_default_connection(connection=None): :type connection: :class:`gcloud.datastore.connection.Connection` :param connection: A connection provided to be the default. """ - connection = connection or get_connection() - _DEFAULTS.connection = connection + _DEFAULTS.connection = connection or Connection.from_environment() def get_default_connection(): @@ -163,7 +165,7 @@ def dataset_id(): @staticmethod def connection(): """Return the implicit default connection..""" - return get_connection() + return Connection.from_environment() def __init__(self, connection=None, dataset_id=None, implicit=False): if connection is not None or not implicit: diff --git a/gcloud/datastore/_testing.py b/gcloud/datastore/_testing.py index 97e43222c32e..5b4390a7d9fe 100644 --- a/gcloud/datastore/_testing.py +++ b/gcloud/datastore/_testing.py @@ -31,3 +31,36 @@ def _setup_defaults(test_case, *args, **kwargs): def _tear_down_defaults(test_case): _implicit_environ._DEFAULTS = test_case._replaced_defaults + + +class _NoCommitBatch(object): + + def __init__(self, dataset_id, connection): + from gcloud.datastore.batch import Batch + self._batch = Batch(dataset_id, connection) + + def __enter__(self): + from gcloud.datastore.connection import _CONNECTIONS + _CONNECTIONS.push(self._batch) + return self._batch + + def __exit__(self, *args): + from gcloud.datastore.connection import _CONNECTIONS + _CONNECTIONS.pop() + + +class _NoCommitTransaction(object): + + def __init__(self, dataset_id, connection, transaction_id='TRANSACTION'): + from gcloud.datastore.transaction import Transaction + xact = self._transaction = Transaction(dataset_id, connection) + xact._id = transaction_id + + def __enter__(self): + from gcloud.datastore.connection import _CONNECTIONS + _CONNECTIONS.push(self._transaction) + return self._transaction + + def __exit__(self, *args): + from gcloud.datastore.connection import _CONNECTIONS + _CONNECTIONS.pop() diff --git a/gcloud/datastore/api.py b/gcloud/datastore/api.py index 246641af5a18..2ce4850afbb4 100644 --- a/gcloud/datastore/api.py +++ b/gcloud/datastore/api.py @@ -20,6 +20,7 @@ from gcloud.datastore import _implicit_environ from gcloud.datastore.batch import Batch +from gcloud.datastore.connection import _CONNECTIONS from gcloud.datastore.entity import Entity from gcloud.datastore.transaction import Transaction from gcloud.datastore import helpers @@ -53,7 +54,7 @@ def _require_dataset_id(dataset_id=None, first_key=None): """ if dataset_id is not None: return dataset_id - top = Batch.current() + top = _CONNECTIONS.top if top is not None: return top.dataset_id if first_key is not None: @@ -65,28 +66,6 @@ def _require_dataset_id(dataset_id=None, first_key=None): return dataset_id -def _require_connection(connection=None): - """Infer a connection from the environment, if not passed explicitly. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional. - - :rtype: :class:`gcloud.datastore.connection.Connection` - :returns: A connection based on the current environment. - :raises: :class:`EnvironmentError` if ``connection`` is ``None``, and - cannot be inferred from the environment. - """ - if connection is None: - top = Batch.current() - if top is not None: - connection = top.connection - else: - connection = _implicit_environ.get_default_connection() - if connection is None: - raise EnvironmentError('Connection could not be inferred.') - return connection - - def _extended_lookup(connection, dataset_id, key_pbs, missing=None, deferred=None, eventual=False, transaction_id=None): @@ -200,7 +179,7 @@ def get(keys, missing=None, deferred=None, connection=None, dataset_id=None): if not keys: return [] - connection = _require_connection(connection) + connection = _implicit_environ._require_connection(connection) dataset_id = _require_dataset_id(dataset_id, keys[0]) if list(set([key.dataset_id for key in keys])) != [dataset_id]: @@ -259,10 +238,10 @@ def put(entities, connection=None, dataset_id=None): if not entities: return - connection = _require_connection(connection) + connection = _implicit_environ._require_connection(connection) dataset_id = _require_dataset_id(dataset_id, entities[0].key) - current = Batch.current() + current = _CONNECTIONS.top in_batch = current is not None if not in_batch: current = Batch(dataset_id=dataset_id, connection=connection) @@ -294,11 +273,11 @@ def delete(keys, connection=None, dataset_id=None): if not keys: return - connection = _require_connection(connection) + connection = _implicit_environ._require_connection(connection) dataset_id = _require_dataset_id(dataset_id, keys[0]) # We allow partial keys to attempt a delete, the backend will fail. - current = Batch.current() + current = _CONNECTIONS.top in_batch = current is not None if not in_batch: current = Batch(dataset_id=dataset_id, connection=connection) @@ -324,7 +303,7 @@ def allocate_ids(incomplete_key, num_ids, connection=None): :returns: The (complete) keys allocated with ``incomplete_key`` as root. :raises: :class:`ValueError` if ``incomplete_key`` is not a partial key. """ - connection = _require_connection(connection) + connection = _implicit_environ._require_connection(connection) if not incomplete_key.is_partial: raise ValueError(('Key is not partial.', incomplete_key)) diff --git a/gcloud/datastore/batch.py b/gcloud/datastore/batch.py index 0b1185322542..1e437078720e 100644 --- a/gcloud/datastore/batch.py +++ b/gcloud/datastore/batch.py @@ -21,16 +21,13 @@ https://cloud.google.com/datastore/docs/concepts/entities#Datastore_Batch_operations """ -from gcloud._helpers import _LocalStack from gcloud.datastore import _implicit_environ from gcloud.datastore import helpers +from gcloud.datastore.connection import _CONNECTIONS from gcloud.datastore.key import _dataset_ids_equal from gcloud.datastore import _datastore_v1_pb2 as datastore_pb -_BATCHES = _LocalStack() - - class Batch(object): """An abstraction representing a collected group of updates / deletes. @@ -90,7 +87,7 @@ def __init__(self, dataset_id=None, connection=None): @staticmethod def current(): """Return the topmost batch / transaction, or None.""" - return _BATCHES.top + return _CONNECTIONS.top @property def dataset_id(self): @@ -229,7 +226,7 @@ def rollback(self): pass def __enter__(self): - _BATCHES.push(self) + _CONNECTIONS.push(self) self.begin() return self @@ -240,7 +237,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): else: self.rollback() finally: - _BATCHES.pop() + _CONNECTIONS.pop() def _assign_entity_to_mutation(mutation_pb, entity, auto_id_entities): diff --git a/gcloud/datastore/connection.py b/gcloud/datastore/connection.py index 9a9f9ccc583f..47cadfeb6bf6 100644 --- a/gcloud/datastore/connection.py +++ b/gcloud/datastore/connection.py @@ -17,10 +17,14 @@ import os from gcloud import connection +from gcloud._helpers import _LocalStack from gcloud.exceptions import make_exception from gcloud.datastore import _datastore_v1_pb2 as datastore_pb +_CONNECTIONS = _LocalStack() + + SCOPE = ('https://www.googleapis.com/auth/datastore', 'https://www.googleapis.com/auth/userinfo.email') """The scopes required for authenticating as a Cloud Datastore consumer.""" diff --git a/gcloud/datastore/test___init__.py b/gcloud/datastore/test___init__.py index dd22d2787ad0..5ea79002792c 100644 --- a/gcloud/datastore/test___init__.py +++ b/gcloud/datastore/test___init__.py @@ -44,3 +44,25 @@ def call_set_connection(connection=None): self.assertEqual(SET_DATASET_CALLED, [DATASET_ID]) self.assertEqual(SET_CONNECTION_CALLED, [CONNECTION]) + + +class Test_get_connection(unittest2.TestCase): + + def _callFUT(self): + from gcloud.datastore import get_connection + return get_connection() + + def test_it(self): + from gcloud import credentials + from gcloud.datastore.connection import SCOPE + from gcloud.datastore.connection import Connection + from gcloud.test_credentials import _Client + from gcloud._testing import _Monkey + + client = _Client() + with _Monkey(credentials, client=client): + found = self._callFUT() + self.assertTrue(isinstance(found, Connection)) + self.assertTrue(found._credentials is client._signed) + self.assertEqual(found._credentials._scopes, SCOPE) + self.assertTrue(client._get_app_default_called) diff --git a/gcloud/datastore/test__implicit_environ.py b/gcloud/datastore/test__implicit_environ.py index 63f2c8c270f7..e68e004c33e5 100644 --- a/gcloud/datastore/test__implicit_environ.py +++ b/gcloud/datastore/test__implicit_environ.py @@ -281,42 +281,74 @@ def test_descriptor_for_dataset_id(self): 'dataset_id' in _implicit_environ._DEFAULTS.__dict__) def test_descriptor_for_connection(self): + from gcloud import credentials from gcloud._testing import _Monkey + from gcloud.test_credentials import _Client + from gcloud.connection import Connection from gcloud.datastore import _implicit_environ self.assertFalse( 'connection' in _implicit_environ._DEFAULTS.__dict__) - DEFAULT = object() - - with _Monkey(_implicit_environ, get_connection=lambda: DEFAULT): + client = _Client() + with _Monkey(credentials, client=client): lazy_loaded = _implicit_environ._DEFAULTS.connection - self.assertEqual(lazy_loaded, DEFAULT) + self.assertTrue(isinstance(lazy_loaded, Connection)) self.assertTrue( 'connection' in _implicit_environ._DEFAULTS.__dict__) -class Test_get_connection(unittest2.TestCase): +class Test__require_connection(unittest2.TestCase): - def _callFUT(self): - from gcloud.datastore._implicit_environ import get_connection - return get_connection() + _MARKER = object() - def test_it(self): - from gcloud import credentials - from gcloud.datastore.connection import SCOPE - from gcloud.datastore.connection import Connection - from gcloud.test_credentials import _Client - from gcloud._testing import _Monkey + def _callFUT(self, passed=_MARKER): + from gcloud.datastore._implicit_environ import _require_connection + if passed is self._MARKER: + return _require_connection() + return _require_connection(passed) - client = _Client() - with _Monkey(credentials, client=client): - found = self._callFUT() - self.assertTrue(isinstance(found, Connection)) - self.assertTrue(found._credentials is client._signed) - self.assertEqual(found._credentials._scopes, SCOPE) - self.assertTrue(client._get_app_default_called) + def _monkey(self, connection): + from gcloud.datastore._testing import _monkey_defaults + return _monkey_defaults(connection=connection) + + def test_implicit_unset(self): + with self._monkey(None): + with self.assertRaises(EnvironmentError): + self._callFUT() + + def test_implicit_unset_w_existing_batch(self): + from gcloud.datastore._testing import _NoCommitBatch + ID = 'DATASET' + CONNECTION = object() + with self._monkey(None): + with _NoCommitBatch(dataset_id=ID, connection=CONNECTION): + self.assertEqual(self._callFUT(), CONNECTION) + + def test_implicit_unset_w_existing_transaction(self): + from gcloud.datastore._testing import _NoCommitTransaction + ID = 'DATASET' + CONNECTION = object() + with self._monkey(None): + with _NoCommitTransaction(dataset_id=ID, connection=CONNECTION): + self.assertEqual(self._callFUT(), CONNECTION) + + def test_implicit_unset_passed_explicitly(self): + CONNECTION = object() + with self._monkey(None): + self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) + + def test_implicit_set(self): + IMPLICIT_CONNECTION = object() + with self._monkey(IMPLICIT_CONNECTION): + self.assertTrue(self._callFUT() is IMPLICIT_CONNECTION) + + def test_implicit_set_passed_explicitly(self): + IMPLICIT_CONNECTION = object() + CONNECTION = object() + with self._monkey(IMPLICIT_CONNECTION): + self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) class Test_set_default_connection(unittest2.TestCase): @@ -342,13 +374,17 @@ def test_set_explicit(self): self.assertEqual(_implicit_environ.get_default_connection(), fake_cnxn) def test_set_implicit(self): + from gcloud import credentials from gcloud._testing import _Monkey + from gcloud.test_credentials import _Client from gcloud.datastore import _implicit_environ + from gcloud.datastore.connection import Connection self.assertEqual(_implicit_environ.get_default_connection(), None) - fake_cnxn = object() - with _Monkey(_implicit_environ, get_connection=lambda: fake_cnxn): - self._callFUT() + client = _Client() + with _Monkey(credentials, client=client): + found = self._callFUT() - self.assertEqual(_implicit_environ.get_default_connection(), fake_cnxn) + found = _implicit_environ.get_default_connection() + self.assertTrue(isinstance(found, Connection)) diff --git a/gcloud/datastore/test_api.py b/gcloud/datastore/test_api.py index 85cee99ee0d3..3144662ee147 100644 --- a/gcloud/datastore/test_api.py +++ b/gcloud/datastore/test_api.py @@ -41,6 +41,7 @@ def test_implicit_unset_w_keys(self): self.assertEqual(self._callFUT(first_key=_Key(ID)), ID) def test_implicit_unset_w_existing_batch_wo_keys(self): + from gcloud.datastore._testing import _NoCommitBatch ID = 'DATASET' with self._monkey(None): with _NoCommitBatch(dataset_id=ID, connection=object()): @@ -48,6 +49,7 @@ def test_implicit_unset_w_existing_batch_wo_keys(self): def test_implicit_unset_w_existing_batch_w_keys(self): from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitBatch ID = 'DATASET' OTHER = 'OTHER' with self._monkey(None): @@ -55,6 +57,7 @@ def test_implicit_unset_w_existing_batch_w_keys(self): self.assertEqual(self._callFUT(first_key=_Key(OTHER)), ID) def test_implicit_unset_w_existing_transaction_wo_keys(self): + from gcloud.datastore._testing import _NoCommitTransaction ID = 'DATASET' with self._monkey(None): with _NoCommitTransaction(dataset_id=ID, connection=object()): @@ -62,6 +65,7 @@ def test_implicit_unset_w_existing_transaction_wo_keys(self): def test_implicit_unset_w_existing_transaction_w_keys(self): from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitTransaction ID = 'DATASET' OTHER = 'OTHER' with self._monkey(None): @@ -108,56 +112,6 @@ def test_id_implicit_set_passed_explicitly_w_keys(self): self.assertEqual(self._callFUT(ID, first_key=_Key(OTHER)), ID) -class Test__require_connection(unittest2.TestCase): - - _MARKER = object() - - def _callFUT(self, passed=_MARKER): - from gcloud.datastore.api import _require_connection - if passed is self._MARKER: - return _require_connection() - return _require_connection(passed) - - def _monkey(self, connection): - from gcloud.datastore._testing import _monkey_defaults - return _monkey_defaults(connection=connection) - - def test_implicit_unset(self): - with self._monkey(None): - with self.assertRaises(EnvironmentError): - self._callFUT() - - def test_implicit_unset_w_existing_batch(self): - ID = 'DATASET' - CONNECTION = object() - with self._monkey(None): - with _NoCommitBatch(dataset_id=ID, connection=CONNECTION): - self.assertEqual(self._callFUT(), CONNECTION) - - def test_implicit_unset_w_existing_transaction(self): - ID = 'DATASET' - CONNECTION = object() - with self._monkey(None): - with _NoCommitTransaction(dataset_id=ID, connection=CONNECTION): - self.assertEqual(self._callFUT(), CONNECTION) - - def test_implicit_unset_passed_explicitly(self): - CONNECTION = object() - with self._monkey(None): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - def test_implicit_set(self): - IMPLICIT_CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT() is IMPLICIT_CONNECTION) - - def test_implicit_set_passed_explicitly(self): - IMPLICIT_CONNECTION = object() - CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - class Test_get_function(unittest2.TestCase): def setUp(self): @@ -499,6 +453,7 @@ def test_implicit_wo_transaction(self): def test_w_transaction(self): from gcloud.datastore.key import Key from gcloud.datastore.test_connection import _Connection + from gcloud.datastore._testing import _NoCommitTransaction DATASET_ID = 'DATASET' KIND = 'Kind' @@ -661,6 +616,7 @@ def test_existing_batch_w_completed_key(self): from gcloud.datastore.test_batch import _Connection from gcloud.datastore.test_batch import _Entity from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitBatch # Build basic mocks needed to delete. _DATASET = 'DATASET' @@ -687,6 +643,7 @@ def test_implicit_connection(self): from gcloud.datastore.test_batch import _Connection from gcloud.datastore.test_batch import _Entity from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitBatch # Build basic mocks needed to delete. _DATASET = 'DATASET' @@ -804,6 +761,7 @@ def test_wo_batch_w_key_different_than_default_dataset_id(self): def test_w_existing_batch(self): from gcloud.datastore.test_batch import _Connection from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitBatch # Build basic mocks needed to delete. _DATASET = 'DATASET' @@ -825,6 +783,7 @@ def test_w_existing_batch(self): def test_w_existing_transaction(self): from gcloud.datastore.test_batch import _Connection from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitTransaction # Build basic mocks needed to delete. _DATASET = 'DATASET' @@ -847,6 +806,7 @@ def test_implicit_connection_and_dataset_id(self): from gcloud.datastore._testing import _monkey_defaults from gcloud.datastore.test_batch import _Connection from gcloud.datastore.test_batch import _Key + from gcloud.datastore._testing import _NoCommitBatch # Build basic mocks needed to delete. _DATASET = 'DATASET' @@ -918,39 +878,6 @@ def test_with_already_completed_key(self): COMPLETE_KEY, 2) -class _NoCommitBatch(object): - - def __init__(self, dataset_id, connection): - from gcloud.datastore.batch import Batch - self._batch = Batch(dataset_id, connection) - - def __enter__(self): - from gcloud.datastore.batch import _BATCHES - _BATCHES.push(self._batch) - return self._batch - - def __exit__(self, *args): - from gcloud.datastore.batch import _BATCHES - _BATCHES.pop() - - -class _NoCommitTransaction(object): - - def __init__(self, dataset_id, connection, transaction_id='TRANSACTION'): - from gcloud.datastore.transaction import Transaction - xact = self._transaction = Transaction(dataset_id, connection) - xact._id = transaction_id - - def __enter__(self): - from gcloud.datastore.batch import _BATCHES - _BATCHES.push(self._transaction) - return self._transaction - - def __exit__(self, *args): - from gcloud.datastore.batch import _BATCHES - _BATCHES.pop() - - class _HttpMultiple(object): def __init__(self, *responses): diff --git a/gcloud/datastore/test_batch.py b/gcloud/datastore/test_batch.py index de86cc23d2ac..ada2228d3ddd 100644 --- a/gcloud/datastore/test_batch.py +++ b/gcloud/datastore/test_batch.py @@ -279,21 +279,21 @@ def test_commit_w_auto_id_entities(self): self.assertEqual(key._id, _NEW_ID) def test_as_context_mgr_wo_error(self): - from gcloud.datastore.batch import _BATCHES + from gcloud.datastore.connection import _CONNECTIONS _DATASET = 'DATASET' _PROPERTIES = {'foo': 'bar'} connection = _Connection() entity = _Entity(_PROPERTIES) key = entity.key = _Key(_DATASET) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) with self._makeOne(dataset_id=_DATASET, connection=connection) as batch: - self.assertEqual(list(_BATCHES), [batch]) + self.assertEqual(list(_CONNECTIONS), [batch]) batch.put(entity) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) insert_auto_ids = list(batch.mutation.insert_auto_id) self.assertEqual(len(insert_auto_ids), 0) @@ -305,7 +305,7 @@ def test_as_context_mgr_wo_error(self): self.assertEqual(connection._committed, [(_DATASET, batch.mutation)]) def test_as_context_mgr_nested(self): - from gcloud.datastore.batch import _BATCHES + from gcloud.datastore.connection import _CONNECTIONS _DATASET = 'DATASET' _PROPERTIES = {'foo': 'bar'} connection = _Connection() @@ -314,20 +314,20 @@ def test_as_context_mgr_nested(self): entity2 = _Entity(_PROPERTIES) key2 = entity2.key = _Key(_DATASET) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) with self._makeOne(dataset_id=_DATASET, connection=connection) as batch1: - self.assertEqual(list(_BATCHES), [batch1]) + self.assertEqual(list(_CONNECTIONS), [batch1]) batch1.put(entity1) with self._makeOne(dataset_id=_DATASET, connection=connection) as batch2: - self.assertEqual(list(_BATCHES), [batch2, batch1]) + self.assertEqual(list(_CONNECTIONS), [batch2, batch1]) batch2.put(entity2) - self.assertEqual(list(_BATCHES), [batch1]) + self.assertEqual(list(_CONNECTIONS), [batch1]) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) insert_auto_ids = list(batch1.mutation.insert_auto_id) self.assertEqual(len(insert_auto_ids), 0) @@ -350,25 +350,25 @@ def test_as_context_mgr_nested(self): (_DATASET, batch1.mutation)]) def test_as_context_mgr_w_error(self): - from gcloud.datastore.batch import _BATCHES + from gcloud.datastore.connection import _CONNECTIONS _DATASET = 'DATASET' _PROPERTIES = {'foo': 'bar'} connection = _Connection() entity = _Entity(_PROPERTIES) key = entity.key = _Key(_DATASET) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) try: with self._makeOne(dataset_id=_DATASET, connection=connection) as batch: - self.assertEqual(list(_BATCHES), [batch]) + self.assertEqual(list(_CONNECTIONS), [batch]) batch.put(entity) raise ValueError("testing") except ValueError: pass - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) insert_auto_ids = list(batch.mutation.insert_auto_id) self.assertEqual(len(insert_auto_ids), 0) diff --git a/gcloud/datastore/test_transaction.py b/gcloud/datastore/test_transaction.py index d678712a80a0..e34aa3e49920 100644 --- a/gcloud/datastore/test_transaction.py +++ b/gcloud/datastore/test_transaction.py @@ -73,7 +73,7 @@ def test_ctor_with_env(self): self.assertEqual(xact._status, self._getTargetClass()._INITIAL) def test_current(self): - from gcloud.datastore.test_api import _NoCommitBatch + from gcloud.datastore._testing import _NoCommitBatch _DATASET = 'DATASET' connection = _Connection() xact1 = self._makeOne(_DATASET, connection) diff --git a/gcloud/storage/__init__.py b/gcloud/storage/__init__.py index 8097e51c9b6c..344d9ea6ad44 100644 --- a/gcloud/storage/__init__.py +++ b/gcloud/storage/__init__.py @@ -43,7 +43,6 @@ from gcloud._helpers import get_default_project from gcloud._helpers import set_default_project from gcloud.storage import _implicit_environ -from gcloud.storage._implicit_environ import get_connection from gcloud.storage._implicit_environ import get_default_bucket from gcloud.storage._implicit_environ import get_default_connection from gcloud.storage._implicit_environ import set_default_connection @@ -105,3 +104,20 @@ def set_defaults(bucket=None, project=None, connection=None): # NOTE: `set_default_bucket` is called after `set_default_connection` # since `set_default_bucket` falls back to implicit connection. set_default_bucket(bucket=bucket) + + +def get_connection(): + """Shortcut method to establish a connection to Cloud Storage. + + Use this if you are going to access several buckets with the same + set of credentials: + + >>> from gcloud import storage + >>> connection = storage.get_connection() + >>> bucket1 = storage.get_bucket('bucket1', connection=connection) + >>> bucket2 = storage.get_bucket('bucket2', connection=connection) + + :rtype: :class:`gcloud.storage.connection.Connection` + :returns: A connection defined with the proper credentials. + """ + return Connection.from_environment() diff --git a/gcloud/storage/_helpers.py b/gcloud/storage/_helpers.py index a5acb45c645e..b90f316e33ee 100644 --- a/gcloud/storage/_helpers.py +++ b/gcloud/storage/_helpers.py @@ -20,8 +20,7 @@ from Crypto.Hash import MD5 import base64 -from gcloud.storage._implicit_environ import get_default_connection -from gcloud.storage.batch import Batch +from gcloud.storage._implicit_environ import _require_connection class _PropertyMixin(object): @@ -113,30 +112,6 @@ def patch(self, connection=None): self._set_properties(api_response) -def _require_connection(connection=None): - """Infer a connection from the environment, if not passed explicitly. - - :type connection: :class:`gcloud.storage.connection.Connection` - :param connection: Optional. - - :rtype: :class:`gcloud.storage.connection.Connection` - :returns: A connection based on the current environment. - :raises: :class:`EnvironmentError` if ``connection`` is ``None``, and - cannot be inferred from the environment. - """ - # NOTE: We use current Batch directly since it inherits from Connection. - if connection is None: - connection = Batch.current() - - if connection is None: - connection = get_default_connection() - - if connection is None: - raise EnvironmentError('Connection could not be inferred.') - - return connection - - def _scalar_property(fieldname): """Create a property descriptor around the :class:`_PropertyMixin` helpers. """ diff --git a/gcloud/storage/_implicit_environ.py b/gcloud/storage/_implicit_environ.py index b90f99ed67b8..d7fe11094996 100644 --- a/gcloud/storage/_implicit_environ.py +++ b/gcloud/storage/_implicit_environ.py @@ -21,6 +21,7 @@ from gcloud._helpers import _lazy_property_deco from gcloud.storage.connection import Connection +from gcloud.storage.connection import _CONNECTIONS class _DefaultsContainer(object): @@ -37,7 +38,7 @@ class _DefaultsContainer(object): @staticmethod def connection(): """Return the implicit default connection..""" - return get_connection() + return Connection.from_environment() def __init__(self, bucket=None, connection=None, implicit=False): self.bucket = bucket @@ -54,30 +55,36 @@ def get_default_bucket(): return _DEFAULTS.bucket -def get_default_connection(): - """Get default connection. +def _require_connection(connection=None): + """Infer a connection from the environment, if not passed explicitly. - :rtype: :class:`gcloud.storage.connection.Connection` or ``NoneType`` - :returns: The default connection if one has been set. + :type connection: :class:`gcloud.storage.connection.Connection` + :param connection: Optional. + + :rtype: :class:`gcloud.storage.connection.Connection` + :returns: A connection based on the current environment. + :raises: :class:`EnvironmentError` if ``connection`` is ``None``, and + cannot be inferred from the environment. """ - return _DEFAULTS.connection + if connection is None: + connection = _CONNECTIONS.top + if connection is None: + connection = get_default_connection() -def get_connection(): - """Shortcut method to establish a connection to Cloud Storage. + if connection is None: + raise EnvironmentError('Connection could not be inferred.') - Use this if you are going to access several buckets with the same - set of credentials: + return connection - >>> from gcloud import storage - >>> connection = storage.get_connection() - >>> bucket1 = storage.get_bucket('bucket1', connection=connection) - >>> bucket2 = storage.get_bucket('bucket2', connection=connection) - :rtype: :class:`gcloud.storage.connection.Connection` - :returns: A connection defined with the proper credentials. +def get_default_connection(): + """Get default connection. + + :rtype: :class:`gcloud.storage.connection.Connection` or ``NoneType`` + :returns: The default connection if one has been set. """ - return Connection.from_environment() + return _DEFAULTS.connection def set_default_connection(connection=None): @@ -86,7 +93,7 @@ def set_default_connection(connection=None): :type connection: :class:`gcloud.storage.connection.Connection` :param connection: A connection provided to be the default. """ - connection = connection or get_connection() + connection = connection or Connection.from_environment() _DEFAULTS.connection = connection diff --git a/gcloud/storage/acl.py b/gcloud/storage/acl.py index 84e3c7911876..71c709b75636 100644 --- a/gcloud/storage/acl.py +++ b/gcloud/storage/acl.py @@ -78,7 +78,7 @@ when sending metadata for ACLs to the API. """ -from gcloud.storage._helpers import _require_connection +from gcloud.storage._implicit_environ import _require_connection class _ACLEntity(object): diff --git a/gcloud/storage/api.py b/gcloud/storage/api.py index 1fdd6ad2e484..d0dfbac25234 100644 --- a/gcloud/storage/api.py +++ b/gcloud/storage/api.py @@ -20,7 +20,7 @@ from gcloud.exceptions import NotFound from gcloud._helpers import get_default_project -from gcloud.storage._helpers import _require_connection +from gcloud.storage._implicit_environ import _require_connection from gcloud.storage.bucket import Bucket from gcloud.storage.iterator import Iterator diff --git a/gcloud/storage/batch.py b/gcloud/storage/batch.py index 151e99a18888..5142877f6f95 100644 --- a/gcloud/storage/batch.py +++ b/gcloud/storage/batch.py @@ -26,13 +26,10 @@ import six -from gcloud._helpers import _LocalStack from gcloud.exceptions import make_exception from gcloud.storage import _implicit_environ from gcloud.storage.connection import Connection - - -_BATCHES = _LocalStack() +from gcloud.storage.connection import _CONNECTIONS class MIMEApplicationHTTP(MIMEApplication): @@ -245,10 +242,10 @@ def finish(self): @staticmethod def current(): """Return the topmost batch, or None.""" - return _BATCHES.top + return _CONNECTIONS.top def __enter__(self): - _BATCHES.push(self) + _CONNECTIONS.push(self) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -256,7 +253,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: self.finish() finally: - _BATCHES.pop() + _CONNECTIONS.pop() def _generate_faux_mime_message(parser, response, content): diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 5190528115bb..7dd98e422983 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -32,7 +32,7 @@ from gcloud.credentials import generate_signed_url from gcloud.exceptions import NotFound from gcloud.storage._helpers import _PropertyMixin -from gcloud.storage._helpers import _require_connection +from gcloud.storage._implicit_environ import _require_connection from gcloud.storage._helpers import _scalar_property from gcloud.storage import _implicit_environ from gcloud.storage.acl import ObjectACL diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 68dcad9032d7..f1f22913d6ac 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -24,8 +24,8 @@ from gcloud._helpers import get_default_project from gcloud.exceptions import NotFound from gcloud.storage._helpers import _PropertyMixin -from gcloud.storage._helpers import _require_connection from gcloud.storage._helpers import _scalar_property +from gcloud.storage._implicit_environ import _require_connection from gcloud.storage.acl import BucketACL from gcloud.storage.acl import DefaultObjectACL from gcloud.storage.iterator import Iterator diff --git a/gcloud/storage/connection.py b/gcloud/storage/connection.py index 2740a47063db..24754822b273 100644 --- a/gcloud/storage/connection.py +++ b/gcloud/storage/connection.py @@ -15,6 +15,10 @@ """Create / interact with gcloud storage connections.""" from gcloud import connection as base_connection +from gcloud._helpers import _LocalStack + + +_CONNECTIONS = _LocalStack() SCOPE = ('https://www.googleapis.com/auth/devstorage.full_control', diff --git a/gcloud/storage/test___init__.py b/gcloud/storage/test___init__.py index e47f98fd777f..f50e85280bab 100644 --- a/gcloud/storage/test___init__.py +++ b/gcloud/storage/test___init__.py @@ -153,3 +153,24 @@ def call_set_connection(connection=None): self.assertEqual(SET_PROJECT_CALLED, [PROJECT]) self.assertEqual(SET_CONNECTION_CALLED, [CONNECTION]) self.assertEqual(SET_BUCKET_CALLED, [BUCKET]) + + +class Test_get_connection(unittest2.TestCase): + + def _callFUT(self, *args, **kw): + from gcloud.storage import get_connection + return get_connection(*args, **kw) + + def test_it(self): + from gcloud import credentials + from gcloud.storage import SCOPE + from gcloud.storage.connection import Connection + from gcloud.test_credentials import _Client + from gcloud._testing import _Monkey + client = _Client() + with _Monkey(credentials, client=client): + found = self._callFUT() + self.assertTrue(isinstance(found, Connection)) + self.assertTrue(found._credentials is client._signed) + self.assertEqual(found._credentials._scopes, SCOPE) + self.assertTrue(client._get_app_default_called) diff --git a/gcloud/storage/test__helpers.py b/gcloud/storage/test__helpers.py index 104ecb38bd05..f179c4649cba 100644 --- a/gcloud/storage/test__helpers.py +++ b/gcloud/storage/test__helpers.py @@ -155,44 +155,6 @@ def _patch_property(self, name, value): self.assertEqual(test._patched, ('solfege', 'Latido')) -class Test__require_connection(unittest2.TestCase): - - def _callFUT(self, connection=None): - from gcloud.storage._helpers import _require_connection - return _require_connection(connection=connection) - - def _monkey(self, connection): - from gcloud.storage._testing import _monkey_defaults - return _monkey_defaults(connection=connection) - - def test_implicit_unset(self): - with self._monkey(None): - with self.assertRaises(EnvironmentError): - self._callFUT() - - def test_implicit_unset_w_existing_batch(self): - CONNECTION = object() - with self._monkey(None): - with _NoCommitBatch(connection=CONNECTION): - self.assertEqual(self._callFUT(), CONNECTION) - - def test_implicit_unset_passed_explicitly(self): - CONNECTION = object() - with self._monkey(None): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - def test_implicit_set(self): - IMPLICIT_CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT() is IMPLICIT_CONNECTION) - - def test_implicit_set_passed_explicitly(self): - IMPLICIT_CONNECTION = object() - CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - class Test__base64_md5hash(unittest2.TestCase): def _callFUT(self, bytes_to_sign): @@ -286,18 +248,3 @@ def __init__(self): def b64encode(self, value): self._called_b64encode.append(value) return value - - -class _NoCommitBatch(object): - - def __init__(self, connection): - self._connection = connection - - def __enter__(self): - from gcloud.storage.batch import _BATCHES - _BATCHES.push(self._connection) - return self._connection - - def __exit__(self, *args): - from gcloud.storage.batch import _BATCHES - _BATCHES.pop() diff --git a/gcloud/storage/test__implicit_environ.py b/gcloud/storage/test__implicit_environ.py index 2176a7090066..80e49b9f72d3 100644 --- a/gcloud/storage/test__implicit_environ.py +++ b/gcloud/storage/test__implicit_environ.py @@ -25,6 +25,44 @@ def test_wo_override(self): self.assertTrue(self._callFUT() is None) +class Test__require_connection(unittest2.TestCase): + + def _callFUT(self, connection=None): + from gcloud.storage._implicit_environ import _require_connection + return _require_connection(connection=connection) + + def _monkey(self, connection): + from gcloud.storage._testing import _monkey_defaults + return _monkey_defaults(connection=connection) + + def test_implicit_unset(self): + with self._monkey(None): + with self.assertRaises(EnvironmentError): + self._callFUT() + + def test_implicit_unset_w_existing_batch(self): + CONNECTION = object() + with self._monkey(None): + with _NoCommitBatch(connection=CONNECTION): + self.assertEqual(self._callFUT(), CONNECTION) + + def test_implicit_unset_passed_explicitly(self): + CONNECTION = object() + with self._monkey(None): + self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) + + def test_implicit_set(self): + IMPLICIT_CONNECTION = object() + with self._monkey(IMPLICIT_CONNECTION): + self.assertTrue(self._callFUT() is IMPLICIT_CONNECTION) + + def test_implicit_set_passed_explicitly(self): + IMPLICIT_CONNECTION = object() + CONNECTION = object() + with self._monkey(IMPLICIT_CONNECTION): + self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) + + class Test_get_default_connection(unittest2.TestCase): def setUp(self): @@ -43,27 +81,6 @@ def test_wo_override(self): self.assertTrue(self._callFUT() is None) -class Test_get_connection(unittest2.TestCase): - - def _callFUT(self, *args, **kw): - from gcloud.storage._implicit_environ import get_connection - return get_connection(*args, **kw) - - def test_it(self): - from gcloud import credentials - from gcloud.storage import SCOPE - from gcloud.storage.connection import Connection - from gcloud.test_credentials import _Client - from gcloud._testing import _Monkey - client = _Client() - with _Monkey(credentials, client=client): - found = self._callFUT() - self.assertTrue(isinstance(found, Connection)) - self.assertTrue(found._credentials is client._signed) - self.assertEqual(found._credentials._scopes, SCOPE) - self.assertTrue(client._get_app_default_called) - - class Test_set_default_connection(unittest2.TestCase): def setUp(self): @@ -87,26 +104,20 @@ def test_set_explicit(self): self.assertEqual(_implicit_environ.get_default_connection(), fake_cnxn) def test_set_implicit(self): + from gcloud import credentials from gcloud._testing import _Monkey + from gcloud.test_credentials import _Client from gcloud.storage import _implicit_environ + from gcloud.storage.connection import Connection self.assertEqual(_implicit_environ.get_default_connection(), None) - fake_cnxn = object() - _called_args = [] - _called_kwargs = [] - - def mock_get_connection(*args, **kwargs): - _called_args.append(args) - _called_kwargs.append(kwargs) - return fake_cnxn - - with _Monkey(_implicit_environ, get_connection=mock_get_connection): + client = _Client() + with _Monkey(credentials, client=client): self._callFUT() - self.assertEqual(_implicit_environ.get_default_connection(), fake_cnxn) - self.assertEqual(_called_args, [()]) - self.assertEqual(_called_kwargs, [{}]) + found = _implicit_environ.get_default_connection() + self.assertTrue(isinstance(found, Connection)) class Test_lazy_loading(unittest2.TestCase): @@ -120,17 +131,34 @@ def tearDown(self): _tear_down_defaults(self) def test_descriptor_for_connection(self): + from gcloud import credentials from gcloud._testing import _Monkey + from gcloud.test_credentials import _Client from gcloud.storage import _implicit_environ + from gcloud.storage.connection import Connection self.assertFalse( 'connection' in _implicit_environ._DEFAULTS.__dict__) - DEFAULT = object() - - with _Monkey(_implicit_environ, get_connection=lambda: DEFAULT): + client = _Client() + with _Monkey(credentials, client=client): lazy_loaded = _implicit_environ._DEFAULTS.connection - self.assertEqual(lazy_loaded, DEFAULT) + self.assertTrue(isinstance(lazy_loaded, Connection)) self.assertTrue( 'connection' in _implicit_environ._DEFAULTS.__dict__) + + +class _NoCommitBatch(object): + + def __init__(self, connection): + self._connection = connection + + def __enter__(self): + from gcloud.storage.connection import _CONNECTIONS + _CONNECTIONS.push(self._connection) + return self._connection + + def __exit__(self, *args): + from gcloud.storage.connection import _CONNECTIONS + _CONNECTIONS.pop() diff --git a/gcloud/storage/test_batch.py b/gcloud/storage/test_batch.py index 94695671090e..b6b8e50d5101 100644 --- a/gcloud/storage/test_batch.py +++ b/gcloud/storage/test_batch.py @@ -222,6 +222,20 @@ def test_finish_empty(self): self.assertRaises(ValueError, batch.finish) self.assertTrue(connection.http is http) + def test_current(self): + from gcloud.storage.connection import _CONNECTIONS + klass = self._getTargetClass() + http = _HTTP() # no requests expected + connection = _Connection(http=http) + self.assertTrue(klass.current() is None) + batch = self._makeOne(connection) + _CONNECTIONS.push(batch) + try: + self.assertTrue(klass.current() is batch) + finally: + _CONNECTIONS.pop() + self.assertTrue(klass.current() is None) + def _check_subrequest_no_payload(self, chunk, method, url): lines = chunk.splitlines() # blank + 2 headers + blank + request + blank + blank @@ -370,27 +384,27 @@ def test_finish_nonempty_non_multipart_response(self): self.assertRaises(ValueError, batch.finish) def test_as_context_mgr_wo_error(self): - from gcloud.storage.batch import _BATCHES + from gcloud.storage.connection import _CONNECTIONS URL = 'http://example.com/api' expected = _Response() expected['content-type'] = 'multipart/mixed; boundary="DEADBEEF="' http = _HTTP((expected, _THREE_PART_MIME_RESPONSE)) connection = _Connection(http=http) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) target1 = _MockObject() target2 = _MockObject() target3 = _MockObject() with self._makeOne(connection) as batch: - self.assertEqual(list(_BATCHES), [batch]) + self.assertEqual(list(_CONNECTIONS), [batch]) batch._make_request('POST', URL, {'foo': 1, 'bar': 2}, target_object=target1) batch._make_request('PATCH', URL, {'bar': 3}, target_object=target2) batch._make_request('DELETE', URL, target_object=target3) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) self.assertEqual(len(batch._requests), 3) self.assertEqual(batch._requests[0][0], 'POST') self.assertEqual(batch._requests[1][0], 'PATCH') @@ -404,19 +418,19 @@ def test_as_context_mgr_wo_error(self): def test_as_context_mgr_w_error(self): from gcloud.storage.batch import _FutureDict - from gcloud.storage.batch import _BATCHES + from gcloud.storage.connection import _CONNECTIONS URL = 'http://example.com/api' http = _HTTP() connection = _Connection(http=http) - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) target1 = _MockObject() target2 = _MockObject() target3 = _MockObject() try: with self._makeOne(connection) as batch: - self.assertEqual(list(_BATCHES), [batch]) + self.assertEqual(list(_CONNECTIONS), [batch]) batch._make_request('POST', URL, {'foo': 1, 'bar': 2}, target_object=target1) batch._make_request('PATCH', URL, {'bar': 3}, @@ -426,7 +440,7 @@ def test_as_context_mgr_w_error(self): except ValueError: pass - self.assertEqual(list(_BATCHES), []) + self.assertEqual(list(_CONNECTIONS), []) self.assertEqual(len(http._requests), 0) self.assertEqual(len(batch._requests), 3) self.assertEqual(batch._target_objects, [target1, target2, target3])