diff --git a/gcloud/datastore/__init__.py b/gcloud/datastore/__init__.py index db0e8dfa2d0d..5b6d2d9815bb 100644 --- a/gcloud/datastore/__init__.py +++ b/gcloud/datastore/__init__.py @@ -54,13 +54,6 @@ from gcloud.datastore._implicit_environ import get_default_dataset_id from gcloud.datastore._implicit_environ import set_default_connection from gcloud.datastore._implicit_environ import set_default_dataset_id -from gcloud.datastore.api import allocate_ids -from gcloud.datastore.api import delete -from gcloud.datastore.api import delete_multi -from gcloud.datastore.api import get -from gcloud.datastore.api import get_multi -from gcloud.datastore.api import put -from gcloud.datastore.api import put_multi from gcloud.datastore.batch import Batch from gcloud.datastore.connection import SCOPE from gcloud.datastore.connection import Connection diff --git a/gcloud/datastore/api.py b/gcloud/datastore/api.py deleted file mode 100644 index a54c1bb33904..000000000000 --- a/gcloud/datastore/api.py +++ /dev/null @@ -1,426 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Methods for interacting with Google Cloud Datastore. - -Allows interacting with the datastore via user-friendly Key, Entity and -Query objects rather than via protobufs. -""" - -from gcloud.datastore import _implicit_environ -from gcloud.datastore.batch import Batch -from gcloud.datastore.entity import Entity -from gcloud.datastore.transaction import Transaction -from gcloud.datastore import helpers - - -_MAX_LOOPS = 128 -"""Maximum number of iterations to wait for deferred keys.""" - - -def _require_dataset_id(dataset_id=None, first_key=None): - """Infer a dataset ID from the environment, if not passed explicitly. - - Order of precedence: - - - Passed `dataset_id` (if not None). - - `dataset_id` of current batch / transaction (if current exists). - - `dataset_id` of first key - - `dataset_id` inferred from the environment (if `set_default_dataset_id` - has been called). - - :type dataset_id: string - :param dataset_id: Optional. - - :type first_key: :class:`gcloud.datastore.key.Key` or None - :param first_key: Optional: first key being manipulated. - - :rtype: string - :returns: A dataset ID based on the current environment. - :raises: :class:`EnvironmentError` if ``dataset_id`` is ``None``, - and cannot be inferred from the environment. - """ - if dataset_id is not None: - return dataset_id - top = Batch.current() - if top is not None: - return top.dataset_id - if first_key is not None: - return first_key.dataset_id - - dataset_id = _implicit_environ.get_default_dataset_id() - if dataset_id is None: - raise EnvironmentError('Dataset ID could not be inferred.') - return dataset_id - - -def _require_connection(connection=None): - """Infer a connection from the environment, if not passed explicitly. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional. - - :rtype: :class:`gcloud.datastore.connection.Connection` - :returns: A connection based on the current environment. - :raises: :class:`EnvironmentError` if ``connection`` is ``None``, and - cannot be inferred from the environment. - """ - if connection is None: - top = Batch.current() - if top is not None: - connection = top.connection - else: - connection = _implicit_environ.get_default_connection() - if connection is None: - raise EnvironmentError('Connection could not be inferred.') - return connection - - -def _extended_lookup(connection, dataset_id, key_pbs, - missing=None, deferred=None, - eventual=False, transaction_id=None): - """Repeat lookup until all keys found (unless stop requested). - - Helper method for :func:`get`. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: The connection used to connect to datastore. - - :type dataset_id: string - :param dataset_id: The ID of the dataset of which to make the request. - - :type key_pbs: list of :class:`gcloud.datastore._datastore_v1_pb2.Key` - :param key_pbs: The keys to retrieve from the datastore. - - :type missing: an empty list or None. - :param missing: If a list is passed, the key-only entity protobufs - returned by the backend as "missing" will be copied - into it. Use only as a keyword param. - - :type deferred: an empty list or None. - :param deferred: If a list is passed, the key protobufs returned - by the backend as "deferred" will be copied into it. - Use only as a keyword param. - - :type eventual: boolean - :param eventual: If False (the default), request ``STRONG`` read - consistency. If True, request ``EVENTUAL`` read - consistency. - - :type transaction_id: string - :param transaction_id: If passed, make the request in the scope of - the given transaction. Incompatible with - ``eventual==True``. - - :rtype: list of :class:`gcloud.datastore._datastore_v1_pb2.Entity` - :returns: The requested entities. - :raises: :class:`ValueError` if missing / deferred are not null or - empty list. - """ - if missing is not None and missing != []: - raise ValueError('missing must be None or an empty list') - - if deferred is not None and deferred != []: - raise ValueError('deferred must be None or an empty list') - - results = [] - - loop_num = 0 - while loop_num < _MAX_LOOPS: # loop against possible deferred. - loop_num += 1 - - results_found, missing_found, deferred_found = connection.lookup( - dataset_id=dataset_id, - key_pbs=key_pbs, - eventual=eventual, - transaction_id=transaction_id, - ) - - results.extend(results_found) - - if missing is not None: - missing.extend(missing_found) - - if deferred is not None: - deferred.extend(deferred_found) - break - - if len(deferred_found) == 0: - break - - # We have deferred keys, and the user didn't ask to know about - # them, so retry (but only with the deferred ones). - key_pbs = deferred_found - - return results - - -def get_multi(keys, missing=None, deferred=None, - connection=None, dataset_id=None): - """Retrieves entities, along with their attributes. - - :type keys: list of :class:`gcloud.datastore.key.Key` - :param keys: The keys to be retrieved from the datastore. - - :type missing: an empty list or None. - :param missing: If a list is passed, the key-only entities returned - by the backend as "missing" will be copied into it. - Use only as a keyword param. - - :type deferred: an empty list or None. - :param deferred: If a list is passed, the keys returned - by the backend as "deferred" will be copied into it. - Use only as a keyword param. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional. The connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - - :rtype: list of :class:`gcloud.datastore.entity.Entity` - :returns: The requested entities. - :raises: EnvironmentError if ``connection`` or ``dataset_id`` not passed, - and cannot be inferred from the environment. ValueError if - one or more of ``keys`` has a dataset ID which does not match - the passed / inferred dataset ID. - """ - if not keys: - return [] - - connection = _require_connection(connection) - dataset_id = _require_dataset_id(dataset_id, keys[0]) - - if list(set([key.dataset_id for key in keys])) != [dataset_id]: - raise ValueError('Keys do not match dataset ID') - - transaction = Transaction.current() - - entity_pbs = _extended_lookup( - connection, - dataset_id=dataset_id, - key_pbs=[k.to_protobuf() for k in keys], - missing=missing, - deferred=deferred, - transaction_id=transaction and transaction.id, - ) - - if missing is not None: - missing[:] = [ - helpers.entity_from_protobuf(missed_pb) - for missed_pb in missing] - - if deferred is not None: - deferred[:] = [ - helpers.key_from_protobuf(deferred_pb) - for deferred_pb in deferred] - - entities = [] - for entity_pb in entity_pbs: - entities.append(helpers.entity_from_protobuf(entity_pb)) - - return entities - - -def get(key, missing=None, deferred=None, connection=None, dataset_id=None): - """Retrieves entity from a single key (if it exists). - - .. note:: - - This is just a thin wrapper over :func:`gcloud.datastore.get_multi`. - The backend API does not make a distinction between a single key or - multiple keys in a lookup request. - - :type key: :class:`gcloud.datastore.key.Key` - :param key: The key to be retrieved from the datastore. - - :type missing: an empty list or None. - :param missing: If a list is passed, the key-only entities returned - by the backend as "missing" will be copied into it. - Use only as a keyword param. - - :type deferred: an empty list or None. - :param deferred: If a list is passed, the keys returned - by the backend as "deferred" will be copied into it. - Use only as a keyword param. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional. The connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - - :rtype: :class:`gcloud.datastore.entity.Entity` or ``NoneType`` - :returns: The requested entity if it exists. - """ - entities = get_multi([key], missing=missing, deferred=deferred, - connection=connection, dataset_id=dataset_id) - if entities: - return entities[0] - - -def put_multi(entities, connection=None, dataset_id=None): - """Save the entities in the Cloud Datastore. - - :type entities: list of :class:`gcloud.datastore.entity.Entity` - :param entities: The entities to be saved to the datastore. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - - :raises: EnvironmentError if ``connection`` or ``dataset_id`` not passed, - and cannot be inferred from the environment. ValueError if - one or more entities has a key with a dataset ID not matching - the passed / inferred dataset ID. - """ - if isinstance(entities, Entity): - raise ValueError("Pass a sequence of entities") - - if not entities: - return - - connection = _require_connection(connection) - dataset_id = _require_dataset_id(dataset_id, entities[0].key) - - current = Batch.current() - in_batch = current is not None - if not in_batch: - current = Batch(dataset_id=dataset_id, connection=connection) - for entity in entities: - current.put(entity) - if not in_batch: - current.commit() - - -def put(entity, connection=None, dataset_id=None): - """Save the entity in the Cloud Datastore. - - .. note:: - - This is just a thin wrapper over :func:`gcloud.datastore.put_multi`. - The backend API does not make a distinction between a single entity or - multiple entities in a commit request. - - :type entity: :class:`gcloud.datastore.entity.Entity` - :param entity: The entity to be saved to the datastore. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - """ - put_multi([entity], connection=connection, dataset_id=dataset_id) - - -def delete_multi(keys, connection=None, dataset_id=None): - """Delete the keys in the Cloud Datastore. - - :type keys: list of :class:`gcloud.datastore.key.Key` - :param keys: The keys to be deleted from the datastore. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - - :raises: EnvironmentError if ``connection`` or ``dataset_id`` not passed, - and cannot be inferred from the environment. ValueError if - one or more keys has a dataset ID not matching the passed / - inferred dataset ID. - """ - if not keys: - return - - connection = _require_connection(connection) - dataset_id = _require_dataset_id(dataset_id, keys[0]) - - # We allow partial keys to attempt a delete, the backend will fail. - current = Batch.current() - in_batch = current is not None - if not in_batch: - current = Batch(dataset_id=dataset_id, connection=connection) - for key in keys: - current.delete(key) - if not in_batch: - current.commit() - - -def delete(key, connection=None, dataset_id=None): - """Delete the key in the Cloud Datastore. - - .. note:: - - This is just a thin wrapper over :func:`gcloud.datastore.delete_multi`. - The backend API does not make a distinction between a single key or - multiple keys in a commit request. - - :type key: :class:`gcloud.datastore.key.Key` - :param key: The key to be deleted from the datastore. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional connection used to connect to datastore. - If not passed, inferred from the environment. - - :type dataset_id: :class:`gcloud.datastore.connection.Connection` - :param dataset_id: Optional. The dataset ID used to connect to datastore. - If not passed, inferred from the environment. - """ - delete_multi([key], connection=connection, dataset_id=dataset_id) - - -def allocate_ids(incomplete_key, num_ids, connection=None): - """Allocates a list of IDs from a partial key. - - :type incomplete_key: A :class:`gcloud.datastore.key.Key` - :param incomplete_key: Partial key to use as base for allocated IDs. - - :type num_ids: integer - :param num_ids: The number of IDs to allocate. - - :type connection: :class:`gcloud.datastore.connection.Connection` - :param connection: Optional. The connection used to connect to datastore. - - :rtype: list of :class:`gcloud.datastore.key.Key` - :returns: The (complete) keys allocated with ``incomplete_key`` as root. - :raises: :class:`ValueError` if ``incomplete_key`` is not a partial key. - """ - connection = _require_connection(connection) - - if not incomplete_key.is_partial: - raise ValueError(('Key is not partial.', incomplete_key)) - - incomplete_key_pb = incomplete_key.to_protobuf() - incomplete_key_pbs = [incomplete_key_pb] * num_ids - - allocated_key_pbs = connection.allocate_ids(incomplete_key.dataset_id, - incomplete_key_pbs) - allocated_ids = [allocated_key_pb.path_element[-1].id - for allocated_key_pb in allocated_key_pbs] - return [incomplete_key.completed_key(allocated_id) - for allocated_id in allocated_ids] diff --git a/gcloud/datastore/client.py b/gcloud/datastore/client.py index b1aa8006fb65..b5cb0f142f89 100644 --- a/gcloud/datastore/client.py +++ b/gcloud/datastore/client.py @@ -13,13 +13,9 @@ # limitations under the License. """Convenience wrapper for invoking APIs/factories w/ a dataset ID.""" -from gcloud.datastore.api import delete -from gcloud.datastore.api import delete_multi -from gcloud.datastore.api import get -from gcloud.datastore.api import get_multi -from gcloud.datastore.api import put -from gcloud.datastore.api import put_multi +from gcloud.datastore import helpers from gcloud.datastore.batch import Batch +from gcloud.datastore.entity import Entity from gcloud.datastore.key import Key from gcloud.datastore.query import Query from gcloud.datastore.transaction import Transaction @@ -27,6 +23,89 @@ from gcloud.datastore._implicit_environ import get_connection +_MAX_LOOPS = 128 +"""Maximum number of iterations to wait for deferred keys.""" + + +def _extended_lookup(connection, dataset_id, key_pbs, + missing=None, deferred=None, + eventual=False, transaction_id=None): + """Repeat lookup until all keys found (unless stop requested). + + Helper function for :method:`Client.get_multi`. + + :type connection: :class:`gcloud.datastore.connection.Connection` + :param connection: The connection used to connect to datastore. + + :type dataset_id: string + :param dataset_id: The ID of the dataset of which to make the request. + + :type key_pbs: list of :class:`gcloud.datastore._datastore_v1_pb2.Key` + :param key_pbs: The keys to retrieve from the datastore. + + :type missing: an empty list or None. + :param missing: If a list is passed, the key-only entity protobufs + returned by the backend as "missing" will be copied + into it. Use only as a keyword param. + + :type deferred: an empty list or None. + :param deferred: If a list is passed, the key protobufs returned + by the backend as "deferred" will be copied into it. + Use only as a keyword param. + + :type eventual: boolean + :param eventual: If False (the default), request ``STRONG`` read + consistency. If True, request ``EVENTUAL`` read + consistency. + + :type transaction_id: string + :param transaction_id: If passed, make the request in the scope of + the given transaction. Incompatible with + ``eventual==True``. + + :rtype: list of :class:`gcloud.datastore._datastore_v1_pb2.Entity` + :returns: The requested entities. + :raises: :class:`ValueError` if missing / deferred are not null or + empty list. + """ + if missing is not None and missing != []: + raise ValueError('missing must be None or an empty list') + + if deferred is not None and deferred != []: + raise ValueError('deferred must be None or an empty list') + + results = [] + + loop_num = 0 + while loop_num < _MAX_LOOPS: # loop against possible deferred. + loop_num += 1 + + results_found, missing_found, deferred_found = connection.lookup( + dataset_id=dataset_id, + key_pbs=key_pbs, + eventual=eventual, + transaction_id=transaction_id, + ) + + results.extend(results_found) + + if missing is not None: + missing.extend(missing_found) + + if deferred is not None: + deferred.extend(deferred_found) + break + + if len(deferred_found) == 0: + break + + # We have deferred keys, and the user didn't ask to know about + # them, so retry (but only with the deferred ones). + key_pbs = deferred_found + + return results + + class Client(object): """Convenience wrapper for invoking APIs/factories w/ a dataset ID. @@ -51,53 +130,191 @@ def __init__(self, dataset_id=None, namespace=None, connection=None): self.namespace = namespace def get(self, key, missing=None, deferred=None): - """Proxy to :func:`gcloud.datastore.api.get`. + """Retrieve an entity from a single key (if it exists). - Passes our ``dataset_id``. + .. note:: + + This is just a thin wrapper over :method:`get_multi`. + The backend API does not make a distinction between a single key or + multiple keys in a lookup request. + + :type key: :class:`gcloud.datastore.key.Key` + :param key: The key to be retrieved from the datastore. + + :type missing: an empty list or None. + :param missing: If a list is passed, the key-only entities returned + by the backend as "missing" will be copied into it. + Use only as a keyword param. + + :type deferred: an empty list or None. + :param deferred: If a list is passed, the keys returned + by the backend as "deferred" will be copied into it. + Use only as a keyword param. + + :rtype: :class:`gcloud.datastore.entity.Entity` or ``NoneType`` + :returns: The requested entity if it exists. """ - return get(key, missing=missing, deferred=deferred, - connection=self.connection, dataset_id=self.dataset_id) + entities = self.get_multi(keys=[key], missing=missing, + deferred=deferred) + if entities: + return entities[0] def get_multi(self, keys, missing=None, deferred=None): - """Proxy to :func:`gcloud.datastore.api.get_multi`. + """Retrieve entities, along with their attributes. - Passes our ``dataset_id``. + :type keys: list of :class:`gcloud.datastore.key.Key` + :param keys: The keys to be retrieved from the datastore. + + :type missing: an empty list or None. + :param missing: If a list is passed, the key-only entities returned + by the backend as "missing" will be copied into it. + Use only as a keyword param. + + :type deferred: an empty list or None. + :param deferred: If a list is passed, the keys returned + by the backend as "deferred" will be copied into it. + Use only as a keyword param. + + :rtype: list of :class:`gcloud.datastore.entity.Entity` + :returns: The requested entities. + :raises: ValueError if one or more of ``keys`` has a dataset ID which + does not match our dataset ID. """ - return get_multi(keys, missing=missing, deferred=deferred, - connection=self.connection, - dataset_id=self.dataset_id) + if not keys: + return [] + + ids = list(set([key.dataset_id for key in keys])) + if ids != [self.dataset_id]: + raise ValueError('Keys do not match dataset ID') + + transaction = Transaction.current() + + entity_pbs = _extended_lookup( + connection=self.connection, + dataset_id=self.dataset_id, + key_pbs=[k.to_protobuf() for k in keys], + missing=missing, + deferred=deferred, + transaction_id=transaction and transaction.id, + ) + + if missing is not None: + missing[:] = [ + helpers.entity_from_protobuf(missed_pb) + for missed_pb in missing] + + if deferred is not None: + deferred[:] = [ + helpers.key_from_protobuf(deferred_pb) + for deferred_pb in deferred] + + return [helpers.entity_from_protobuf(entity_pb) + for entity_pb in entity_pbs] def put(self, entity): - """Proxy to :func:`gcloud.datastore.api.put`. + """Save an entity in the Cloud Datastore. - Passes our ``dataset_id``. + .. note:: + + This is just a thin wrapper over :meth:`put_multi`. + The backend API does not make a distinction between a single + entity or multiple entities in a commit request. + + :type entity: :class:`gcloud.datastore.entity.Entity` + :param entity: The entity to be saved to the datastore. """ - return put(entity, connection=self.connection, - dataset_id=self.dataset_id) + self.put_multi(entities=[entity]) def put_multi(self, entities): - """Proxy to :func:`gcloud.datastore.api.put_multi`. + """Save entities in the Cloud Datastore. - Passes our ``dataset_id``. + :type entities: list of :class:`gcloud.datastore.entity.Entity` + :param entities: The entities to be saved to the datastore. + + :raises: ValueError if ``entities`` is a single entity. """ - return put_multi(entities, connection=self.connection, - dataset_id=self.dataset_id) + if isinstance(entities, Entity): + raise ValueError("Pass a sequence of entities") + + if not entities: + return + + current = Batch.current() + in_batch = current is not None + + if not in_batch: + current = Batch(dataset_id=self.dataset_id, + connection=self.connection) + for entity in entities: + current.put(entity) + + if not in_batch: + current.commit() def delete(self, key): - """Proxy to :func:`gcloud.datastore.api.delete`. + """Delete the key in the Cloud Datastore. - Passes our ``dataset_id``. + .. note:: + + This is just a thin wrapper over :meth:`delete_multi`. + The backend API does not make a distinction between a single key or + multiple keys in a commit request. + + :type key: :class:`gcloud.datastore.key.Key` + :param key: The key to be deleted from the datastore. """ - return delete(key, connection=self.connection, - dataset_id=self.dataset_id) + return self.delete_multi(keys=[key]) def delete_multi(self, keys): - """Proxy to :func:`gcloud.datastore.api.delete_multi`. + """Delete keys from the Cloud Datastore. - Passes our ``dataset_id``. + :type keys: list of :class:`gcloud.datastore.key.Key` + :param keys: The keys to be deleted from the datastore. + """ + if not keys: + return + + # We allow partial keys to attempt a delete, the backend will fail. + current = Batch.current() + in_batch = current is not None + + if not in_batch: + current = Batch(dataset_id=self.dataset_id, + connection=self.connection) + for key in keys: + current.delete(key) + + if not in_batch: + current.commit() + + def allocate_ids(self, incomplete_key, num_ids): + """Allocate a list of IDs from a partial key. + + :type incomplete_key: A :class:`gcloud.datastore.key.Key` + :param incomplete_key: Partial key to use as base for allocated IDs. + + :type num_ids: integer + :param num_ids: The number of IDs to allocate. + + :rtype: list of :class:`gcloud.datastore.key.Key` + :returns: The (complete) keys allocated with ``incomplete_key`` as + root. + :raises: :class:`ValueError` if ``incomplete_key`` is not a + partial key. """ - return delete_multi(keys, connection=self.connection, - dataset_id=self.dataset_id) + if not incomplete_key.is_partial: + raise ValueError(('Key is not partial.', incomplete_key)) + + incomplete_key_pb = incomplete_key.to_protobuf() + incomplete_key_pbs = [incomplete_key_pb] * num_ids + + conn = self.connection + allocated_key_pbs = conn.allocate_ids(incomplete_key.dataset_id, + incomplete_key_pbs) + allocated_ids = [allocated_key_pb.path_element[-1].id + for allocated_key_pb in allocated_key_pbs] + return [incomplete_key.completed_key(allocated_id) + for allocated_id in allocated_ids] def key(self, *path_args, **kwargs): """Proxy to :class:`gcloud.datastore.key.Key`. diff --git a/gcloud/datastore/test_api.py b/gcloud/datastore/test_api.py deleted file mode 100644 index 8e8a9a6d3407..000000000000 --- a/gcloud/datastore/test_api.py +++ /dev/null @@ -1,1096 +0,0 @@ -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest2 - - -def _make_entity_pb(dataset_id, kind, integer_id, name=None, str_val=None): - from gcloud.datastore import _datastore_v1_pb2 as datastore_pb - - entity_pb = datastore_pb.Entity() - entity_pb.key.partition_id.dataset_id = dataset_id - path_element = entity_pb.key.path_element.add() - path_element.kind = kind - path_element.id = integer_id - if name is not None and str_val is not None: - prop = entity_pb.property.add() - prop.name = name - prop.value.string_value = str_val - - return entity_pb - - -class Test__require_dataset_id(unittest2.TestCase): - - _MARKER = object() - - def _callFUT(self, passed=_MARKER, first_key=None): - from gcloud.datastore.api import _require_dataset_id - if passed is self._MARKER: - return _require_dataset_id(first_key=first_key) - return _require_dataset_id(dataset_id=passed, first_key=first_key) - - def _monkey(self, dataset_id): - from gcloud.datastore._testing import _monkey_defaults - return _monkey_defaults(dataset_id=dataset_id) - - def test_implicit_unset_wo_keys(self): - with self._monkey(None): - with self.assertRaises(EnvironmentError): - self._callFUT() - - def test_implicit_unset_w_keys(self): - from gcloud.datastore.test_batch import _Key - ID = 'DATASET' - with self._monkey(None): - self.assertEqual(self._callFUT(first_key=_Key(ID)), ID) - - def test_implicit_unset_w_existing_batch_wo_keys(self): - ID = 'DATASET' - with self._monkey(None): - with _NoCommitBatch(dataset_id=ID, connection=object()): - self.assertEqual(self._callFUT(), ID) - - def test_implicit_unset_w_existing_batch_w_keys(self): - from gcloud.datastore.test_batch import _Key - ID = 'DATASET' - OTHER = 'OTHER' - with self._monkey(None): - with _NoCommitBatch(dataset_id=ID, connection=object()): - self.assertEqual(self._callFUT(first_key=_Key(OTHER)), ID) - - def test_implicit_unset_w_existing_transaction_wo_keys(self): - ID = 'DATASET' - with self._monkey(None): - with _NoCommitTransaction(dataset_id=ID, connection=object()): - self.assertEqual(self._callFUT(), ID) - - def test_implicit_unset_w_existing_transaction_w_keys(self): - from gcloud.datastore.test_batch import _Key - ID = 'DATASET' - OTHER = 'OTHER' - with self._monkey(None): - with _NoCommitTransaction(dataset_id=ID, connection=object()): - self.assertEqual(self._callFUT(first_key=_Key(OTHER)), ID) - - def test_implicit_unset_passed_explicitly_wo_keys(self): - ID = 'DATASET' - with self._monkey(None): - self.assertEqual(self._callFUT(ID), ID) - - def test_implicit_unset_passed_explicitly_w_keys(self): - from gcloud.datastore.test_batch import _Key - ID = 'DATASET' - OTHER = 'OTHER' - with self._monkey(None): - self.assertEqual(self._callFUT(ID, first_key=_Key(OTHER)), ID) - - def test_id_implicit_set_wo_keys(self): - IMPLICIT_ID = 'IMPLICIT' - with self._monkey(IMPLICIT_ID): - stored_id = self._callFUT() - self.assertTrue(stored_id is IMPLICIT_ID) - - def test_id_implicit_set_w_keys(self): - from gcloud.datastore.test_batch import _Key - IMPLICIT_ID = 'IMPLICIT' - OTHER = 'OTHER' - with self._monkey(IMPLICIT_ID): - self.assertEqual(self._callFUT(first_key=_Key(OTHER)), OTHER) - - def test_id_implicit_set_passed_explicitly_wo_keys(self): - ID = 'DATASET' - IMPLICIT_ID = 'IMPLICIT' - with self._monkey(IMPLICIT_ID): - self.assertEqual(self._callFUT(ID), ID) - - def test_id_implicit_set_passed_explicitly_w_keys(self): - from gcloud.datastore.test_batch import _Key - ID = 'DATASET' - IMPLICIT_ID = 'IMPLICIT' - OTHER = 'OTHER' - with self._monkey(IMPLICIT_ID): - self.assertEqual(self._callFUT(ID, first_key=_Key(OTHER)), ID) - - -class Test__require_connection(unittest2.TestCase): - - _MARKER = object() - - def _callFUT(self, passed=_MARKER): - from gcloud.datastore.api import _require_connection - if passed is self._MARKER: - return _require_connection() - return _require_connection(passed) - - def _monkey(self, connection): - from gcloud.datastore._testing import _monkey_defaults - return _monkey_defaults(connection=connection) - - def test_implicit_unset(self): - with self._monkey(None): - with self.assertRaises(EnvironmentError): - self._callFUT() - - def test_implicit_unset_w_existing_batch(self): - ID = 'DATASET' - CONNECTION = object() - with self._monkey(None): - with _NoCommitBatch(dataset_id=ID, connection=CONNECTION): - self.assertEqual(self._callFUT(), CONNECTION) - - def test_implicit_unset_w_existing_transaction(self): - ID = 'DATASET' - CONNECTION = object() - with self._monkey(None): - with _NoCommitTransaction(dataset_id=ID, connection=CONNECTION): - self.assertEqual(self._callFUT(), CONNECTION) - - def test_implicit_unset_passed_explicitly(self): - CONNECTION = object() - with self._monkey(None): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - def test_implicit_set(self): - IMPLICIT_CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT() is IMPLICIT_CONNECTION) - - def test_implicit_set_passed_explicitly(self): - IMPLICIT_CONNECTION = object() - CONNECTION = object() - with self._monkey(IMPLICIT_CONNECTION): - self.assertTrue(self._callFUT(CONNECTION) is CONNECTION) - - -class Test_get_multi_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, keys, missing=None, deferred=None, - connection=None, dataset_id=None): - from gcloud.datastore.api import get_multi - return get_multi(keys, missing=missing, deferred=deferred, - connection=connection, dataset_id=dataset_id) - - def test_wo_connection(self): - from gcloud.datastore.key import Key - - DATASET_ID = 'DATASET' - key = Key('Kind', 1234, dataset_id=DATASET_ID) - self.assertRaises(EnvironmentError, - self._callFUT, [key], dataset_id=DATASET_ID) - - def test_no_keys(self): - results = self._callFUT([]) - self.assertEqual(results, []) - - def test_miss(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - connection = _Connection() - key = Key('Kind', 1234, dataset_id=DATASET_ID) - results = self._callFUT([key], connection=connection, - dataset_id=DATASET_ID) - self.assertEqual(results, []) - - def test_miss_wo_dataset_id(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - connection = _Connection() - key = Key('Kind', 1234, dataset_id=DATASET_ID) - results = self._callFUT([key], connection=connection) - self.assertEqual(results, []) - expected = { - 'dataset_id': DATASET_ID, - 'key_pbs': [key.to_protobuf()], - 'transaction_id': None, - 'eventual': False, - } - self.assertEqual(connection._called_with, expected) - - def test_miss_w_missing(self): - from gcloud.datastore import _datastore_v1_pb2 as datastore_pb - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - - # Make a missing entity pb to be returned from mock backend. - missed = datastore_pb.Entity() - missed.key.partition_id.dataset_id = DATASET_ID - path_element = missed.key.path_element.add() - path_element.kind = KIND - path_element.id = ID - - # Set missing entity on mock connection. - connection = _Connection() - connection._missing = [missed] - - key = Key(KIND, ID, dataset_id=DATASET_ID) - missing = [] - entities = self._callFUT([key], connection=connection, - missing=missing, dataset_id=DATASET_ID) - self.assertEqual(entities, []) - self.assertEqual([missed.key.to_protobuf() for missed in missing], - [key.to_protobuf()]) - - def test_w_missing_non_empty(self): - from gcloud.datastore.key import Key - - DATASET_ID = 'DATASET' - CONNECTION = object() - key = Key('Kind', 1234, dataset_id=DATASET_ID) - - missing = ['this', 'list', 'is', 'not', 'empty'] - self.assertRaises(ValueError, self._callFUT, - [key], connection=CONNECTION, - missing=missing) - - def test_w_deferred_non_empty(self): - from gcloud.datastore.key import Key - - DATASET_ID = 'DATASET' - CONNECTION = object() - key = Key('Kind', 1234, dataset_id=DATASET_ID) - - deferred = ['this', 'list', 'is', 'not', 'empty'] - self.assertRaises(ValueError, self._callFUT, - [key], connection=CONNECTION, - deferred=deferred) - - def test_miss_w_deferred(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - key = Key('Kind', 1234, dataset_id=DATASET_ID) - - # Set deferred entity on mock connection. - connection = _Connection() - connection._deferred = [key.to_protobuf()] - - deferred = [] - entities = self._callFUT([key], connection=connection, - deferred=deferred, dataset_id=DATASET_ID) - self.assertEqual(entities, []) - self.assertEqual([def_key.to_protobuf() for def_key in deferred], - [key.to_protobuf()]) - - def _verifyProtobufCall(self, called_with, URI, conn): - self.assertEqual(called_with['uri'], URI) - self.assertEqual(called_with['method'], 'POST') - self.assertEqual(called_with['headers']['Content-Type'], - 'application/x-protobuf') - self.assertEqual(called_with['headers']['User-Agent'], - conn.USER_AGENT) - - def test_w_deferred_from_backend_but_not_passed(self): - from gcloud.datastore import _datastore_v1_pb2 as datastore_pb - from gcloud.datastore.connection import Connection - from gcloud.datastore.key import Key - from gcloud.datastore import test_connection - - # Shortening name, import line above was too long. - cmp_key_after_req = test_connection._compare_key_pb_after_request - - DATASET_ID = 'DATASET' - key1 = Key('Kind', dataset_id=DATASET_ID) - key2 = Key('Kind', 2345, dataset_id=DATASET_ID) - key_pb1 = key1.to_protobuf() - key_pb2 = key2.to_protobuf() - - # Build mock first response. - rsp_pb1 = datastore_pb.LookupResponse() - entity1 = datastore_pb.Entity() - entity1.key.CopyFrom(key_pb1) - # Add the entity to the "found" part of the response. - rsp_pb1.found.add(entity=entity1) - # Add the second key to the "deferred" part of the response. - rsp_pb1.deferred.add().CopyFrom(key_pb2) - - # Build mock second response. - rsp_pb2 = datastore_pb.LookupResponse() - # Add in entity that was deferred. - entity2 = datastore_pb.Entity() - entity2.key.CopyFrom(key_pb2) - rsp_pb2.found.add(entity=entity2) - - conn = Connection() - # Add mock http object to connection with response from above. - http = conn._http = _HttpMultiple( - ({'status': '200'}, rsp_pb1.SerializeToString()), - ({'status': '200'}, rsp_pb2.SerializeToString()), - ) - - missing = [] - found = self._callFUT([key1, key2], missing=missing, connection=conn) - self.assertEqual(len(found), 2) - self.assertEqual(len(missing), 0) - - # Check the actual contents on the response. - self.assertEqual(found[0].key.path, key1.path) - self.assertEqual(found[0].key.dataset_id, key1.dataset_id) - self.assertEqual(found[1].key.path, key2.path) - self.assertEqual(found[1].key.dataset_id, key2.dataset_id) - - # Check that our http object was called correctly. - cw = http._called_with - rq_class = datastore_pb.LookupRequest - request = rq_class() - self.assertEqual(len(cw), 2) - - # Make URI to check for requests. - URI = '/'.join([ - conn.api_base_url, - 'datastore', - conn.API_VERSION, - 'datasets', - DATASET_ID, - 'lookup', - ]) - - # Make sure the first called with argument checks out. - self._verifyProtobufCall(cw[0], URI, conn) - request.ParseFromString(cw[0]['body']) - keys = list(request.key) - self.assertEqual(len(keys), 2) - cmp_key_after_req(self, key_pb1, keys[0]) - cmp_key_after_req(self, key_pb2, keys[1]) - - # Make sure the second called with argument checks out. - self._verifyProtobufCall(cw[1], URI, conn) - request.ParseFromString(cw[1]['body']) - keys = list(request.key) - self.assertEqual(len(keys), 1) - cmp_key_after_req(self, key_pb2, keys[0]) - - def test_hit(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - PATH = [{'kind': KIND, 'id': ID}] - - # Make a found entity pb to be returned from mock backend. - entity_pb = _make_entity_pb(DATASET_ID, KIND, ID, 'foo', 'Foo') - - # Make a connection to return the entity pb. - connection = _Connection(entity_pb) - - key = Key(KIND, ID, dataset_id=DATASET_ID) - result, = self._callFUT([key], connection=connection, - dataset_id=DATASET_ID) - new_key = result.key - - # Check the returned value is as expected. - self.assertFalse(new_key is key) - self.assertEqual(new_key.dataset_id, DATASET_ID) - self.assertEqual(new_key.path, PATH) - self.assertEqual(list(result), ['foo']) - self.assertEqual(result['foo'], 'Foo') - - def test_hit_multiple_keys_same_dataset(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID1 = 1234 - ID2 = 2345 - - # Make a found entity pb to be returned from mock backend. - entity_pb1 = _make_entity_pb(DATASET_ID, KIND, ID1) - entity_pb2 = _make_entity_pb(DATASET_ID, KIND, ID2) - - # Make a connection to return the entity pbs. - connection = _Connection(entity_pb1, entity_pb2) - - key1 = Key(KIND, ID1, dataset_id=DATASET_ID) - key2 = Key(KIND, ID2, dataset_id=DATASET_ID) - retrieved1, retrieved2 = self._callFUT( - [key1, key2], connection=connection, dataset_id=DATASET_ID) - - # Check values match. - self.assertEqual(retrieved1.key.path, key1.path) - self.assertEqual(dict(retrieved1), {}) - self.assertEqual(retrieved2.key.path, key2.path) - self.assertEqual(dict(retrieved2), {}) - - def test_hit_multiple_keys_different_dataset(self): - from gcloud.datastore.key import Key - - DATASET_ID1 = 'DATASET' - DATASET_ID2 = 'DATASET-ALT' - - # Make sure our IDs are actually different. - self.assertNotEqual(DATASET_ID1, DATASET_ID2) - - key1 = Key('KIND', 1234, dataset_id=DATASET_ID1) - key2 = Key('KIND', 1234, dataset_id=DATASET_ID2) - with self.assertRaises(ValueError): - self._callFUT([key1, key2], connection=object(), - dataset_id=DATASET_ID1) - - def test_implicit_wo_transaction(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - PATH = [{'kind': KIND, 'id': ID}] - - # Make a found entity pb to be returned from mock backend. - entity_pb = _make_entity_pb(DATASET_ID, KIND, ID, 'foo', 'Foo') - - # Make a connection to return the entity pb. - CUSTOM_CONNECTION = _Connection(entity_pb) - - key = Key(KIND, ID, dataset_id=DATASET_ID) - with _monkey_defaults(connection=CUSTOM_CONNECTION, - dataset_id=DATASET_ID): - result, = self._callFUT([key]) - - expected_called_with = { - 'dataset_id': DATASET_ID, - 'key_pbs': [key.to_protobuf()], - 'transaction_id': None, - 'eventual': False, - } - self.assertEqual(CUSTOM_CONNECTION._called_with, expected_called_with) - - new_key = result.key - # Check the returned value is as expected. - self.assertFalse(new_key is key) - self.assertEqual(new_key.dataset_id, DATASET_ID) - self.assertEqual(new_key.path, PATH) - self.assertEqual(list(result), ['foo']) - self.assertEqual(result['foo'], 'Foo') - - def test_w_transaction(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - PATH = [{'kind': KIND, 'id': ID}] - TRANSACTION = 'TRANSACTION' - - # Make a found entity pb to be returned from mock backend. - entity_pb = _make_entity_pb(DATASET_ID, KIND, ID, 'foo', 'Foo') - - # Make a connection to return the entity pb. - CUSTOM_CONNECTION = _Connection(entity_pb) - - key = Key(KIND, ID, dataset_id=DATASET_ID) - with _NoCommitTransaction(DATASET_ID, CUSTOM_CONNECTION, TRANSACTION): - result, = self._callFUT([key], connection=CUSTOM_CONNECTION, - dataset_id=DATASET_ID) - - expected_called_with = { - 'dataset_id': DATASET_ID, - 'key_pbs': [key.to_protobuf()], - 'transaction_id': TRANSACTION, - 'eventual': False, - } - self.assertEqual(CUSTOM_CONNECTION._called_with, expected_called_with) - - new_key = result.key - # Check the returned value is as expected. - self.assertFalse(new_key is key) - self.assertEqual(new_key.dataset_id, DATASET_ID) - self.assertEqual(new_key.path, PATH) - self.assertEqual(list(result), ['foo']) - self.assertEqual(result['foo'], 'Foo') - - def test_max_loops(self): - from gcloud._testing import _Monkey - from gcloud.datastore import api - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - - # Make a found entity pb to be returned from mock backend. - entity_pb = _make_entity_pb(DATASET_ID, KIND, ID, 'foo', 'Foo') - - # Make a connection to return the entity pb. - connection = _Connection(entity_pb) - - key = Key(KIND, ID, dataset_id=DATASET_ID) - deferred = [] - missing = [] - with _Monkey(api, _MAX_LOOPS=-1): - result = self._callFUT([key], missing=missing, deferred=deferred, - connection=connection, - dataset_id=DATASET_ID) - - # Make sure we have no results, even though the connection has been - # set up as in `test_hit` to return a single result. - self.assertEqual(result, []) - self.assertEqual(missing, []) - self.assertEqual(deferred, []) - - -class Test_get_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, key, missing=None, deferred=None, - connection=None, dataset_id=None): - from gcloud.datastore.api import get - return get(key, missing=missing, deferred=deferred, - connection=connection, dataset_id=dataset_id) - - def test_hit(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - KIND = 'Kind' - ID = 1234 - PATH = [{'kind': KIND, 'id': ID}] - - # Make a found entity pb to be returned from mock backend. - entity_pb = _make_entity_pb(DATASET_ID, KIND, ID, 'foo', 'Foo') - - # Make a connection to return the entity pb. - connection = _Connection(entity_pb) - - key = Key(KIND, ID, dataset_id=DATASET_ID) - result = self._callFUT(key, connection=connection, - dataset_id=DATASET_ID) - new_key = result.key - - # Check the returned value is as expected. - self.assertFalse(new_key is key) - self.assertEqual(new_key.dataset_id, DATASET_ID) - self.assertEqual(new_key.path, PATH) - self.assertEqual(list(result), ['foo']) - self.assertEqual(result['foo'], 'Foo') - - def test_miss(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - connection = _Connection() - key = Key('Kind', 1234, dataset_id=DATASET_ID) - result = self._callFUT(key, connection=connection, - dataset_id=DATASET_ID) - self.assertTrue(result is None) - - -class Test_put_multi_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, entities, connection=None, dataset_id=None): - from gcloud.datastore.api import put_multi - return put_multi(entities, connection=connection, - dataset_id=dataset_id) - - def test_no_connection(self): - from gcloud.datastore import _implicit_environ - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - entity = _Entity(foo=u'bar') - entity.key = _Key(_DATASET) - - self.assertEqual(_implicit_environ.get_default_connection(), None) - with self.assertRaises(EnvironmentError): - self._callFUT([entity], dataset_id=_DATASET) - - def test_no_dataset_id(self): - from gcloud.datastore import _implicit_environ - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - entity = _Entity(foo=u'bar') - entity.key = _Key(_DATASET) - - self.assertEqual(_implicit_environ.get_default_connection(), None) - result = self._callFUT([entity], connection=connection) - - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - upserts = list(mutation.upsert) - self.assertEqual(len(upserts), 1) - self.assertEqual(upserts[0].key, entity.key.to_protobuf()) - properties = list(upserts[0].property) - self.assertEqual(properties[0].name, 'foo') - self.assertEqual(properties[0].value.string_value, u'bar') - self.assertTrue(transaction_id is None) - - def test_no_entities(self): - from gcloud.datastore import _implicit_environ - - self.assertEqual(_implicit_environ.get_default_connection(), None) - result = self._callFUT([]) - self.assertEqual(result, None) - - def test_w_single_empty_entity(self): - # https://github.com/GoogleCloudPlatform/gcloud-python/issues/649 - from gcloud.datastore.entity import Entity - self.assertRaises(ValueError, self._callFUT, Entity()) - - def test_no_batch_w_partial_key(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - entity = _Entity(foo=u'bar') - key = entity.key = _Key(_DATASET) - key._id = None - - result = self._callFUT([entity], connection=connection, - dataset_id=_DATASET) - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - inserts = list(mutation.insert_auto_id) - self.assertEqual(len(inserts), 1) - self.assertEqual(inserts[0].key, key.to_protobuf()) - properties = list(inserts[0].property) - self.assertEqual(properties[0].name, 'foo') - self.assertEqual(properties[0].value.string_value, u'bar') - self.assertTrue(transaction_id is None) - - def test_existing_batch_w_completed_key(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - entity = _Entity(foo=u'bar') - key = entity.key = _Key(_DATASET) - - # Set up Batch on stack so we can check it is used. - with _NoCommitBatch(_DATASET, connection) as CURR_BATCH: - result = self._callFUT([entity], connection=connection) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - upserts = list(CURR_BATCH.mutation.upsert) - self.assertEqual(len(upserts), 1) - self.assertEqual(upserts[0].key, key.to_protobuf()) - properties = list(upserts[0].property) - self.assertEqual(properties[0].name, 'foo') - self.assertEqual(properties[0].value.string_value, u'bar') - self.assertEqual(len(CURR_BATCH.mutation.delete), 0) - - def test_implicit_connection(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - entity = _Entity(foo=u'bar') - key = entity.key = _Key(_DATASET) - - with _monkey_defaults(connection=connection): - # Set up Batch on stack so we can check it is used. - with _NoCommitBatch(_DATASET, connection) as CURR_BATCH: - result = self._callFUT([entity]) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - self.assertEqual(len(CURR_BATCH.mutation.upsert), 1) - upserts = list(CURR_BATCH.mutation.upsert) - self.assertEqual(len(upserts), 1) - self.assertEqual(upserts[0].key, key.to_protobuf()) - properties = list(upserts[0].property) - self.assertEqual(properties[0].name, 'foo') - self.assertEqual(properties[0].value.string_value, u'bar') - self.assertEqual(len(CURR_BATCH.mutation.delete), 0) - - -class Test_put_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, entity, connection=None, dataset_id=None): - from gcloud.datastore.api import put - return put(entity, connection=connection, dataset_id=dataset_id) - - def test_implicit_connection(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Entity - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - entity = _Entity(foo=u'bar') - key = entity.key = _Key(_DATASET) - - with _monkey_defaults(connection=connection): - # Set up Batch on stack so we can check it is used. - with _NoCommitBatch(_DATASET, connection) as CURR_BATCH: - result = self._callFUT(entity) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - self.assertEqual(len(CURR_BATCH.mutation.upsert), 1) - upserts = list(CURR_BATCH.mutation.upsert) - self.assertEqual(len(upserts), 1) - self.assertEqual(upserts[0].key, key.to_protobuf()) - properties = list(upserts[0].property) - self.assertEqual(properties[0].name, 'foo') - self.assertEqual(properties[0].value.string_value, u'bar') - self.assertEqual(len(CURR_BATCH.mutation.delete), 0) - - -class Test_delete_multi_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, keys, connection=None, dataset_id=None): - from gcloud.datastore.api import delete_multi - return delete_multi(keys, connection=connection, dataset_id=dataset_id) - - def test_no_connection(self): - from gcloud.datastore import _implicit_environ - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - key = _Key(_DATASET) - - self.assertEqual(_implicit_environ.get_default_connection(), None) - with self.assertRaises(EnvironmentError): - self._callFUT([key], dataset_id=_DATASET) - - def test_no_dataset_id(self): - from gcloud.datastore import _implicit_environ - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - self.assertEqual(_implicit_environ.get_default_connection(), None) - - result = self._callFUT([key], connection=connection) - - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - self.assertEqual(list(mutation.delete), [key.to_protobuf()]) - self.assertTrue(transaction_id is None) - - def test_no_keys(self): - from gcloud.datastore import _implicit_environ - - self.assertEqual(_implicit_environ.get_default_connection(), None) - result = self._callFUT([]) - self.assertEqual(result, None) - - def test_no_batch(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - result = self._callFUT([key], connection=connection, - dataset_id=_DATASET) - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - self.assertEqual(list(mutation.delete), [key.to_protobuf()]) - self.assertTrue(transaction_id is None) - - def test_wo_batch_w_key_different_than_default_dataset_id(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DEFAULT_DATASET = 'DEFAULT' - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - with _monkey_defaults(connection=connection, - dataset_id=_DEFAULT_DATASET): - result = self._callFUT([key]) - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - self.assertEqual(list(mutation.delete), [key.to_protobuf()]) - self.assertTrue(transaction_id is None) - - def test_w_existing_batch(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - # Set up Batch on stack so we can check it is used. - with _NoCommitBatch(_DATASET, connection) as CURR_BATCH: - result = self._callFUT([key]) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - self.assertEqual(len(CURR_BATCH.mutation.upsert), 0) - deletes = list(CURR_BATCH.mutation.delete) - self.assertEqual(len(deletes), 1) - self.assertEqual(deletes[0], key._key) - self.assertEqual(len(connection._committed), 0) - - def test_w_existing_transaction(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - # Set up Batch on stack so we can check it is used. - with _NoCommitTransaction(_DATASET, connection) as CURR_BATCH: - result = self._callFUT([key]) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - self.assertEqual(len(CURR_BATCH.mutation.upsert), 0) - deletes = list(CURR_BATCH.mutation.delete) - self.assertEqual(len(deletes), 1) - self.assertEqual(deletes[0], key._key) - self.assertEqual(len(connection._committed), 0) - - def test_implicit_connection_and_dataset_id(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - with _monkey_defaults(connection=connection, dataset_id=_DATASET): - # Set up Batch on stack so we can check it is used. - with _NoCommitBatch(_DATASET, connection) as CURR_BATCH: - result = self._callFUT([key]) - - self.assertEqual(result, None) - self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) - self.assertEqual(len(CURR_BATCH.mutation.upsert), 0) - deletes = list(CURR_BATCH.mutation.delete) - self.assertEqual(len(deletes), 1) - self.assertEqual(deletes[0], key._key) - self.assertEqual(len(connection._committed), 0) - - -class Test_delete_function(unittest2.TestCase): - - def setUp(self): - from gcloud.datastore._testing import _setup_defaults - _setup_defaults(self) - - def tearDown(self): - from gcloud.datastore._testing import _tear_down_defaults - _tear_down_defaults(self) - - def _callFUT(self, key, connection=None, dataset_id=None): - from gcloud.datastore.api import delete - return delete(key, connection=connection, dataset_id=dataset_id) - - def test_no_batch(self): - from gcloud.datastore.test_batch import _Connection - from gcloud.datastore.test_batch import _Key - - # Build basic mocks needed to delete. - _DATASET = 'DATASET' - connection = _Connection() - key = _Key(_DATASET) - - result = self._callFUT(key, connection=connection, - dataset_id=_DATASET) - self.assertEqual(result, None) - self.assertEqual(len(connection._committed), 1) - dataset_id, mutation, transaction_id = connection._committed[0] - self.assertEqual(dataset_id, _DATASET) - self.assertEqual(list(mutation.delete), [key.to_protobuf()]) - self.assertTrue(transaction_id is None) - - -class Test_allocate_ids_function(unittest2.TestCase): - - def _callFUT(self, incomplete_key, num_ids, connection=None): - from gcloud.datastore.api import allocate_ids - return allocate_ids(incomplete_key, num_ids, connection=connection) - - def test_w_explicit_connection(self): - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - DATASET_ID = 'DATASET' - INCOMPLETE_KEY = Key('KIND', dataset_id=DATASET_ID) - CONNECTION = _Connection() - NUM_IDS = 2 - result = self._callFUT(INCOMPLETE_KEY, NUM_IDS, connection=CONNECTION) - - # Check the IDs returned match. - self.assertEqual([key.id for key in result], list(range(NUM_IDS))) - - # Check connection is called correctly. - self.assertEqual(CONNECTION._called_dataset_id, DATASET_ID) - self.assertEqual(len(CONNECTION._called_key_pbs), NUM_IDS) - - def test_w_implicit_connection(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - CUSTOM_CONNECTION = _Connection() - NUM_IDS = 2 - with _monkey_defaults(connection=CUSTOM_CONNECTION, - dataset_id='DATASET'): - INCOMPLETE_KEY = Key('KIND') - result = self._callFUT(INCOMPLETE_KEY, NUM_IDS) - - # Check the IDs returned. - self.assertEqual([key.id for key in result], list(range(NUM_IDS))) - - def test_with_already_completed_key(self): - from gcloud.datastore._testing import _monkey_defaults - from gcloud.datastore.key import Key - from gcloud.datastore.test_connection import _Connection - - CUSTOM_CONNECTION = _Connection() - with _monkey_defaults(connection=CUSTOM_CONNECTION, - dataset_id='DATASET'): - COMPLETE_KEY = Key('KIND', 1234) - self.assertRaises(ValueError, self._callFUT, - COMPLETE_KEY, 2) - - -class _NoCommitBatch(object): - - def __init__(self, dataset_id, connection): - from gcloud.datastore.batch import Batch - self._batch = Batch(dataset_id, connection) - - def __enter__(self): - from gcloud.datastore.batch import _BATCHES - _BATCHES.push(self._batch) - return self._batch - - def __exit__(self, *args): - from gcloud.datastore.batch import _BATCHES - _BATCHES.pop() - - -class _NoCommitTransaction(object): - - def __init__(self, dataset_id, connection, transaction_id='TRANSACTION'): - from gcloud.datastore.transaction import Transaction - xact = self._transaction = Transaction(dataset_id, connection) - xact._id = transaction_id - - def __enter__(self): - from gcloud.datastore.batch import _BATCHES - _BATCHES.push(self._transaction) - return self._transaction - - def __exit__(self, *args): - from gcloud.datastore.batch import _BATCHES - _BATCHES.pop() - - -class _HttpMultiple(object): - - def __init__(self, *responses): - self._called_with = [] - self._responses = list(responses) - - def request(self, **kw): - self._called_with.append(kw) - result, self._responses = self._responses[0], self._responses[1:] - return result diff --git a/gcloud/datastore/test_batch.py b/gcloud/datastore/test_batch.py index 492cf14e8755..c00c58156f23 100644 --- a/gcloud/datastore/test_batch.py +++ b/gcloud/datastore/test_batch.py @@ -279,8 +279,8 @@ def test_commit_w_auto_id_entities(self): self.assertEqual(connection._committed, [(_DATASET, batch.mutation, None)]) - self.assertFalse(key.is_partial) - self.assertEqual(key._id, _NEW_ID) + self.assertFalse(entity.key.is_partial) + self.assertEqual(entity.key._id, _NEW_ID) def test_as_context_mgr_wo_error(self): from gcloud.datastore.batch import _BATCHES @@ -451,4 +451,6 @@ def to_protobuf(self): def completed_key(self, new_id): assert self.is_partial - self._id = new_id + new_key = self.__class__(self.dataset_id) + new_key._id = new_id + return new_key diff --git a/gcloud/datastore/test_client.py b/gcloud/datastore/test_client.py index 1405a791ccf7..96334c805fef 100644 --- a/gcloud/datastore/test_client.py +++ b/gcloud/datastore/test_client.py @@ -15,6 +15,22 @@ import unittest2 +def _make_entity_pb(dataset_id, kind, integer_id, name=None, str_val=None): + from gcloud.datastore import _datastore_v1_pb2 as datastore_pb + + entity_pb = datastore_pb.Entity() + entity_pb.key.partition_id.dataset_id = dataset_id + path_element = entity_pb.key.path_element.add() + path_element.kind = kind + path_element.id = integer_id + if name is not None and str_val is not None: + prop = entity_pb.property.add() + prop.name = name + prop.value.string_value = str_val + + return entity_pb + + class TestClient(unittest2.TestCase): DATASET_ID = 'DATASET' @@ -36,6 +52,7 @@ def test_ctor_w_dataset_id_no_environ(self): def test_ctor_w_implicit_inputs(self): from gcloud._testing import _Monkey from gcloud.datastore import client as _MUT + OTHER = 'other' conn = object() klass = self._getTargetClass() @@ -58,241 +75,485 @@ def test_ctor_w_explicit_inputs(self): self.assertEqual(client.namespace, NAMESPACE) self.assertTrue(client.connection is conn) - def test_get_defaults(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey - - _called_with = [] - - def _get(*args, **kw): - _called_with.append((args, kw)) - - client = self._makeOne() - key = object() - - with _Monkey(MUT, get=_get): - client.get(key) - - self.assertEqual(_called_with[0][0], (key,)) - self.assertTrue(_called_with[0][1]['missing'] is None) - self.assertTrue(_called_with[0][1]['deferred'] is None) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_get_explicit(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey - - _called_with = [] - - def _get(*args, **kw): - _called_with.append((args, kw)) - - conn = object() - client = self._makeOne(connection=conn) - key, missing, deferred = object(), [], [] - - with _Monkey(MUT, get=_get): - client.get(key, missing, deferred) - - self.assertEqual(_called_with[0][0], (key,)) - self.assertTrue(_called_with[0][1]['missing'] is missing) - self.assertTrue(_called_with[0][1]['deferred'] is deferred) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_get_multi_defaults(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey - + def test_get_miss(self): _called_with = [] def _get_multi(*args, **kw): _called_with.append((args, kw)) + return [] + + connection = object() + client = self._makeOne(connection=connection) + client.get_multi = _get_multi - client = self._makeOne() key = object() - with _Monkey(MUT, get_multi=_get_multi): - client.get_multi([key]) + self.assertTrue(client.get(key) is None) - self.assertEqual(_called_with[0][0], ([key],)) + self.assertEqual(_called_with[0][0], ()) + self.assertEqual(_called_with[0][1]['keys'], [key]) self.assertTrue(_called_with[0][1]['missing'] is None) self.assertTrue(_called_with[0][1]['deferred'] is None) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_get_multi_explicit(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + def test_get_hit(self): _called_with = [] + _entity = object() def _get_multi(*args, **kw): _called_with.append((args, kw)) + return [_entity] + + connection = object() + client = self._makeOne(connection=connection) + client.get_multi = _get_multi - conn = object() - client = self._makeOne(connection=conn) key, missing, deferred = object(), [], [] - with _Monkey(MUT, get_multi=_get_multi): - client.get_multi([key], missing, deferred) + self.assertTrue(client.get(key, missing, deferred) is _entity) - self.assertEqual(_called_with[0][0], ([key],)) + self.assertEqual(_called_with[0][0], ()) + self.assertEqual(_called_with[0][1]['keys'], [key]) self.assertTrue(_called_with[0][1]['missing'] is missing) self.assertTrue(_called_with[0][1]['deferred'] is deferred) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_put_wo_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey - _called_with = [] - - def _put(*args, **kw): - _called_with.append((args, kw)) - - client = self._makeOne() - entity = object() + def test_get_multi_no_keys(self): + connection = object() + client = self._makeOne(connection=connection) + results = client.get_multi([]) + self.assertEqual(results, []) - with _Monkey(MUT, put=_put): - client.put(entity) + def test_get_multi_miss(self): + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection - self.assertEqual(_called_with[0][0], (entity,)) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + connection = _Connection() + client = self._makeOne(connection=connection) + key = Key('Kind', 1234, dataset_id=self.DATASET_ID) + results = client.get_multi([key]) + self.assertEqual(results, []) - def test_put_w_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + def test_get_multi_miss_w_missing(self): + from gcloud.datastore import _datastore_v1_pb2 as datastore_pb + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection - _called_with = [] + KIND = 'Kind' + ID = 1234 - def _put(*args, **kw): - _called_with.append((args, kw)) + # Make a missing entity pb to be returned from mock backend. + missed = datastore_pb.Entity() + missed.key.partition_id.dataset_id = self.DATASET_ID + path_element = missed.key.path_element.add() + path_element.kind = KIND + path_element.id = ID + + # Set missing entity on mock connection. + connection = _Connection() + connection._missing = [missed] + client = self._makeOne(connection=connection) + + key = Key(KIND, ID, dataset_id=self.DATASET_ID) + missing = [] + entities = client.get_multi([key], missing=missing) + self.assertEqual(entities, []) + self.assertEqual([missed.key.to_protobuf() for missed in missing], + [key.to_protobuf()]) + + def test_get_multi_w_missing_non_empty(self): + from gcloud.datastore.key import Key + + CONNECTION = object() + client = self._makeOne(connection=CONNECTION) + key = Key('Kind', 1234, dataset_id=self.DATASET_ID) + + missing = ['this', 'list', 'is', 'not', 'empty'] + self.assertRaises(ValueError, client.get_multi, + [key], missing=missing) + + def test_get_multi_w_deferred_non_empty(self): + from gcloud.datastore.key import Key + + CONNECTION = object() + client = self._makeOne(connection=CONNECTION) + key = Key('Kind', 1234, dataset_id=self.DATASET_ID) + + deferred = ['this', 'list', 'is', 'not', 'empty'] + self.assertRaises(ValueError, client.get_multi, + [key], deferred=deferred) + + def test_get_multi_miss_w_deferred(self): + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection + + key = Key('Kind', 1234, dataset_id=self.DATASET_ID) + + # Set deferred entity on mock connection. + connection = _Connection() + connection._deferred = [key.to_protobuf()] + client = self._makeOne(connection=connection) + + deferred = [] + entities = client.get_multi([key], deferred=deferred) + self.assertEqual(entities, []) + self.assertEqual([def_key.to_protobuf() for def_key in deferred], + [key.to_protobuf()]) + + def _verifyProtobufCall(self, called_with, URI, conn): + self.assertEqual(called_with['uri'], URI) + self.assertEqual(called_with['method'], 'POST') + self.assertEqual(called_with['headers']['Content-Type'], + 'application/x-protobuf') + self.assertEqual(called_with['headers']['User-Agent'], + conn.USER_AGENT) + + def test_get_multi_w_deferred_from_backend_but_not_passed(self): + from gcloud.datastore import _datastore_v1_pb2 as datastore_pb + from gcloud.datastore.connection import Connection + from gcloud.datastore.key import Key + from gcloud.datastore import test_connection + + # Shortening name, import line above was too long. + cmp_key_after_req = test_connection._compare_key_pb_after_request + + key1 = Key('Kind', dataset_id=self.DATASET_ID) + key2 = Key('Kind', 2345, dataset_id=self.DATASET_ID) + key_pb1 = key1.to_protobuf() + key_pb2 = key2.to_protobuf() + + # Build mock first response. + rsp_pb1 = datastore_pb.LookupResponse() + entity1 = datastore_pb.Entity() + entity1.key.CopyFrom(key_pb1) + # Add the entity to the "found" part of the response. + rsp_pb1.found.add(entity=entity1) + # Add the second key to the "deferred" part of the response. + rsp_pb1.deferred.add().CopyFrom(key_pb2) + + # Build mock second response. + rsp_pb2 = datastore_pb.LookupResponse() + # Add in entity that was deferred. + entity2 = datastore_pb.Entity() + entity2.key.CopyFrom(key_pb2) + rsp_pb2.found.add(entity=entity2) + + connection = Connection() + client = self._makeOne(connection=connection) + # Add mock http object to connection with response from above. + http = connection._http = _HttpMultiple( + ({'status': '200'}, rsp_pb1.SerializeToString()), + ({'status': '200'}, rsp_pb2.SerializeToString()), + ) + + missing = [] + found = client.get_multi([key1, key2], missing=missing) + self.assertEqual(len(found), 2) + self.assertEqual(len(missing), 0) + + # Check the actual contents on the response. + self.assertEqual(found[0].key.path, key1.path) + self.assertEqual(found[0].key.dataset_id, key1.dataset_id) + self.assertEqual(found[1].key.path, key2.path) + self.assertEqual(found[1].key.dataset_id, key2.dataset_id) + + # Check that our http object was called correctly. + cw = http._called_with + rq_class = datastore_pb.LookupRequest + request = rq_class() + self.assertEqual(len(cw), 2) + + # Make URI to check for requests. + URI = '/'.join([ + connection.api_base_url, + 'datastore', + connection.API_VERSION, + 'datasets', + self.DATASET_ID, + 'lookup', + ]) + + # Make sure the first called with argument checks out. + self._verifyProtobufCall(cw[0], URI, connection) + request.ParseFromString(cw[0]['body']) + keys = list(request.key) + self.assertEqual(len(keys), 2) + cmp_key_after_req(self, key_pb1, keys[0]) + cmp_key_after_req(self, key_pb2, keys[1]) + + # Make sure the second called with argument checks out. + self._verifyProtobufCall(cw[1], URI, connection) + request.ParseFromString(cw[1]['body']) + keys = list(request.key) + self.assertEqual(len(keys), 1) + cmp_key_after_req(self, key_pb2, keys[0]) + + def test_get_multi_hit(self): + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection + + KIND = 'Kind' + ID = 1234 + PATH = [{'kind': KIND, 'id': ID}] - entity, conn = object(), object() - client = self._makeOne(connection=conn) + # Make a found entity pb to be returned from mock backend. + entity_pb = _make_entity_pb(self.DATASET_ID, KIND, ID, 'foo', 'Foo') - with _Monkey(MUT, put=_put): - client.put(entity) + # Make a connection to return the entity pb. + connection = _Connection(entity_pb) + client = self._makeOne(connection=connection) - self.assertEqual(_called_with[0][0], (entity,)) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + key = Key(KIND, ID, dataset_id=self.DATASET_ID) + result, = client.get_multi([key]) + new_key = result.key - def test_put_multi_wo_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + # Check the returned value is as expected. + self.assertFalse(new_key is key) + self.assertEqual(new_key.dataset_id, self.DATASET_ID) + self.assertEqual(new_key.path, PATH) + self.assertEqual(list(result), ['foo']) + self.assertEqual(result['foo'], 'Foo') - _called_with = [] + def test_get_multi_hit_multiple_keys_same_dataset(self): + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection - def _put_multi(*args, **kw): - _called_with.append((args, kw)) + KIND = 'Kind' + ID1 = 1234 + ID2 = 2345 - client = self._makeOne() - entity = object() + # Make a found entity pb to be returned from mock backend. + entity_pb1 = _make_entity_pb(self.DATASET_ID, KIND, ID1) + entity_pb2 = _make_entity_pb(self.DATASET_ID, KIND, ID2) - with _Monkey(MUT, put_multi=_put_multi): - client.put_multi([entity]) + # Make a connection to return the entity pbs. + connection = _Connection(entity_pb1, entity_pb2) + client = self._makeOne(connection=connection) - self.assertEqual(_called_with[0][0], ([entity],)) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + key1 = Key(KIND, ID1, dataset_id=self.DATASET_ID) + key2 = Key(KIND, ID2, dataset_id=self.DATASET_ID) + retrieved1, retrieved2 = client.get_multi([key1, key2]) - def test_put_multi_w_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + # Check values match. + self.assertEqual(retrieved1.key.path, key1.path) + self.assertEqual(dict(retrieved1), {}) + self.assertEqual(retrieved2.key.path, key2.path) + self.assertEqual(dict(retrieved2), {}) - _called_with = [] + def test_get_multi_hit_multiple_keys_different_dataset(self): + from gcloud.datastore.key import Key - def _put_multi(*args, **kw): - _called_with.append((args, kw)) + DATASET_ID1 = 'DATASET' + DATASET_ID2 = 'DATASET-ALT' - entity, conn = object(), object() - client = self._makeOne(connection=conn) + # Make sure our IDs are actually different. + self.assertNotEqual(DATASET_ID1, DATASET_ID2) - with _Monkey(MUT, put_multi=_put_multi): - client.put_multi([entity]) + key1 = Key('KIND', 1234, dataset_id=DATASET_ID1) + key2 = Key('KIND', 1234, dataset_id=DATASET_ID2) + client = self._makeOne(connection=object()) - self.assertEqual(_called_with[0][0], ([entity],)) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + with self.assertRaises(ValueError): + client.get_multi([key1, key2]) - def test_delete_wo_connection(self): - from gcloud.datastore import client as MUT + def test_get_multi_max_loops(self): from gcloud._testing import _Monkey + from gcloud.datastore import client as _MUT + from gcloud.datastore.key import Key + from gcloud.datastore.test_connection import _Connection - _called_with = [] - - def _delete(*args, **kw): - _called_with.append((args, kw)) + KIND = 'Kind' + ID = 1234 - client = self._makeOne() - key = object() + # Make a found entity pb to be returned from mock backend. + entity_pb = _make_entity_pb(self.DATASET_ID, KIND, ID, 'foo', 'Foo') - with _Monkey(MUT, delete=_delete): - client.delete(key) + # Make a connection to return the entity pb. + connection = _Connection(entity_pb) + client = self._makeOne(connection=connection) - self.assertEqual(_called_with[0][0], (key,)) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + key = Key(KIND, ID, dataset_id=self.DATASET_ID) + deferred = [] + missing = [] + with _Monkey(_MUT, _MAX_LOOPS=-1): + result = client.get_multi([key], missing=missing, + deferred=deferred) - def test_delete_w_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + # Make sure we have no results, even though the connection has been + # set up as in `test_hit` to return a single result. + self.assertEqual(result, []) + self.assertEqual(missing, []) + self.assertEqual(deferred, []) + def test_put(self): _called_with = [] - def _delete(*args, **kw): + def _put_multi(*args, **kw): _called_with.append((args, kw)) - key, conn = object(), object() - client = self._makeOne(connection=conn) - with _Monkey(MUT, delete=_delete): - client.delete(key) - - self.assertEqual(_called_with[0][0], (key,)) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_delete_multi_wo_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey + client = self._makeOne() + client.put_multi = _put_multi + entity = object() + client.put(entity) + + self.assertEqual(_called_with[0][0], ()) + self.assertEqual(_called_with[0][1]['entities'], [entity]) + + def test_put_multi_no_entities(self): + client = self._makeOne(connection=object()) + self.assertEqual(client.put_multi([]), None) + + def test_put_multi_w_single_empty_entity(self): + # https://github.com/GoogleCloudPlatform/gcloud-python/issues/649 + from gcloud.datastore.entity import Entity + + client = self._makeOne(connection=object()) + self.assertRaises(ValueError, client.put_multi, Entity()) + + def test_put_multi_no_batch_w_partial_key(self): + from gcloud.datastore.test_batch import _Connection + from gcloud.datastore.test_batch import _Entity + from gcloud.datastore.test_batch import _Key + + connection = _Connection() + client = self._makeOne(connection=connection) + entity = _Entity(foo=u'bar') + key = entity.key = _Key(self.DATASET_ID) + key._id = None + + result = client.put_multi([entity]) + self.assertTrue(result is None) + + self.assertEqual(len(connection._committed), 1) + dataset_id, mutation, transaction_id = connection._committed[0] + self.assertEqual(dataset_id, self.DATASET_ID) + inserts = list(mutation.insert_auto_id) + self.assertEqual(len(inserts), 1) + self.assertEqual(inserts[0].key, key.to_protobuf()) + properties = list(inserts[0].property) + self.assertEqual(properties[0].name, 'foo') + self.assertEqual(properties[0].value.string_value, u'bar') + self.assertTrue(transaction_id is None) + + def test_put_multi_existing_batch_w_completed_key(self): + from gcloud.datastore.test_batch import _Connection + from gcloud.datastore.test_batch import _Entity + from gcloud.datastore.test_batch import _Key + + connection = _Connection() + client = self._makeOne(connection=connection) + entity = _Entity(foo=u'bar') + key = entity.key = _Key(self.DATASET_ID) + + with _NoCommitBatch(self.DATASET_ID, connection) as CURR_BATCH: + result = client.put_multi([entity]) + + self.assertEqual(result, None) + self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) + upserts = list(CURR_BATCH.mutation.upsert) + self.assertEqual(len(upserts), 1) + self.assertEqual(upserts[0].key, key.to_protobuf()) + properties = list(upserts[0].property) + self.assertEqual(properties[0].name, 'foo') + self.assertEqual(properties[0].value.string_value, u'bar') + self.assertEqual(len(CURR_BATCH.mutation.delete), 0) + + def test_delete(self): _called_with = [] def _delete_multi(*args, **kw): _called_with.append((args, kw)) client = self._makeOne() + client.delete_multi = _delete_multi key = object() - with _Monkey(MUT, delete_multi=_delete_multi): - client.delete_multi([key]) - - self.assertEqual(_called_with[0][0], ([key],)) - self.assertTrue(_called_with[0][1]['connection'] is self.CONNECTION) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) - - def test_delete_multi_w_connection(self): - from gcloud.datastore import client as MUT - from gcloud._testing import _Monkey - - _called_with = [] - - def _delete_multi(*args, **kw): - _called_with.append((args, kw)) - - key, conn = object(), object() - client = self._makeOne(connection=conn) - with _Monkey(MUT, delete_multi=_delete_multi): - client.delete_multi([key]) - - self.assertEqual(_called_with[0][0], ([key],)) - self.assertTrue(_called_with[0][1]['connection'] is conn) - self.assertEqual(_called_with[0][1]['dataset_id'], self.DATASET_ID) + client.delete(key) + + self.assertEqual(_called_with[0][0], ()) + self.assertEqual(_called_with[0][1]['keys'], [key]) + + def test_delete_multi_no_keys(self): + client = self._makeOne(connection=object()) + result = client.delete_multi([]) + self.assertEqual(result, None) + + def test_delete_multi_no_batch(self): + from gcloud.datastore.test_batch import _Connection + from gcloud.datastore.test_batch import _Key + + connection = _Connection() + client = self._makeOne(connection=connection) + key = _Key(self.DATASET_ID) + + result = client.delete_multi([key]) + self.assertEqual(result, None) + self.assertEqual(len(connection._committed), 1) + dataset_id, mutation, transaction_id = connection._committed[0] + self.assertEqual(dataset_id, self.DATASET_ID) + self.assertEqual(list(mutation.delete), [key.to_protobuf()]) + self.assertTrue(transaction_id is None) + + def test_delete_multi_w_existing_batch(self): + from gcloud.datastore.test_batch import _Connection + from gcloud.datastore.test_batch import _Key + + connection = _Connection() + client = self._makeOne(connection=connection) + key = _Key(self.DATASET_ID) + + with _NoCommitBatch(self.DATASET_ID, connection) as CURR_BATCH: + result = client.delete_multi([key]) + + self.assertEqual(result, None) + self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) + self.assertEqual(len(CURR_BATCH.mutation.upsert), 0) + deletes = list(CURR_BATCH.mutation.delete) + self.assertEqual(len(deletes), 1) + self.assertEqual(deletes[0], key._key) + self.assertEqual(len(connection._committed), 0) + + def test_delete_multi_w_existing_transaction(self): + from gcloud.datastore.test_batch import _Connection + from gcloud.datastore.test_batch import _Key + + connection = _Connection() + client = self._makeOne(connection=connection) + key = _Key(self.DATASET_ID) + + with _NoCommitTransaction(self.DATASET_ID, connection) as CURR_BATCH: + result = client.delete_multi([key]) + + self.assertEqual(result, None) + self.assertEqual(len(CURR_BATCH.mutation.insert_auto_id), 0) + self.assertEqual(len(CURR_BATCH.mutation.upsert), 0) + deletes = list(CURR_BATCH.mutation.delete) + self.assertEqual(len(deletes), 1) + self.assertEqual(deletes[0], key._key) + self.assertEqual(len(connection._committed), 0) + + def test_allocate_ids_w_partial_key(self): + from gcloud.datastore.test_batch import _Key + from gcloud.datastore.test_connection import _Connection + + connection = _Connection() + client = self._makeOne(connection=connection) + NUM_IDS = 2 + + INCOMPLETE_KEY = _Key(self.DATASET_ID) + INCOMPLETE_KEY._id = None + result = client.allocate_ids(INCOMPLETE_KEY, NUM_IDS) + + # Check the IDs returned. + self.assertEqual([key._id for key in result], list(range(NUM_IDS))) + + def test_allocate_ids_with_completed_key(self): + from gcloud.datastore.test_batch import _Key + from gcloud.datastore.test_connection import _Connection + + connection = _Connection() + client = self._makeOne(connection=connection) + COMPLETE_KEY = _Key(self.DATASET_ID) + self.assertRaises(ValueError, client.allocate_ids, COMPLETE_KEY, 2) def test_key_w_dataset_id(self): KIND = 'KIND' @@ -304,6 +565,7 @@ def test_key_w_dataset_id(self): def test_key_wo_dataset_id(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + KIND = 'KIND' ID = 1234 client = self._makeOne() @@ -359,6 +621,7 @@ def test_key_w_namespace_collision(self): def test_batch_wo_connection(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + client = self._makeOne() with _Monkey(MUT, Batch=_Dummy): @@ -373,8 +636,9 @@ def test_batch_wo_connection(self): def test_batch_w_connection(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey - conn = object() - client = self._makeOne(connection=conn) + + connection = object() + client = self._makeOne(connection=connection) with _Monkey(MUT, Batch=_Dummy): batch = client.batch() @@ -382,11 +646,13 @@ def test_batch_w_connection(self): self.assertTrue(isinstance(batch, _Dummy)) self.assertEqual(batch.args, ()) self.assertEqual(batch.kwargs, - {'dataset_id': self.DATASET_ID, 'connection': conn}) + {'dataset_id': self.DATASET_ID, + 'connection': connection}) def test_transaction_wo_connection(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + client = self._makeOne() with _Monkey(MUT, Transaction=_Dummy): @@ -401,6 +667,7 @@ def test_transaction_wo_connection(self): def test_transaction_w_connection(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + conn = object() client = self._makeOne(connection=conn) @@ -424,6 +691,7 @@ def test_query_w_dataset_id(self): def test_query_w_defaults(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + client = self._makeOne() with _Monkey(MUT, Query=_Dummy): @@ -440,6 +708,7 @@ def test_query_w_defaults(self): def test_query_explicit(self): from gcloud.datastore import client as MUT from gcloud._testing import _Monkey + KIND = 'KIND' NAMESPACE = 'NAMESPACE' ANCESTOR = object() @@ -519,3 +788,48 @@ class _Dummy(object): def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs + + +class _HttpMultiple(object): + + def __init__(self, *responses): + self._called_with = [] + self._responses = list(responses) + + def request(self, **kw): + self._called_with.append(kw) + result, self._responses = self._responses[0], self._responses[1:] + return result + + +class _NoCommitBatch(object): + + def __init__(self, dataset_id, connection): + from gcloud.datastore.batch import Batch + self._batch = Batch(dataset_id, connection) + + def __enter__(self): + from gcloud.datastore.batch import _BATCHES + _BATCHES.push(self._batch) + return self._batch + + def __exit__(self, *args): + from gcloud.datastore.batch import _BATCHES + _BATCHES.pop() + + +class _NoCommitTransaction(object): + + def __init__(self, dataset_id, connection, transaction_id='TRANSACTION'): + from gcloud.datastore.transaction import Transaction + xact = self._transaction = Transaction(dataset_id, connection) + xact._id = transaction_id + + def __enter__(self): + from gcloud.datastore.batch import _BATCHES + _BATCHES.push(self._transaction) + return self._transaction + + def __exit__(self, *args): + from gcloud.datastore.batch import _BATCHES + _BATCHES.pop() diff --git a/gcloud/datastore/test_transaction.py b/gcloud/datastore/test_transaction.py index 07b45d1825a8..8376425d964a 100644 --- a/gcloud/datastore/test_transaction.py +++ b/gcloud/datastore/test_transaction.py @@ -73,7 +73,7 @@ def test_ctor_with_env(self): self.assertEqual(xact._status, self._getTargetClass()._INITIAL) def test_current(self): - from gcloud.datastore.test_api import _NoCommitBatch + from gcloud.datastore.test_client import _NoCommitBatch _DATASET = 'DATASET' connection = _Connection() xact1 = self._makeOne(_DATASET, connection) diff --git a/system_tests/datastore.py b/system_tests/datastore.py index 4ee3dcb0bd0a..f9fedf146090 100644 --- a/system_tests/datastore.py +++ b/system_tests/datastore.py @@ -24,6 +24,7 @@ _implicit_environ._DATASET_ENV_VAR_NAME = 'GCLOUD_TESTS_DATASET_ID' +client = datastore.Client() class TestDatastore(unittest2.TestCase): @@ -32,16 +33,16 @@ def setUp(self): self.case_entities_to_delete = [] def tearDown(self): - with datastore.Transaction(): + with client.transaction(): keys = [entity.key for entity in self.case_entities_to_delete] - datastore.delete_multi(keys) + client.delete_multi(keys) class TestDatastoreAllocateIDs(TestDatastore): def test_allocate_ids(self): num_ids = 10 - allocated_keys = datastore.allocate_ids(datastore.Key('Kind'), num_ids) + allocated_keys = client.allocate_ids(client.key('Kind'), num_ids) self.assertEqual(len(allocated_keys), num_ids) unique_ids = set() @@ -82,7 +83,7 @@ def _get_post(self, id_or_name=None, post_content=None): def _generic_test_post(self, name=None, key_id=None): entity = self._get_post(id_or_name=(name or key_id)) - datastore.put(entity) + client.put(entity) # Register entity to be deleted. self.case_entities_to_delete.append(entity) @@ -91,7 +92,7 @@ def _generic_test_post(self, name=None, key_id=None): self.assertEqual(entity.key.name, name) if key_id is not None: self.assertEqual(entity.key.id, key_id) - retrieved_entity = datastore.get(entity.key) + retrieved_entity = client.get(entity.key) # Check the given and retrieved are the the same. self.assertEqual(retrieved_entity, entity) @@ -126,7 +127,7 @@ def test_save_multiple(self): self.case_entities_to_delete.append(entity2) keys = [entity1.key, entity2.key] - matches = datastore.get_multi(keys) + matches = client.get_multi(keys) self.assertEqual(len(matches), 2) def test_empty_kind(self): @@ -145,7 +146,7 @@ def test_save_key_self_reference(self): entity['fullName'] = u'Full name' entity['linkedTo'] = key # Self reference. - datastore.put(entity) + client.put(entity) self.case_entities_to_delete.append(entity) query = datastore.Query(kind='Person') @@ -330,12 +331,12 @@ def test_transaction(self): entity['url'] = u'www.google.com' with datastore.Transaction() as xact: - result = datastore.get(entity.key) + result = client.get(entity.key) if result is None: xact.put(entity) self.case_entities_to_delete.append(entity) # This will always return after the transaction. - retrieved_entity = datastore.get(entity.key) + retrieved_entity = client.get(entity.key) self.case_entities_to_delete.append(retrieved_entity) self.assertEqual(retrieved_entity, entity)