Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Updating datastore samples to use eventually consistent helper #331

Merged
merged 1 commit into from
May 6, 2016
Merged

Updating datastore samples to use eventually consistent helper #331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 85 additions & 84 deletions datastore/api/snippets_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import time

from gcloud import datastore
from gcp.testing import eventually_consistent
from gcp.testing.flaky import flaky
import pytest
import snippets
Expand All @@ -28,17 +27,8 @@ def __init__(self, *args, **kwargs):
def cleanup(self):
with self.batch():
self.delete_multi(
[x.key for x in self.entities_to_delete] +
self.keys_to_delete)


# This is pretty hacky, but make datastore wait 1s after any
# put operation to in order to account for eventual consistency.
class WaitingClient(CleanupClient):
def put_multi(self, *args, **kwargs):
result = super(WaitingClient, self).put_multi(*args, **kwargs)
time.sleep(1)
return result
list(set([x.key for x in self.entities_to_delete])) +
list(set(self.keys_to_delete)))


@pytest.yield_fixture
Expand All @@ -48,13 +38,6 @@ def client(cloud_config):
client.cleanup()


@pytest.yield_fixture
def waiting_client(cloud_config):
client = WaitingClient(cloud_config.project)
yield client
client.cleanup()


@flaky
class TestDatastoreSnippets:
# These tests mostly just test the absence of exceptions.
Expand Down Expand Up @@ -118,20 +101,23 @@ def test_batch_lookup(self, client):
def test_batch_delete(self, client):
snippets.batch_delete(client)

def test_unindexed_property_query(self, waiting_client):
tasks = snippets.unindexed_property_query(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_unindexed_property_query(self, client):
tasks = snippets.unindexed_property_query(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_basic_query(self, waiting_client):
tasks = snippets.basic_query(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_basic_query(self, client):
tasks = snippets.basic_query(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_projection_query(self, waiting_client):
priorities, percents = snippets.projection_query(waiting_client)
waiting_client.entities_to_delete.extend(
waiting_client.query(kind='Task').fetch())
@eventually_consistent.mark
def test_projection_query(self, client):
priorities, percents = snippets.projection_query(client)
client.entities_to_delete.extend(
client.query(kind='Task').fetch())
assert priorities
assert percents

Expand All @@ -143,63 +129,74 @@ def test_ancestor_query(self, client):
def test_run_query(self, client):
snippets.run_query(client)

def test_cursor_paging(self, waiting_client):
def test_cursor_paging(self, client):
for n in range(6):
waiting_client.entities_to_delete.append(
snippets.insert(waiting_client))

page_one, cursor_one, page_two, cursor_two = snippets.cursor_paging(
waiting_client)

assert len(page_one) == 5
assert len(page_two) == 1
assert cursor_one
assert cursor_two

def test_property_filter(self, waiting_client):
tasks = snippets.property_filter(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
client.entities_to_delete.append(
snippets.insert(client))

@eventually_consistent.call
def _():
results = snippets.cursor_paging(client)
page_one, cursor_one, page_two, cursor_two = results

assert len(page_one) == 5
assert len(page_two) == 1
assert cursor_one
assert cursor_two

@eventually_consistent.mark
def test_property_filter(self, client):
tasks = snippets.property_filter(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_composite_filter(self, waiting_client):
tasks = snippets.composite_filter(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_composite_filter(self, client):
tasks = snippets.composite_filter(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_key_filter(self, waiting_client):
tasks = snippets.key_filter(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_key_filter(self, client):
tasks = snippets.key_filter(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_ascending_sort(self, waiting_client):
tasks = snippets.ascending_sort(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_ascending_sort(self, client):
tasks = snippets.ascending_sort(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_descending_sort(self, waiting_client):
tasks = snippets.descending_sort(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_descending_sort(self, client):
tasks = snippets.descending_sort(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_multi_sort(self, waiting_client):
tasks = snippets.multi_sort(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_multi_sort(self, client):
tasks = snippets.multi_sort(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_keys_only_query(self, waiting_client):
keys = snippets.keys_only_query(waiting_client)
waiting_client.entities_to_delete.extend(
waiting_client.query(kind='Task').fetch())
@eventually_consistent.mark
def test_keys_only_query(self, client):
keys = snippets.keys_only_query(client)
client.entities_to_delete.extend(
client.query(kind='Task').fetch())
assert keys

def test_distinct_query(self, waiting_client):
tasks = snippets.distinct_query(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_distinct_query(self, client):
tasks = snippets.distinct_query(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_distinct_on_query(self, waiting_client):
tasks = snippets.distinct_on_query(waiting_client)
waiting_client.entities_to_delete.extend(tasks)
@eventually_consistent.mark
def test_distinct_on_query(self, client):
tasks = snippets.distinct_on_query(client)
client.entities_to_delete.extend(tasks)
assert tasks

def test_kindless_query(self, client):
Expand Down Expand Up @@ -251,29 +248,33 @@ def transactional_single_entity_group_read_only(self, client):
assert task_list
assert tasks_in_list

def test_namespace_run_query(self, waiting_client):
@eventually_consistent.mark
def test_namespace_run_query(self, client):
all_namespaces, filtered_namespaces = snippets.namespace_run_query(
waiting_client)
client)
assert all_namespaces
assert filtered_namespaces
assert 'google' in filtered_namespaces

def test_kind_run_query(self, waiting_client):
kinds = snippets.kind_run_query(waiting_client)
waiting_client.entities_to_delete.extend(
waiting_client.query(kind='Task').fetch())
@eventually_consistent.mark
def test_kind_run_query(self, client):
kinds = snippets.kind_run_query(client)
client.entities_to_delete.extend(
client.query(kind='Task').fetch())
assert kinds
assert 'Task' in kinds

def test_property_run_query(self, waiting_client):
kinds = snippets.property_run_query(waiting_client)
waiting_client.entities_to_delete.extend(
waiting_client.query(kind='Task').fetch())
@eventually_consistent.mark
def test_property_run_query(self, client):
kinds = snippets.property_run_query(client)
client.entities_to_delete.extend(
client.query(kind='Task').fetch())
assert kinds
assert 'Task' in kinds

def test_property_by_kind_run_query(self, waiting_client):
reprs = snippets.property_by_kind_run_query(waiting_client)
waiting_client.entities_to_delete.extend(
waiting_client.query(kind='Task').fetch())
@eventually_consistent.mark
def test_property_by_kind_run_query(self, client):
reprs = snippets.property_by_kind_run_query(client)
client.entities_to_delete.extend(
client.query(kind='Task').fetch())
assert reprs
20 changes: 13 additions & 7 deletions datastore/api/tasks_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# limitations under the License.

from gcloud import datastore
from gcp.testing import eventually_consistent
from gcp.testing.flaky import flaky
import pytest
import tasks
Expand Down Expand Up @@ -55,8 +56,11 @@ def test_mark_done(client):
def test_list_tasks(client):
task1_key = tasks.add_task(client, 'Test task 1')
task2_key = tasks.add_task(client, 'Test task 2')
task_list = tasks.list_tasks(client)
assert [x.key for x in task_list] == [task1_key, task2_key]

@eventually_consistent.call
def _():
task_list = tasks.list_tasks(client)
assert [x.key for x in task_list] == [task1_key, task2_key]


@flaky
Expand All @@ -72,9 +76,11 @@ def test_format_tasks(client):
tasks.add_task(client, 'Test task 2')
tasks.mark_done(client, task1_key.id)

output = tasks.format_tasks(tasks.list_tasks(client))
@eventually_consistent.call
def _():
output = tasks.format_tasks(tasks.list_tasks(client))

assert 'Test task 1' in output
assert 'Test task 2' in output
assert 'done' in output
assert 'created' in output
assert 'Test task 1' in output
assert 'Test task 2' in output
assert 'done' in output
assert 'created' in output