From e7ed47844418982a5d9845321e674a289a663534 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 6 Feb 2020 11:20:40 -0500 Subject: [PATCH 1/4] Test deterministic dask tokenization. --- requirements/tests.txt | 3 ++- sparse/tests/test_dask_interop.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 sparse/tests/test_dask_interop.py diff --git a/requirements/tests.txt b/requirements/tests.txt index 0c580890..69252706 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,3 +1,4 @@ +dask pytest>=3.5 pytest-black -pytest-cov \ No newline at end of file +pytest-cov diff --git a/sparse/tests/test_dask_interop.py b/sparse/tests/test_dask_interop.py new file mode 100644 index 00000000..4ff58156 --- /dev/null +++ b/sparse/tests/test_dask_interop.py @@ -0,0 +1,11 @@ +from dask.base import tokenize +import sparse + + +def test_deterministic_token(): + a = sparse.COO(data=[1, 2, 3], coords=[10, 20, 30], shape=(40,)) + b = sparse.COO(data=[1, 2, 3], coords=[10, 20, 30], shape=(40,)) + assert tokenize(a) == tokenize(b) + # One of these things is not like the other.... + c = sparse.COO(data=[1, 2, 4], coords=[10, 20, 30], shape=(40,)) + assert tokenize(a) != tokenize(c) From 136bc85647b2f8b34a2488899f5c5eb6c3f49195 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 6 Feb 2020 11:15:14 -0500 Subject: [PATCH 2/4] Implement __dask_tokenize__. --- sparse/_coo/core.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sparse/_coo/core.py b/sparse/_coo/core.py index bc5cc93f..bf014ed8 100644 --- a/sparse/_coo/core.py +++ b/sparse/_coo/core.py @@ -284,6 +284,12 @@ def __setstate__(self, state): self.coords, self.data, self.shape, self.fill_value = state self._cache = None + def __dask_tokenize__(self): + "Produce a deterministic, content-based hash for dask." + from dask.base import normalize_token + return normalize_token( + (type(self), self.coords, self.data, self.shape, self.fill_value)) + def copy(self, deep=True): """Return a copy of the array. From cb4ddf5b451588f6e481adfa0bc18d54f4f85b2a Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 6 Feb 2020 14:01:44 -0500 Subject: [PATCH 3/4] Add missing test dependency. --- requirements/tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index 69252706..7c28c17d 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,4 +1,4 @@ -dask +dask[array] pytest>=3.5 pytest-black pytest-cov From 1aac3e47c0bb7d8b80daf9a4fcace2c49ebaf525 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Thu, 6 Feb 2020 14:53:30 -0500 Subject: [PATCH 4/4] Apply black. --- sparse/_coo/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparse/_coo/core.py b/sparse/_coo/core.py index bf014ed8..74820fb4 100644 --- a/sparse/_coo/core.py +++ b/sparse/_coo/core.py @@ -287,8 +287,10 @@ def __setstate__(self, state): def __dask_tokenize__(self): "Produce a deterministic, content-based hash for dask." from dask.base import normalize_token + return normalize_token( - (type(self), self.coords, self.data, self.shape, self.fill_value)) + (type(self), self.coords, self.data, self.shape, self.fill_value) + ) def copy(self, deep=True): """Return a copy of the array.