docs: cloudpickle is an interesting extension of the pickle functiona…

…lity (#361) Signed-off-by: H. Felix Wittmann <hfwittmann@gmail.com>
kedro-org · Oct 10, 2023 · 527706d · 527706d
1 parent 78913d8
commit 527706d
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 2 deletions.
diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md
@@ -1,17 +1,20 @@
 # Upcoming Release
 ## Major features and improvements
 ## Bug fixes and other changes
+* Updated `PickleDataset` to explicitly mention `cloudpickle` support.
 ## Upcoming deprecations for Kedro-Datasets 2.0.0
 
+## Community contributions
+Many thanks to the following Kedroids for contributing PRs to this release:
+* [Felix Wittmann](https://github.com/hfwittmann)
+
 # Release 1.7.1
 ## Bug fixes and other changes
 * Pin `tables` version on `kedro-datasets` for Python < 3.8.
 
 ## Upcoming deprecations for Kedro-Datasets 2.0.0
 * Renamed dataset and error classes, in accordance with the [Kedro lexicon](https://github.com/kedro-org/kedro/wiki/Kedro-documentation-style-guide#kedro-lexicon). Dataset classes ending with "DataSet" are deprecated and will be removed in 2.0.0.
 
-## Community contributions
-
 # Release 1.7.0:
 ## Major features and improvements
 * Added `polars.GenericDataSet`, a `GenericDataSet` backed by [polars](https://www.pola.rs/), a lightning fast dataframe package built entirely using Rust.

diff --git a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py
@@ -110,6 +110,8 @@ def __init__(  # noqa: PLR0913
                 dill.load: https://dill.readthedocs.io/en/latest/index.html#dill.load
                 compress_pickle.load:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.load
+                cloudpickle.load:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py
                 All defaults are preserved.
             save_args: Pickle options for saving pickle files.
                 You can pass in arguments that the backend dump function specified accepts, e.g:
@@ -118,6 +120,8 @@ def __init__(  # noqa: PLR0913
                 dill.dump: https://dill.readthedocs.io/en/latest/index.html#dill.dump
                 compress_pickle.dump:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dump
+                cloudpickle.dump:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py
                 All defaults are preserved.
             version: If specified, should be an instance of
                 ``kedro.io.core.Version``. If its ``load`` attribute is

diff --git a/kedro-datasets/kedro_datasets/redis/redis_dataset.py b/kedro-datasets/kedro_datasets/redis/redis_dataset.py
@@ -79,6 +79,7 @@ def __init__(  # noqa: PLR0913
             * `pickle`
             * `dill`
             * `compress_pickle`
+            * `cloudpickle`
 
         Example backends that are incompatible:
             * `torch`
@@ -94,13 +95,17 @@ def __init__(  # noqa: PLR0913
                 dill.loads: https://dill.readthedocs.io/en/latest/index.html#dill.loads
                 compress_pickle.loads:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.loads
+                cloudpickle.loads:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py
                 All defaults are preserved.
             save_args: Pickle options for saving pickle files.
                 You can pass in arguments that the backend dump function specified accepts, e.g:
                 pickle.dumps: https://docs.python.org/3/library/pickle.html#pickle.dump
                 dill.dumps: https://dill.readthedocs.io/en/latest/index.html#dill.dumps
                 compress_pickle.dumps:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dumps
+                cloudpickle.dumps:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py
                 All defaults are preserved.
             credentials: Credentials required to get access to the redis server.
                 E.g. `{"password": None}`.

diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py
@@ -153,6 +153,7 @@ def _collect_requirements(requires):
     "biopython~=1.73",
     "blacken-docs==1.9.2",
     "black~=22.0",
+    "cloudpickle<=2.0.0",
     "compress-pickle[lz4]~=2.1.0",
     "coverage[toml]",
     "dask[complete]~=2021.10",  # pinned by Snyk to avoid a vulnerability

diff --git a/kedro-datasets/tests/pickle/test_pickle_dataset.py b/kedro-datasets/tests/pickle/test_pickle_dataset.py
@@ -68,6 +68,7 @@ class TestPickleDataset:
             ("pickle", None, None),
             ("joblib", None, None),
             ("dill", None, None),
+            ("cloudpickle", None, None),
             ("compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}),
         ],
         indirect=True,

diff --git a/kedro-datasets/tests/redis/test_redis_dataset.py b/kedro-datasets/tests/redis/test_redis_dataset.py
@@ -76,6 +76,7 @@ class TestPickleDataset:
         [
             ("a", "pickle", None, None),
             (1, "dill", None, None),
+            (2, "cloudpickle", None, None),
             ("key", "compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}),
         ],
         indirect=True,