cloudpickle is an interesting extension of the pickle functionality.

Quote from their github: "cloudpickle makes it possible to serialize Python constructs not supported by the default pickle module from the Python standard library." Its already implicity supported, but would be useful to have it in the docs. Signed-off-by: H. Felix Wittmann <hfwittmann@gmail.com>
kedro-org · Sep 30, 2023 · 3b1672e · 3b1672e
1 parent 7b3ac6c
commit 3b1672e
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 0 deletions.
diff --git a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py
@@ -108,6 +108,8 @@ def __init__(
                 dill.load: https://dill.readthedocs.io/en/latest/index.html#dill.load
                 compress_pickle.load:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.load
+                cloudpickle.load:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py#L472
                 All defaults are preserved.
             save_args: Pickle options for saving pickle files.
                 You can pass in arguments that the backend dump function specified accepts, e.g:
@@ -116,6 +118,8 @@ def __init__(
                 dill.dump: https://dill.readthedocs.io/en/latest/index.html#dill.dump
                 compress_pickle.dump:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dump
+                cloudpickle.dump:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py#L470
                 All defaults are preserved.
             version: If specified, should be an instance of
                 ``kedro.io.core.Version``. If its ``load`` attribute is

diff --git a/kedro-datasets/kedro_datasets/redis/redis_dataset.py b/kedro-datasets/kedro_datasets/redis/redis_dataset.py
@@ -79,6 +79,7 @@ def __init__(
             * `pickle`
             * `dill`
             * `compress_pickle`
+            * `cloudpickle`
 
         Example backends that are incompatible:
             * `torch`
@@ -94,13 +95,17 @@ def __init__(
                 dill.loads: https://dill.readthedocs.io/en/latest/index.html#dill.loads
                 compress_pickle.loads:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.loads
+                cloudpickle.loads:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py#L463
                 All defaults are preserved.
             save_args: Pickle options for saving pickle files.
                 You can pass in arguments that the backend dump function specified accepts, e.g:
                 pickle.dumps: https://docs.python.org/3/library/pickle.html#pickle.dump
                 dill.dumps: https://dill.readthedocs.io/en/latest/index.html#dill.dumps
                 compress_pickle.dumps:
                 https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dumps
+                cloudpickle.dumps:
+                https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py#L463                
                 All defaults are preserved.
             credentials: Credentials required to get access to the redis server.
                 E.g. `{"password": None}`.

diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py
@@ -152,6 +152,7 @@ def _collect_requirements(requires):
     "biopython~=1.73",
     "blacken-docs==1.9.2",
     "black~=22.0",
+    "cloudpickle~=2.2.1",
     "compress-pickle[lz4]~=2.1.0",
     "coverage[toml]",
     "dask[complete]~=2021.10",  # pinned by Snyk to avoid a vulnerability

diff --git a/kedro-datasets/tests/pickle/test_pickle_dataset.py b/kedro-datasets/tests/pickle/test_pickle_dataset.py
@@ -65,6 +65,7 @@ class TestPickleDataset:
             ("pickle", None, None),
             ("joblib", None, None),
             ("dill", None, None),
+            ("cloudpickle", None, None),
             ("compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}),
         ],
         indirect=True,

diff --git a/kedro-datasets/tests/redis/test_redis_dataset.py b/kedro-datasets/tests/redis/test_redis_dataset.py
@@ -73,6 +73,7 @@ class TestPickleDataset:
         [
             ("a", "pickle", None, None),
             (1, "dill", None, None),
+            (2, "cloudpickle", None, None),
             ("key", "compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}),
         ],
         indirect=True,