diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md index 0b7ac02cc..63f6ae91a 100755 --- a/kedro-datasets/RELEASE.md +++ b/kedro-datasets/RELEASE.md @@ -1,8 +1,13 @@ # Upcoming Release ## Major features and improvements ## Bug fixes and other changes +* Updated `PickleDataset` to explicitly mention `cloudpickle` support. ## Upcoming deprecations for Kedro-Datasets 2.0.0 +## Community contributions +Many thanks to the following Kedroids for contributing PRs to this release: +* [Felix Wittmann](https://github.com/hfwittmann) + # Release 1.7.1 ## Bug fixes and other changes * Pin `tables` version on `kedro-datasets` for Python < 3.8. @@ -10,8 +15,6 @@ ## Upcoming deprecations for Kedro-Datasets 2.0.0 * Renamed dataset and error classes, in accordance with the [Kedro lexicon](https://github.com/kedro-org/kedro/wiki/Kedro-documentation-style-guide#kedro-lexicon). Dataset classes ending with "DataSet" are deprecated and will be removed in 2.0.0. -## Community contributions - # Release 1.7.0: ## Major features and improvements * Added `polars.GenericDataSet`, a `GenericDataSet` backed by [polars](https://www.pola.rs/), a lightning fast dataframe package built entirely using Rust. diff --git a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py index 21d3b8c71..21f97b713 100644 --- a/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py +++ b/kedro-datasets/kedro_datasets/pickle/pickle_dataset.py @@ -110,6 +110,8 @@ def __init__( # noqa: PLR0913 dill.load: https://dill.readthedocs.io/en/latest/index.html#dill.load compress_pickle.load: https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.load + cloudpickle.load: + https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py All defaults are preserved. save_args: Pickle options for saving pickle files. You can pass in arguments that the backend dump function specified accepts, e.g: @@ -118,6 +120,8 @@ def __init__( # noqa: PLR0913 dill.dump: https://dill.readthedocs.io/en/latest/index.html#dill.dump compress_pickle.dump: https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dump + cloudpickle.dump: + https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py All defaults are preserved. version: If specified, should be an instance of ``kedro.io.core.Version``. If its ``load`` attribute is diff --git a/kedro-datasets/kedro_datasets/redis/redis_dataset.py b/kedro-datasets/kedro_datasets/redis/redis_dataset.py index 9979cf386..1e782059b 100644 --- a/kedro-datasets/kedro_datasets/redis/redis_dataset.py +++ b/kedro-datasets/kedro_datasets/redis/redis_dataset.py @@ -79,6 +79,7 @@ def __init__( # noqa: PLR0913 * `pickle` * `dill` * `compress_pickle` + * `cloudpickle` Example backends that are incompatible: * `torch` @@ -94,6 +95,8 @@ def __init__( # noqa: PLR0913 dill.loads: https://dill.readthedocs.io/en/latest/index.html#dill.loads compress_pickle.loads: https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.loads + cloudpickle.loads: + https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py All defaults are preserved. save_args: Pickle options for saving pickle files. You can pass in arguments that the backend dump function specified accepts, e.g: @@ -101,6 +104,8 @@ def __init__( # noqa: PLR0913 dill.dumps: https://dill.readthedocs.io/en/latest/index.html#dill.dumps compress_pickle.dumps: https://lucianopaz.github.io/compress_pickle/html/api/compress_pickle.html#compress_pickle.compress_pickle.dumps + cloudpickle.dumps: + https://github.com/cloudpipe/cloudpickle/blob/0f330b6afe55313fc1efc090a7d350f5ad5c9317/tests/cloudpickle_test.py All defaults are preserved. credentials: Credentials required to get access to the redis server. E.g. `{"password": None}`. diff --git a/kedro-datasets/setup.py b/kedro-datasets/setup.py index 340ad5e67..a22e83f81 100644 --- a/kedro-datasets/setup.py +++ b/kedro-datasets/setup.py @@ -153,6 +153,7 @@ def _collect_requirements(requires): "biopython~=1.73", "blacken-docs==1.9.2", "black~=22.0", + "cloudpickle<=2.0.0", "compress-pickle[lz4]~=2.1.0", "coverage[toml]", "dask[complete]~=2021.10", # pinned by Snyk to avoid a vulnerability diff --git a/kedro-datasets/tests/pickle/test_pickle_dataset.py b/kedro-datasets/tests/pickle/test_pickle_dataset.py index be09d6291..e53a8b675 100644 --- a/kedro-datasets/tests/pickle/test_pickle_dataset.py +++ b/kedro-datasets/tests/pickle/test_pickle_dataset.py @@ -68,6 +68,7 @@ class TestPickleDataset: ("pickle", None, None), ("joblib", None, None), ("dill", None, None), + ("cloudpickle", None, None), ("compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}), ], indirect=True, diff --git a/kedro-datasets/tests/redis/test_redis_dataset.py b/kedro-datasets/tests/redis/test_redis_dataset.py index a2ec3bf83..f569d7d22 100644 --- a/kedro-datasets/tests/redis/test_redis_dataset.py +++ b/kedro-datasets/tests/redis/test_redis_dataset.py @@ -76,6 +76,7 @@ class TestPickleDataset: [ ("a", "pickle", None, None), (1, "dill", None, None), + (2, "cloudpickle", None, None), ("key", "compress_pickle", {"compression": "lz4"}, {"compression": "lz4"}), ], indirect=True,