diff --git a/README.md b/README.md index bf2f342afb4..740e0d4a14d 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ the Apache License Version 2.0. Projects Showcase

- šŸŽ‰ Version 0.66.0 is out. Check out the release notes + šŸŽ‰ Version 0.67.0 is out. Check out the release notes here.
šŸ–„ļø Download our VS Code Extension here. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index eb216cddd9c..1131caeb096 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,103 @@ +# 0.67.0 + +## Highlights + +- **Improved Sagemaker Orchestrator:** Now supports warm pools for AWS Sagemaker, enhancing performance and reducing startup times for TrainingJobs. +- **New DAG Visualizer:** Shipped major enhancements to the DAG Visualizer for Pipeline Runs: + - Preview of the actual DAG before pipeline completion + - Visual adjustments for improved clarity + - Real-time updates during pipeline execution +- **Environment Variable References in Configurations:** Introduced the ability to reference environment variables in both code and configuration files using the syntax ${ENV_VARIABLE_NAME}, increasing flexibility in setups. +- **Enhanced UX for Major Cloud Providers:** Displaying direct pipeline/log URL when working with major cloud platforms. +- **Skypilot with Kubernetes Support:** Added compatibility for running Skypilot orchestrator on Kubernetes clusters. +- **Updated Deepchecks Integration:** The Deepchecks integration has been refreshed with the latest features and improvements. + + +## Features and Improvements + +- **AWS Integration:** + - Added permissions to workflow to enable assuming AWS role. + - Fixed expired credentials error when using the docker service connector. +- **Error Handling:** Improved error messages for stack components of uninstalled integrations. +- **API Key Management:** Added an option to write API keys to a file instead of using the CLI. + +## Pipeline Execution: + +- Implemented fixes for executing steps as single step pipelines. +- Added filter option for templatable runs. +- Added additional filtering options for pipeline runs. +- MLflow Integration: Linked registered models in MLflow with the corresponding MLflow run. +- Analytics: Added missing analytics event to improve user insights. + +## Documentation Updates + +- Updated documentation for various integrations including: + - Lightning AI orchestrator + - Kubeflow + - Comet experiment tracker + - Neptune + - Hugging Face deployer + - Weights & Biases (wandb) +- Added documentation for run templates. +- Fixed incorrect method name in Pigeon docs. +- Various small documentation fixes and improvements. + +## Bug Fixes + +- Fixed YAML formatting issues. +- Resolved RBAC issues for subpages in response models. +- Fixed step output annotation in Discord test. +- Addressed MLFlow integration requirements duplication. +- Fixed Lightning orchestrator functionality. + +## What's Changed +* Error message for stack components of uninstalled integrations by @bcdurak in https://github.com/zenml-io/zenml/pull/2996 +* Enable migration testing for version 0.66.0 by @schustmi in https://github.com/zenml-io/zenml/pull/2998 +* Add permissions to workflow to enable assuming AWS role by @schustmi in https://github.com/zenml-io/zenml/pull/2999 +* Add option to write api key to file instead of CLI by @schustmi in https://github.com/zenml-io/zenml/pull/3001 +* Fix yaml formatting by @schustmi in https://github.com/zenml-io/zenml/pull/3004 +* Update ZenML Pro links for consistency.- Update ZenML Pro links for cā€¦ by @htahir1 in https://github.com/zenml-io/zenml/pull/3007 +* Fix incorrect method name in Pigeon docs by @strickvl in https://github.com/zenml-io/zenml/pull/3008 +* Fixes for executing steps as single step pipelines by @schustmi in https://github.com/zenml-io/zenml/pull/3006 +* Add filter option for templatable runs by @schustmi in https://github.com/zenml-io/zenml/pull/3000 +* Add missing analytics event by @schustmi in https://github.com/zenml-io/zenml/pull/3009 +* Fix expired credentials error when using the docker service connector by @schustmi in https://github.com/zenml-io/zenml/pull/3002 +* Fix Lightning docs by @strickvl in https://github.com/zenml-io/zenml/pull/3013 +* Remove image builder warning by @htahir1 in https://github.com/zenml-io/zenml/pull/3014 +* Fixed kubeflow docs by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3018 +* Update Comet experiment tracker docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3019 +* Small docs fixes by @strickvl in https://github.com/zenml-io/zenml/pull/3022 +* Feature/cleanup unused file by @AlexejPenner in https://github.com/zenml-io/zenml/pull/3023 +* MLFlow integration requirements duplicate fix by @bcdurak in https://github.com/zenml-io/zenml/pull/3011 +* Fix Neptune docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3026 +* Fix huggingface deployer docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3024 +* Fix step output annotation in Discord test by @wjayesh in https://github.com/zenml-io/zenml/pull/3029 +* Fix RBAC for subpages in response models by @schustmi in https://github.com/zenml-io/zenml/pull/3031 +* Allow env variable placeholders in configurations by @schustmi in https://github.com/zenml-io/zenml/pull/3003 +* Leverage warm pools for AWS Sagemaker by @avishniakov in https://github.com/zenml-io/zenml/pull/3027 +* Updated wandb docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3030 +* Add hyperlint by @htahir1 in https://github.com/zenml-io/zenml/pull/3035 +* Bump NLP template by @avishniakov in https://github.com/zenml-io/zenml/pull/3036 +* Add additional filtering options by @schustmi in https://github.com/zenml-io/zenml/pull/2951 +* Bump starter template version by @schustmi in https://github.com/zenml-io/zenml/pull/3038 +* Docs for run templates by @bcdurak in https://github.com/zenml-io/zenml/pull/3028 +* Update Lightning AI orchestrator documentation by @strickvl in https://github.com/zenml-io/zenml/pull/3016 +* Add default value for PipelineRun.is_templatable by @schustmi in https://github.com/zenml-io/zenml/pull/3040 +* Use a generic OAuth2 client credentials flow to login to the Cloud API by @stefannica in https://github.com/zenml-io/zenml/pull/3041 +* fix lightning orchestrator by @safoinme in https://github.com/zenml-io/zenml/pull/3010 +* Linking registered models in MLflow with the corresponding MLflow run by @aiakide in https://github.com/zenml-io/zenml/pull/3020 +* Bugfixing mlflow registry linting issue by @bcdurak in https://github.com/zenml-io/zenml/pull/3043 +* Enhancing the orchestrator UX for major cloud providers by @bcdurak in https://github.com/zenml-io/zenml/pull/3005 +* Skypilot with Kubernetes by @safoinme in https://github.com/zenml-io/zenml/pull/3033 +* Update deepchecks integration by @wjayesh in https://github.com/zenml-io/zenml/pull/2987 + +## New Contributors +* @aiakide made their first contribution in https://github.com/zenml-io/zenml/pull/3020 + +**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.66.0...0.67.0 + # 0.66.0 ## New Features and Improvements diff --git a/examples/quickstart/configs/training_aws.yaml b/examples/quickstart/configs/training_aws.yaml index 126dc7bb30e..55819ebaf83 100644 --- a/examples/quickstart/configs/training_aws.yaml +++ b/examples/quickstart/configs/training_aws.yaml @@ -3,7 +3,7 @@ enable_cache: True # Environment configuration settings: docker: - parent_image: "715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.66.0-py3.11-aws" + parent_image: "715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.67.0-py3.11-aws" skip_build: True # If you switch this to False remove the parent_image requirements: requirements.txt orchestrator.sagemaker: diff --git a/examples/quickstart/configs/training_azure.yaml b/examples/quickstart/configs/training_azure.yaml index dd73b7c7096..52d8d7b1323 100644 --- a/examples/quickstart/configs/training_azure.yaml +++ b/examples/quickstart/configs/training_azure.yaml @@ -3,7 +3,7 @@ enable_cache: True # Environment configuration settings: docker: - parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.66.0-py3.11-azure" + parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.67.0-py3.11-azure" skip_build: True requirements: requirements.txt # Uncomment the following lines to specify the accelerator for your azureml orchestrator diff --git a/examples/quickstart/configs/training_gcp.yaml b/examples/quickstart/configs/training_gcp.yaml index 320ae7673cb..8e712198870 100644 --- a/examples/quickstart/configs/training_gcp.yaml +++ b/examples/quickstart/configs/training_gcp.yaml @@ -3,7 +3,7 @@ enable_cache: True # Environment configuration settings: docker: - parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.66.0-py3.11-gcp" + parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.67.0-py3.11-gcp" skip_build: True requirements: requirements.txt # Uncomment the following two lines to specify the accelerator for your vertex orchestrator diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index 129daa85a69..98e1b92c080 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -494,7 +494,7 @@ "# Common imports and setup\n", "if CLOUD_PROVIDER.lower() == \"gcp\":\n", " parent_image = (\n", - " \"zenmldocker/zenml-public-pipelines:quickstart-0.66.0-py3.11-gcp\"\n", + " \"zenmldocker/zenml-public-pipelines:quickstart-0.67.0-py3.11-gcp\"\n", " )\n", " skip_build = True\n", "\n", @@ -503,7 +503,7 @@ " SagemakerOrchestratorSettings,\n", " )\n", "\n", - " parent_image = \"339712793861.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.66.0-py3.11-aws\"\n", + " parent_image = \"339712793861.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.67.0-py3.11-aws\"\n", " skip_build = True # if you switch this to False, you need to remove the parent image\n", "\n", " settings[\"orchestrator.sagemaker\"] = SagemakerOrchestratorSettings(\n", @@ -512,7 +512,7 @@ "\n", "elif CLOUD_PROVIDER.lower() == \"azure\":\n", " parent_image = (\n", - " \"zenmldocker/zenml-public-pipelines:quickstart-0.66.0-py3.11-azure\"\n", + " \"zenmldocker/zenml-public-pipelines:quickstart-0.67.0-py3.11-azure\"\n", " )\n", " skip_build = True\n", "\n", diff --git a/examples/quickstart/requirements.txt b/examples/quickstart/requirements.txt index b170d9ff3bb..6f083e05b40 100644 --- a/examples/quickstart/requirements.txt +++ b/examples/quickstart/requirements.txt @@ -1,4 +1,4 @@ -zenml[server]==0.66.0 +zenml[server]==0.67.0 notebook pyarrow datasets diff --git a/examples/quickstart/requirements_aws.txt b/examples/quickstart/requirements_aws.txt index b1d0587fc0d..6cd1fc7628c 100644 --- a/examples/quickstart/requirements_aws.txt +++ b/examples/quickstart/requirements_aws.txt @@ -1,4 +1,4 @@ -zenml[server]==0.66.0 +zenml[server]==0.67.0 notebook pyarrow datasets diff --git a/examples/quickstart/requirements_azure.txt b/examples/quickstart/requirements_azure.txt index 60a07ec9bbf..eb7631c0385 100644 --- a/examples/quickstart/requirements_azure.txt +++ b/examples/quickstart/requirements_azure.txt @@ -1,4 +1,4 @@ -zenml[server]==0.66.0 +zenml[server]==0.67.0 notebook pyarrow datasets diff --git a/examples/quickstart/requirements_gcp.txt b/examples/quickstart/requirements_gcp.txt index 43ec864fecb..6c07e4612c6 100644 --- a/examples/quickstart/requirements_gcp.txt +++ b/examples/quickstart/requirements_gcp.txt @@ -1,4 +1,4 @@ -zenml[server]==0.66.0 +zenml[server]==0.67.0 notebook pyarrow datasets diff --git a/pyproject.toml b/pyproject.toml index 416ad3ef47b..6922df1d452 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "zenml" -version = "0.66.0" +version = "0.67.0" packages = [{ include = "zenml", from = "src" }] description = "ZenML: Write production-ready ML code." authors = ["ZenML GmbH "] diff --git a/src/zenml/VERSION b/src/zenml/VERSION index 367d4880362..24c41e038fb 100644 --- a/src/zenml/VERSION +++ b/src/zenml/VERSION @@ -1 +1 @@ -0.66.0 \ No newline at end of file +0.67.0 \ No newline at end of file diff --git a/src/zenml/integrations/pytorch/materializers/base_pytorch_materializer.py b/src/zenml/integrations/pytorch/materializers/base_pytorch_materializer.py index bad8ecc56f0..d1e786edadf 100644 --- a/src/zenml/integrations/pytorch/materializers/base_pytorch_materializer.py +++ b/src/zenml/integrations/pytorch/materializers/base_pytorch_materializer.py @@ -41,7 +41,10 @@ def load(self, data_type: Type[Any]) -> Any: The loaded PyTorch object. """ with fileio.open(os.path.join(self.uri, self.FILENAME), "rb") as f: - return torch.load(f) + # NOTE (security): The `torch.load` function uses `pickle` as + # the default unpickler, which is NOT secure. This materializer + # is intended for use with trusted data sources. + return torch.load(f) # nosec def save(self, obj: Any) -> None: """Uses `torch.save` to save a PyTorch object. @@ -50,7 +53,10 @@ def save(self, obj: Any) -> None: obj: The PyTorch object to save. """ with fileio.open(os.path.join(self.uri, self.FILENAME), "wb") as f: - torch.save(obj, f, pickle_module=cloudpickle) + # NOTE (security): The `torch.save` function uses `cloudpickle` as + # the default unpickler, which is NOT secure. This materializer + # is intended for use with trusted data sources. + torch.save(obj, f, pickle_module=cloudpickle) # nosec # Alias for the BasePyTorchMaterializer class, allowing users that have already used diff --git a/src/zenml/integrations/pytorch/materializers/pytorch_module_materializer.py b/src/zenml/integrations/pytorch/materializers/pytorch_module_materializer.py index 38ffea18de6..86e36572c8d 100644 --- a/src/zenml/integrations/pytorch/materializers/pytorch_module_materializer.py +++ b/src/zenml/integrations/pytorch/materializers/pytorch_module_materializer.py @@ -61,7 +61,10 @@ def save(self, model: Module) -> None: with fileio.open( os.path.join(self.uri, CHECKPOINT_FILENAME), "wb" ) as f: - torch.save(model.state_dict(), f, pickle_module=cloudpickle) + # NOTE (security): The `torch.save` function uses `cloudpickle` as + # the default unpickler, which is NOT secure. This materializer + # is intended for use with trusted data sources. + torch.save(model.state_dict(), f, pickle_module=cloudpickle) # nosec def extract_metadata(self, model: Module) -> Dict[str, "MetadataType"]: """Extract metadata from the given `Model` object. diff --git a/src/zenml/zen_server/deploy/helm/Chart.yaml b/src/zenml/zen_server/deploy/helm/Chart.yaml index 71f5e69a166..43d77cdd201 100644 --- a/src/zenml/zen_server/deploy/helm/Chart.yaml +++ b/src/zenml/zen_server/deploy/helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: zenml -version: "0.66.0" +version: "0.67.0" description: Open source MLOps framework for portable production ready ML pipelines keywords: - mlops diff --git a/src/zenml/zen_server/deploy/helm/README.md b/src/zenml/zen_server/deploy/helm/README.md index 7a29a482279..c122990462d 100644 --- a/src/zenml/zen_server/deploy/helm/README.md +++ b/src/zenml/zen_server/deploy/helm/README.md @@ -20,8 +20,8 @@ ZenML is an open-source MLOps framework designed to help you create robust, main To install the ZenML chart directly from Amazon ECR, use the following command: ```bash -# example command for version 0.66.0 -helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.66.0 +# example command for version 0.67.0 +helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.67.0 ``` Note: Ensure you have OCI support enabled in your Helm client and that you are authenticated with Amazon ECR. diff --git a/src/zenml/zen_stores/migrations/versions/0.67.0_release.py b/src/zenml/zen_stores/migrations/versions/0.67.0_release.py new file mode 100644 index 00000000000..cbebe847b30 --- /dev/null +++ b/src/zenml/zen_stores/migrations/versions/0.67.0_release.py @@ -0,0 +1,23 @@ +"""Release [0.67.0]. + +Revision ID: 0.67.0 +Revises: 0.66.0 +Create Date: 2024-09-25 10:39:36.711715 + +""" + +# revision identifiers, used by Alembic. +revision = "0.67.0" +down_revision = "0.66.0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Upgrade database schema and/or data, creating a new revision.""" + pass + + +def downgrade() -> None: + """Downgrade database schema and/or data back to the previous revision.""" + pass