From b1901fc27d1c393ba99aa1ea2ca154dd48329af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bar=C4=B1=C5=9F=20Can=20Durak?= <36421093+bcdurak@users.noreply.github.com> Date: Mon, 21 Oct 2024 17:30:25 +0200 Subject: [PATCH] Automating the release process using Github workflows (#3101) * prepare release workflow * again * switching to push * new git diff * again * quickstart * adding trigger string * [run-ci] * first try git push [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * Adding the new version to the necessary files. * release continue * adding the changes bakc * starting the test flow * [test] * [test] * [test] * [test] * dockerfile for dev quickstart * new build structure * coming together * [build] * [build] * [build] * [build] * minor fixes * [build] * [build] * [build] * [build] * new workflow new script [build] * syntax fix [build] * [build] [test] * [test] * fixing the script [test] * fixing the script [test] * more fixes [test] * [test] * [test] * [test] * [test] * [test] * [test] * first full [test] run * first full [test] run * third full [test] run * fourth full [test] run * the [test] got the first bug * [test] * adding fail fast to the strategy * new things * pull correctly * minor fix * new sed * release prepare part * finalize checkpoint * adjusted TODOs * scripts fixed * new release finalizationqq * uncommented version * new gitbook sync script * new gitbook sync * adjusted todos and better warning message * clearing out the last todos * formatting * only count for prs merged to develop * fixed the discord messageqq * taking versioning into consideration * small fix to the tenant management script --------- Co-authored-by: ZenML GmbH --- .github/workflows/release_finalize.yml | 147 ++++++++++++ .github/workflows/release_prepare.yml | 214 ++++++++++++++++++ docker/zenml-quickstart-dev.Dockerfile | 28 +++ release-cloudbuild-preparation.yaml | 153 +++++++++++++ scripts/add-migration-test-version.sh | 42 ++++ scripts/deprecate-previous-docs-to-legacy.sh | 36 +++ scripts/redeploy-release-prep-tenant.py | 221 +++++++++++++++++++ scripts/sync-gitbook-release-spaces.py | 204 +++++++++++++++++ scripts/validate-new-version.sh | 59 +++++ 9 files changed, 1104 insertions(+) create mode 100644 .github/workflows/release_finalize.yml create mode 100644 .github/workflows/release_prepare.yml create mode 100644 docker/zenml-quickstart-dev.Dockerfile create mode 100644 release-cloudbuild-preparation.yaml create mode 100644 scripts/add-migration-test-version.sh create mode 100644 scripts/deprecate-previous-docs-to-legacy.sh create mode 100644 scripts/redeploy-release-prep-tenant.py create mode 100644 scripts/sync-gitbook-release-spaces.py create mode 100644 scripts/validate-new-version.sh diff --git a/.github/workflows/release_finalize.yml b/.github/workflows/release_finalize.yml new file mode 100644 index 00000000000..304cceb07f0 --- /dev/null +++ b/.github/workflows/release_finalize.yml @@ -0,0 +1,147 @@ +--- +name: release-finalize +on: + pull_request: + types: [closed] + branches: [misc/prepare-release-*] +env: + ZENML_ANALYTICS_OPT_IN: false +jobs: + fetch-versions: + if: github.repository == 'zenml-io/zenml' && github.event.pull_request.merged + == true + runs-on: ubuntu-latest + outputs: + old_version: ${{ steps.new-version.outputs.new_version }} + new_version: ${{ steps.old-version.outputs.old_version }} + steps: + # Extract the version + - name: Extract version from branch name + id: new-version + run: | + BRANCH_NAME=${GITHUB_REF#refs/heads/} + NEW_VERSION=${BRANCH_NAME#misc/prepare-release-} + echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV + echo "::set-output name=new_version::${{ env.NEW_VERSION }}" + # Checkout main as develop is already changed + - name: Checkout code + id: checkout-code + uses: actions/checkout@v4.1.1 + with: + ref: main + # Extract the old version + - name: Fetch the old version + id: old-version + run: | + OLD_VERSION=$(cat src/zenml/VERSION) + echo "OLD_VERSION=$OLD_VERSION" >> $GITHUB_ENV + echo "::set-output name=ideal_new_version::${{ env.OLD_VERSION }}" + create-release-branch: + needs: fetch-versions + runs-on: ubuntu-latest + steps: + # Configure Git + - name: Configure git + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + # Check out develop + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: develop + # Create the release branch + - name: Release branch + run: | + git checkout -b release/${{ needs.fetch-versions.outputs.new_version }} + git push --set-upstream origin release/${{ needs.fetch-versions.outputs.new_version }} + add-docs-warning-header: + needs: fetch-versions + runs-on: ubuntu-latest + steps: + # Configure Git + - name: Configure git + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + # Check out the previous release branch + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: release/${{ needs.fetch-versions.outputs.old_version }} + # Create the docs update PR + - name: Create docs update PR + shell: bash + run: | + bash scripts/add-docs-warning.sh ${{ needs.fetch-versions.outputs.old_version }} + add-new-version-to-migration-tests: + needs: fetch-versions + runs-on: ubuntu-latest + steps: + # Configure Git + - name: Configure git + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + # Check out develop + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: develop + # Create the migration test version if necessary + - name: Create docs update PR + shell: bash + run: |- + bash scripts/add-migration-test-version.sh ${{ needs.fetch-versions.outputs.old_version }} ${{ needs.fetch-versions.outputs.new_version }} + order-gitbook-release-spaces: + needs: fetch-versions + runs-on: ubuntu-latest + steps: + # Check out develop + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: develop + # Setting up the Python + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + # Install requests + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + # Adjust the docs + - name: Adjust gitbook docs + env: + ZENML_NEW_VERSION: ${{ needs.fetch-versions.outputs.new_version } + ZENML_OLD_VERSION: ${{ needs.fetch-versions.outputs.old_version } + GITBOOK_API_KEY: ${{secrets.GITBOOK_API_KEY}} + GITBOOK_ORGANIZATION: ${{secrets.GITBOOK_ORGANIZATION}} + GITBOOK_DOCS_COLLECTION: ${{secrets.GITBOOK_DOCS_COLLECTION}} + GITBOOK_LEGACY_COLLECTION: ${{secrets.GITBOOK_LEGACY_COLLECTION}} + run: python scripts/sync-gitbook-release-spaces.py + deprecate-docs-gitbook-legacy: + needs: [fetch-versions, order-gitbook-release-spaces] + runs-on: ubuntu-latest + steps: + # Configure Git + - name: Configure git + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + # Check out legacy docs branch + - name: Checkout code + uses: actions/checkout@v4.1.1 + with: + ref: docs/legacy-docs-page + # Append new version to the legacy docs table + - name: Update legacy docs file + shell: bash + run: |- + bash scripts/deprecate-previous-docs-to-legacy.sh ${{ needs.fetch-versions.outputs.old_version }} diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml new file mode 100644 index 00000000000..79d6533c3d6 --- /dev/null +++ b/.github/workflows/release_prepare.yml @@ -0,0 +1,214 @@ +--- +name: release-prepare +on: + pull_request: + types: [opened] + branches: [misc/prepare-release-*] +env: + ZENML_ANALYTICS_OPT_IN: false +jobs: + prepare-changes: + if: github.repository == 'zenml-io/zenml' + runs-on: ubuntu-latest + steps: + # Check out the code + - name: Checkout code + uses: actions/checkout@v4.1.1 + # Configure Git + - name: Configure git + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + # Extract the new version form the branch name + - name: Extract version from branch name + run: | + BRANCH_NAME=${GITHUB_REF#refs/heads/} + NEW_VERSION=${BRANCH_NAME#misc/prepare-release-} + echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV + # Validate the new version + - name: Validate new version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + run: | + scripts/validate-new-version.sh ${{ env.NEW_VERSION }} + # Send a message to Discord to alert everyone for the release + - name: Send message to Discord + run: | + curl -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "content": "[Prepare Release PR](${{ env.PR_URL }}) opened by @${{ env.PR_AUTHOR }}.\n\n@growth and @product, please do not merge anything to develop until the process is completed.", + "thread_name": "Preparing release: ${{ env.NEW_VERSION }}" + }' \ + ${{ secrets.DISCORD_WEBHOOK_RELEASE }} + # Set up Python + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: '3.12' + # Install ZenML + - name: Install ZenML and dependencies + shell: bash + run: | + scripts/install-zenml-dev.sh --system --integrations "no" + uv pip list + uv pip check || true + # Alembic migration file + - name: Run Alembic merge + shell: bash + run: | + alembic merge -m "Release" heads --rev-id ${{ env.NEW_VERSION }} + scripts/format.sh + git add src/zenml/zen_stores/migrations/versions + # Update the README, pyproject.toml, version and helm files + - name: Update main files + run: | + OLD_VERSION=$(cat src/zenml/VERSION) + echo "OLD_VERSION=$OLD_VERSION" >> $GITHUB_ENV + sed -i "s/${{ env.OLD_VERSION }}/${{ env.NEW_VERSION }}/g" README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md + git add README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md + # Update the Quickstart references + - name: Replace the references in the quickstart example + run: | + find examples/quickstart -type f \( -name "*.txt" -o -name "*.yaml" -o -name "*.ipynb" \) -print0 | + while IFS= read -r -d '' file; do + if [[ "$file" == *.ipynb ]]; then + # For .ipynb files, we need to parse JSON + jq --arg OLD ${{ env.OLD_VERSION }} --arg NEW ${{ env.NEW_VERSION }} \ + '(.cells[] | select(.cell_type == "code") | .source) |= map(gsub($OLD; $NEW))' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file" + else + # For .txt and .yaml files, we can use sed + sed -i "s/${{ env.OLD_VERSION }}/${{ env.NEW_VERSION }}/g" "$file" + fi + done + git add examples/quickstart + # Generate and append release notes + - name: Generate release notes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + RELEASE_NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes -F tag_name=${{ env.NEW_VERSION }} -F target_commitish=${{ github.sha }} -F previous_tag_name=${{ env.OLD_VERSION }} | jq -r '.body') + echo "$RELEASE_NOTES" >> RELEASE_NOTES.md + git add RELEASE_NOTES.md + # Push the changes + - name: Push the changes + run: | + git commit -m "Adding the new version to the necessary files." + git push origin HEAD:${{ github.event.pull_request.head.ref }} + build-test-images: + runs-on: ubuntu-latest + needs: prepare-changes + permissions: + contents: read + id-token: write + steps: + # Check out the prepare-release branch + - name: Checkout code + uses: actions/checkout@v4.1.1 + # Extract the new version form the branch name + - name: Extract version from branch name + run: | + BRANCH_NAME=${GITHUB_REF#refs/heads/} + NEW_VERSION=${BRANCH_NAME#misc/prepare-release-} + echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV + # Sign in to Google + - uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_CLOUDBUILD_EMAIL }} + service_account_key: ${{ secrets.GCP_CLOUDBUILD_KEY }} + project_id: ${{ secrets.GCP_CLOUDBUILD_PROJECT }} + # Submit the Cloudbuild job + - name: Build docker images + run: | + gcloud builds submit \ + --quiet \ + --config=release-cloudbuild-preparation.yaml \ + --substitutions=_ZENML_BRANCH=${{ github.event.pull_request.head.ref }} \ + --substitutions=_NEW_VERSION=${{ env.NEW_VERSION }} + # Sign in to AWS + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: arn:aws:iam::715803424590:role/gh-action-role-zenml-quickstart-ecr + aws-region: eu-central-1 + - name: Login to Amazon ECR + id: login-ecr + run: | + aws ecr get-login-password --region eu-central-1 | docker login 715803424590.dkr.ecr.eu-central-1.amazonaws.com --username AWS --password-stdin + # Publish the AWS image + - name: Pull quickstart image from Dockerhub + run: | + docker pull zenmldocker/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} + - name: Push quickstart image to ECR + run: | + docker tag zenmldocker/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} + docker push 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} + setup-prep-release-tenant: + needs: build-test-images + env: + ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }} + ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }} + runs-on: ubuntu-latest + steps: + # Check out the code + - name: Checkout code + uses: actions/checkout@v4.1.1 + # Setting up Python + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + # Install requests + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + # Deactivate and redeploy the tenant + - name: Run tenant management script + env: + CLOUD_STAGING_CLIENT_ID: ${{ secrets.CLOUD_STAGING_CLIENT_ID }} + CLOUD_STAGING_CLIENT_SECRET: ${{ secrets.CLOUD_STAGING_CLIENT_SECRET }} + RELEASE_TENANT_ID: ${{ secrets.RELEASE_TENANT_ID }} + run: python scripts/redeploy-release-prep-tenant.py + run_quickstart_pipelines: + needs: setup-prep-release-tenant + runs-on: ubuntu-latest + env: + ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }} + ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }} + strategy: + fail-fast: false + matrix: + include: + - cloud: aws + parent_image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} + - cloud: azure + parent_image: zenmldocker/prepare-release:quickstart-azure${{ env.NEW_VERSION }}-${{ + env.NEW_VERSION }} + - cloud: gcp + parent_image: zenmldocker/prepare-release:quickstart-gcp-${{ env.NEW_VERSION }} + steps: + # Check out the code + - name: Checkout code + uses: actions/checkout@v4.1.1 + # Setting up Python + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.12' + # ZenML Integrations + - name: Install ZenML and the required integrations + run: | + scripts/install-zenml-dev.sh --system --integrations "no" + # Run the Quickstart pipeline + - name: Run on ${{ matrix.cloud }} + run: |- + cd examples/quickstart + zenml stack set ${{ matrix.cloud }} + sed -i "s|parent_image:.*|parent_image: \"${{ matrix.parent_image }}\"|" "configs/training_${{ matrix.cloud }}.yaml" + sed -i 's|zenml\[server\]==[^[:space:]]*|git+https://github.com/zenml-io/zenml.git@${{ github.event.pull_request.head.ref }}#egg=zenml[server]|g' requirements_${{ matrix.cloud }}.txt + pip install -r requirements_${{ matrix.cloud }}.txt + zenml integration install ${{ matrix.cloud }} -y + python run.py --model_type=t5-small diff --git a/docker/zenml-quickstart-dev.Dockerfile b/docker/zenml-quickstart-dev.Dockerfile new file mode 100644 index 00000000000..a46fdec93e5 --- /dev/null +++ b/docker/zenml-quickstart-dev.Dockerfile @@ -0,0 +1,28 @@ +ARG BASE_IMAGE + +FROM $BASE_IMAGE AS base + +# Set the working directory +WORKDIR /app + +ARG ZENML_BRANCH +ARG CLOUD_PROVIDER + +# Install the Python requirements +RUN pip install uv + +RUN uv pip install "git+https://github.com/zenml-io/zenml.git@$ZENML_BRANCH" notebook pyarrow datasets transformers transformers[torch] torch sentencepiece + +RUN echo "Cloud Provider: $CLOUD_PROVIDER"; +# Install cloud-specific ZenML integrations +RUN if [ "$CLOUD_PROVIDER" = "aws" ]; then \ + zenml integration install aws s3 -y; \ + elif [ "$CLOUD_PROVIDER" = "azure" ]; then \ + zenml integration install azure -y; \ + elif [ "$CLOUD_PROVIDER" = "gcp" ]; then \ + zenml integration install gcp -y; \ + else \ + echo "No specific cloud integration installed"; \ + fi + +ENV ZENML_REQUIRES_CODE_DOWNLOAD=True \ No newline at end of file diff --git a/release-cloudbuild-preparation.yaml b/release-cloudbuild-preparation.yaml new file mode 100644 index 00000000000..50ea0de0b07 --- /dev/null +++ b/release-cloudbuild-preparation.yaml @@ -0,0 +1,153 @@ +steps: + # login to Dockerhub + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - docker login --username=$$USERNAME --password=$$PASSWORD + id: docker-login + entrypoint: bash + secretEnv: + - USERNAME + - PASSWORD + + # Build base image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build . \ + --platform linux/amd64 \ + -f docker/zenml-dev.Dockerfile \ + -t $$USERNAME/prepare-release:base-${_NEW_VERSION} + + id: build-base + waitFor: ['-'] + entrypoint: bash + secretEnv: + - USERNAME + + # Push base image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - docker push $$USERNAME/prepare-release:base-${_NEW_VERSION} + id: push-base + waitFor: + - docker-login + - build-base + entrypoint: bash + secretEnv: + - USERNAME + + # Build server image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build . \ + --platform linux/amd64 \ + -f docker/zenml-server-dev.Dockerfile \ + -t $$USERNAME/prepare-release:server-${_NEW_VERSION} + + id: build-server + waitFor: ['-'] + entrypoint: bash + secretEnv: + - USERNAME + + # Push server images + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - docker push $$USERNAME/prepare-release:server-${_NEW_VERSION} + id: push-server + waitFor: + - docker-login + - build-server + entrypoint: bash + secretEnv: + - USERNAME + + # Build Quickstart GCP Image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build . \ + --platform linux/amd64 \ + --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_NEW_VERSION} \ + --build-arg CLOUD_PROVIDER=gcp \ + --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ + -f docker/zenml-quickstart-dev.Dockerfile \ + -t $$USERNAME/prepare-release:quickstart-gcp-${_NEW_VERSION} + + id: build-quickstart-gcp + waitFor: + - push-base + entrypoint: bash + secretEnv: + - USERNAME + + # Build Quickstart AWS image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build . \ + --platform linux/amd64 \ + --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_NEW_VERSION} \ + --build-arg CLOUD_PROVIDER=aws \ + --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ + -f docker/zenml-quickstart-dev.Dockerfile \ + -t $$USERNAME/prepare-release:quickstart-aws-${_NEW_VERSION} + id: build-quickstart-aws + waitFor: + - push-base + entrypoint: bash + secretEnv: + - USERNAME + + # Build Quickstart Azure image + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker build . \ + --platform linux/amd64 \ + --build-arg BASE_IMAGE=$$USERNAME/prepare-release:base-${_NEW_VERSION} \ + --build-arg CLOUD_PROVIDER=azure \ + --build-arg ZENML_BRANCH=${_ZENML_BRANCH} \ + -f docker/zenml-quickstart-dev.Dockerfile \ + -t $$USERNAME/prepare-release:quickstart-azure-${_NEW_VERSION} + id: build-quickstart-azure + waitFor: + - push-base + entrypoint: bash + secretEnv: + - USERNAME + + # Push Quickstart images + - name: gcr.io/cloud-builders/docker + args: + - '-c' + - | + docker push $$USERNAME/prepare-release:quickstart-aws-${_NEW_VERSION} + docker push $$USERNAME/prepare-release:quickstart-azure-${_NEW_VERSION} + docker push $$USERNAME/prepare-release:quickstart-gcp-${_NEW_VERSION} + id: push-quickstart + waitFor: + - docker-login + - build-quickstart-gcp + - build-quickstart-aws + - build-quickstart-azure + entrypoint: bash + secretEnv: + - USERNAME + +timeout: 3600s +availableSecrets: + secretManager: + - versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1 + env: PASSWORD + - versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1 + env: USERNAME diff --git a/scripts/add-migration-test-version.sh b/scripts/add-migration-test-version.sh new file mode 100644 index 00000000000..9180407924a --- /dev/null +++ b/scripts/add-migration-test-version.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e +set -x +set -o pipefail + +# Check if both old and new version arguments are provided +if [ $# -ne 2 ]; then + echo "Error: Incorrect number of arguments. Usage: $0 " + exit 1 +fi + +OLD_VERSION=$1 +NEW_VERSION=$2 + +# Fetch the last changes in the alembic history +ALEMBIC_HISTORY=$(alembic history | head -n 1) + +# Check if the first line starts with the old version +if [[ $ALEMBIC_HISTORY == "$OLD_VERSION"* ]]; then + echo "Alembic history starts with $OLD_VERSION. No changes needed." + exit 0 +else + # Branch off + NEW_BRANCH="feature/adding-$NEW_VERSION-to-the-migration-tests" + git checkout -b "$NEW_BRANCH" + + # Add the new version to the VERSIONS list + sed -i '' "/^VERSIONS=(.*)$/s/)/ \"$NEW_VERSION\")/" scripts/test-migrations.sh + echo "Added new version $NEW_VERSION to scripts/test-migrations.sh" + + # Add, commit and push the new changes + git add scripts/test-migrations.sh + git commit -m "Adding the new version to the migration tests." + git push origin "$NEW_BRANCH" + + # Open up a pull request + gh pr create --base "develop" --head "$NEW_BRANCH" \ + --title "Add $NEW_VERSION to the migration tests" \ + --body "This PR adds $NEW_VERSION to the list of version in the migration test script." \ + --label "internal" + exit 0 +fi diff --git a/scripts/deprecate-previous-docs-to-legacy.sh b/scripts/deprecate-previous-docs-to-legacy.sh new file mode 100644 index 00000000000..38e4176bd0f --- /dev/null +++ b/scripts/deprecate-previous-docs-to-legacy.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e +set -x +set -o pipefail + +# Constants +FILE="docs/book/introduction.md" + +# Check if both old and new version arguments are provided +if [ $# -ne 2 ]; then + echo "Error: Incorrect number of arguments. Usage: $0 " + exit 1 +fi + +# Fetch the old version +OLD_VERSION=$1 + +# Create a new branch +NEW_BRANCH="docs/adding-$OLD_VERSION-to-the-legacy-docs" +git checkout -b "$NEW_BRANCH" + +# Create the new URL +LEGACY_URL="https://zenml-io.gitbook.io/zenml-legacy-documentation/v/${OLD_VERSION}/" + +# Add the old version card to the table +sed -i "/<\/tbody>/i${OLD_VERSION}${LEGACY_URL}" "$FILE" + +# Add, commit, push and PR +git add $FILE +git commit -m "Deprecating the docs for version ${OLD_VERSION} to the legacy docs." +git push origin "$NEW_BRANCH" + +gh pr create --base "docs/legacy-docs-page" --head "$NEW_BRANCH" \ + --title "Deprecate $NEW_VERSION docs to the legacy docs" \ + --body "This PR adds $NEW_VERSION to the legacy docs table and links it properly." \ + --label "internal" diff --git a/scripts/redeploy-release-prep-tenant.py b/scripts/redeploy-release-prep-tenant.py new file mode 100644 index 00000000000..e1995056505 --- /dev/null +++ b/scripts/redeploy-release-prep-tenant.py @@ -0,0 +1,221 @@ +# Copyright (c) ZenML GmbH 2024. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. +"""Deactivates and redeploys the release prep tenant with a new server image.""" + +import os +import time + +import requests + + +def get_token(client_id: str, client_secret: str) -> str: + """Get an access token for the staging API. + + Args: + client_id: The client ID for authentication. + client_secret: The client secret for authentication. + + Returns: + The access token as a string. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = "https://staging.cloudapi.zenml.io/auth/login" + data = { + "grant_type": "", + "client_id": client_id, + "client_secret": client_secret, + } + response = requests.post(url, data=data) + + if response.status_code != 200: + raise requests.HTTPError("There was a problem fetching the token.") + + return response.json()["access_token"] + + +def update_tenant(token: str, tenant_id: str, new_version: str) -> None: + """Update a specific tenant. + + Args: + token: The access token for authentication. + tenant_id: The ID of the tenant to update. + new_version: New version of ZenML to be released. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = f"https://staging.cloudapi.zenml.io/tenants/{tenant_id}" + + headers = { + "Authorization": f"Bearer {token}", + "accept": "application/json", + } + + data = { + "zenml_service": { + "admin": { + "image_repository": "dockerhub/prepare-release", + "image_tag": f"server-{new_version}", + }, + }, + } + + response = requests.patch(url, json=data, headers=headers) + if response.status_code != 200: + raise requests.HTTPError("There was a problem updating the token.") + + +def deactivate_tenant(token: str, tenant_id: str) -> None: + """Deactivate a specific tenant. + + Args: + token: The access token for authentication. + tenant_id: The ID of the tenant to deactivate. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = f"https://staging.cloudapi.zenml.io/tenants/{tenant_id}/deactivate" + headers = { + "Authorization": f"Bearer {token}", + "accept": "application/json", + } + response = requests.patch(url, headers=headers) + + if response.status_code != 200: + raise requests.HTTPError( + "There was a problem deactivating the tenant." + ) + + +def get_tenant_status(token: str, tenant_id: str) -> str: + """Get the current status of a specific tenant. + + Args: + token: The access token for authentication. + tenant_id: The ID of the tenant to check. + + Returns: + The current status of the tenant as a string. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = f"https://staging.cloudapi.zenml.io/tenants/{tenant_id}" + headers = { + "Authorization": f"Bearer {token}", + "accept": "application/json", + } + response = requests.get(url, headers=headers) + + if response.status_code != 200: + raise requests.HTTPError("There was a problem fetching the status.") + + return response.json()["status"] + + +def redeploy_tenant(token: str, tenant_id: str) -> None: + """Redeploy a specific tenant. + + Args: + token: The access token for authentication. + tenant_id: The ID of the tenant to redeploy. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = f"https://staging.cloudapi.zenml.io/tenants/{tenant_id}/deploy" + headers = { + "Authorization": f"Bearer {token}", + "accept": "application/json", + } + response = requests.patch(url, headers=headers) + + if response.status_code != 200: + raise requests.HTTPError("There was a problem redeploying the tenant.") + + +def main() -> None: + """Main function to orchestrate the tenant management process. + + This function performs the following steps: + 1. Retrieves necessary environment variables. + 2. Gets an access token. + 3. Deactivates the specified tenant. + 4. Waits for the tenant to be fully deactivated. + 5. Redeploys the tenant. + 6. Waits for the tenant to be fully deployed. + + Raises: + EnvironmentError: If required environment variables are missing. + requests.HTTPError: If any API requests fail. + """ + # Get environment variables + client_id = os.environ.get("CLOUD_STAGING_CLIENT_ID") + client_secret = os.environ.get("CLOUD_STAGING_CLIENT_SECRET") + tenant_id = os.environ.get("RELEASE_TENANT_ID") + new_version = os.environ.get("ZENML_NEW_VERSION") + + if not all([client_id, client_secret, tenant_id, new_version]): + raise EnvironmentError("Missing required environment variables") + + # Get the token + token = get_token(client_id, client_secret) + print("Fetched the token.") + + # Deactivate the tenant + status = get_tenant_status(token, tenant_id) + if status == "available": + deactivate_tenant(token, tenant_id) + print("Tenant deactivation initiated.") + + # Wait until it's deactivated + time.sleep(10) + + status = get_tenant_status(token, tenant_id) + while status == "pending": + print(f"Waiting... Current tenant status: {status}.") + time.sleep(20) + status = get_tenant_status(token, tenant_id) + + if status != "deactivated": + raise RuntimeError("Tenant deactivation failed.") + print("Tenant deactivated.") + + # Update the tenant + update_tenant(token, tenant_id, new_version) + print("Tenant updated.") + + # Redeploy the tenant + redeploy_tenant(token, tenant_id) + print("Tenant redeployment initiated.") + + # Wait until it's deployed + time.sleep(10) + + status = get_tenant_status(token, tenant_id) + while status == "pending": + print(f"Waiting... Current tenant status: {status}.") + time.sleep(20) + status = get_tenant_status(token, tenant_id) + + if status != "available": + raise RuntimeError("Tenant redeployment failed.") + print("Tenant redeployed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/sync-gitbook-release-spaces.py b/scripts/sync-gitbook-release-spaces.py new file mode 100644 index 00000000000..e3eb9370a07 --- /dev/null +++ b/scripts/sync-gitbook-release-spaces.py @@ -0,0 +1,204 @@ +# Copyright (c) ZenML GmbH 2024. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. +"""Syncs the Gitbook spaces accordingly for the new release.""" + +import os + +import requests + +# Constants +BASE_URL = "https://api.gitbook.com/v1" + + +def get_space_id( + name: str, collection: str, organization: str, headers: dict +) -> str: + """Get the ID of a Gitbook space based on its name and collection. + + Args: + name: The name of the space to find. + collection: The ID of the collection containing the space. + organization: The ID of the organization. + headers: The headers for the API request. + + Returns: + The ID of the space as a string. + + Raises: + StopIteration: If the space is not found. + """ + # Make the initial list-spaces call + params = {"limit": 50} + + while True: + response = requests.get( + f"{BASE_URL}/orgs/{organization}/spaces", + headers=headers, + params=params, + ).json() + + # Iterate through the spaces in the current page + for space in response["items"]: + if ( + space.get("parent", None) == collection + and space.get("title", None) == name + ): + return space["id"] + + # Check if there are more pages + if "next" not in response or not response["next"]: + # If no more pages and space not found, raise StopIteration + raise StopIteration( + f"Space '{name}' not found in collection '{collection}'" + ) + + # If space not found in current page, update params for next page + params.update(response["next"]) + + +def duplicate_space(space_id: str, headers: dict) -> str: + """Duplicate a Gitbook space. + + Args: + space_id: The ID of the space to duplicate. + headers: The headers for the API request. + + Returns: + The ID of the newly created space as a string. + + Raises: + requests.HTTPError: If the API request fails. + """ + response = requests.post( + f"{BASE_URL}/spaces/{space_id}/duplicate", headers=headers + ) + if response.status_code != 200: + raise requests.HTTPError("There was a problem duplicating the space.") + return response.json()["id"] + + +def update_space(space_id: str, changes: dict, headers: dict) -> None: + """Update a Gitbook space with the provided changes. + + Args: + space_id: The ID of the space to update. + changes: A dictionary containing the changes to apply. + headers: The headers for the API request. + + Raises: + requests.HTTPError: If the API request fails. + """ + response = requests.patch( + f"{BASE_URL}/spaces/{space_id}", headers=headers, json=changes + ) + if response.status_code != 200: + raise requests.HTTPError("There was a problem updating the space.") + + +def move_space( + space_id: str, target_collection_id: str, headers: dict +) -> None: + """Move a Gitbook space to a different collection. + + Args: + space_id: The ID of the space to move. + target_collection_id: The ID of the collection to move the space to. + headers: The headers for the API request. + + Raises: + requests.HTTPError: If the API request fails. + """ + # Define the endpoint URL + url = f"{BASE_URL}/spaces/{space_id}/move" + + # Create the payload for the request + payload = {"parent": target_collection_id} + + # Make the POST request to move the space + response = requests.post(url, headers=headers, json=payload) + + if response.status_code != 200: + raise requests.HTTPError("There was a problem moving the space.") + + +def main() -> None: + """Main function to sync Gitbook spaces for a new release. + + This function performs the following steps: + 1. Get the Space ID of the previous release. + 2. Duplicate the previous release space. + 3. Rename the duplicate to the new version. + 4. Move the previous release to the legacy collection. + + Raises: + EnvironmentError: If any required environment variables are missing. + """ + # Get environment variables + zenml_new_version = os.environ.get("ZENML_NEW_VERSION") + zenml_old_version = os.environ.get("ZENML_OLD_VERSION") + gitbook_api_key = os.environ.get("GITBOOK_API_KEY") + gitbook_organization = os.environ.get("GITBOOK_ORGANIZATION") + gitbook_docs_collection = os.environ.get("GITBOOK_DOCS_COLLECTION") + gitbook_legacy_collection = os.environ.get("GITBOOK_LEGACY_COLLECTION") + + # Check if all required environment variables are set + if not all( + [ + zenml_new_version, + zenml_old_version, + gitbook_api_key, + gitbook_organization, + gitbook_docs_collection, + gitbook_legacy_collection, + ] + ): + raise EnvironmentError("Missing required environment variables") + + # Create the headers for API requests + headers = { + "Authorization": f"Bearer {gitbook_api_key}", + "Content-Type": "application/json", + } + + # 1. Get the Space ID of the previous release + previous_release_space_id = get_space_id( + name=zenml_old_version, + collection=gitbook_docs_collection, + organization=gitbook_organization, + headers=headers, + ) + + # 2. Duplicate the previous release space + new_release_space_id = duplicate_space( + space_id=previous_release_space_id, + headers=headers, + ) + + # 3: Rename the duplicate to the new name + update_space( + space_id=new_release_space_id, + changes={"title": zenml_new_version}, + headers=headers, + ) + + # 4: Move the previous release to the legacy collection + move_space( + space_id=previous_release_space_id, + target_collection_id=gitbook_legacy_collection, + headers=headers, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate-new-version.sh b/scripts/validate-new-version.sh new file mode 100644 index 00000000000..de624c7ef44 --- /dev/null +++ b/scripts/validate-new-version.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -e +set -o pipefail + +# Constants +LABEL_BREAKING_CHANGE="breaking-change" + +# Check if the planned version is passed as an argument +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Input: new planned version +PLANNED_VERSION="$1" + +# Get the latest release information +LATEST_RELEASE=$(gh release view --json tagName,publishedAt -q '{tag: .tagName, date: .publishedAt}') + +# Extract the tag and date from the latest release JSON +TAG=$(echo "$LATEST_RELEASE" | jq -r .tag) +TAG_DATE=$(echo "$LATEST_RELEASE" | jq -r .date) + +# Check if the latest release information was retrieved successfully +if [ "$TAG" == "null" ]; then + echo "No releases found." + exit 1 +fi + +# Parse the latest tag for versioning +IFS='.' read -r MAJOR MINOR PATCH <<< "$TAG" +IFS='.' read -r P_MAJOR P_MINOR P_PATCH <<< "$PLANNED_VERSION" + +# List PRs merged after the latest release date and filter by breaking change label +PRS=$(gh pr list --search "is:merged merged:>$TAG_DATE label:$LABEL_BREAKING_CHANGE base:develop" --json number,title) + +# Check if the PRs list for breaking changes is empty +if [ -z "$PRS" ]; then + IDEAL_NEW_VERSION="$MAJOR.$MINOR.$((PATCH + 1))" + + # Warn in case of a mismatch + if [ "$PLANNED_VERSION" != "$IDEAL_NEW_VERSION" ]; then + echo "::warning::Warning: The new planned version '$PLANNED_VERSION' is not the same as the ideal version '$IDEAL_NEW_VERSION'." + fi +else + IDEAL_NEW_VERSION="$MAJOR.$((MINOR + 1)).0" + + # Fail in case of a mismatch + if [ "$P_MINOR" -le "$MINOR" ] && [ "$P_MAJOR" -le "$MAJOR" ]; then + PR_NUMBERS=$(echo "$PRS" | jq -r 'map("\(.number):\(.title)") | join(", ")') + echo "::error::Error: There are PRs ($PR_NUMBERS) with breaking changes merged after the release '$TAG'. In this case, the new ideal version is '$IDEAL_NEW_VERSION' instead of '$PLANNED_VERSION'.""" + exit 1 + fi + + if [ "$PLANNED_VERSION" != "$IDEAL_NEW_VERSION" ]; then + echo "::warning::Warning: The new planned version '$PLANNED_VERSION' is not the same as the ideal version '$IDEAL_NEW_VERSION'. Please, make sure this '$PLANNED_VERSION' is selected on purpose." + fi + +fi