Automating the release process using Github workflows (#3101)

* prepare release workflow * again * switching to push * new git diff * again * quickstart * adding trigger string * [run-ci] * first try git push [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * [run-ci] * Adding the new version to the necessary files. * release continue * adding the changes bakc * starting the test flow * [test] * [test] * [test] * [test] * dockerfile for dev quickstart * new build structure * coming together * [build] * [build] * [build] * [build] * minor fixes * [build] * [build] * [build] * [build] * new workflow new script [build] * syntax fix [build] * [build] [test] * [test] * fixing the script [test] * fixing the script [test] * more fixes [test] * [test] * [test] * [test] * [test] * [test] * [test] * first full [test] run * first full [test] run * third full [test] run * fourth full [test] run * the [test] got the first bug * [test] * adding fail fast to the strategy * new things * pull correctly * minor fix * new sed * release prepare part * finalize checkpoint * adjusted TODOs * scripts fixed * new release finalizationqq * uncommented version * new gitbook sync script * new gitbook sync * adjusted todos and better warning message * clearing out the last todos * formatting * only count for prs merged to develop * fixed the discord messageqq * taking versioning into consideration * small fix to the tenant management script --------- Co-authored-by: ZenML GmbH <info@zenml.io>
zenml-io · Oct 21, 2024 · b1901fc · b1901fc
1 parent 30f1765
commit b1901fc
Show file tree

Hide file tree

Showing 9 changed files with 1,104 additions and 0 deletions.
diff --git a/.github/workflows/release_finalize.yml b/.github/workflows/release_finalize.yml
@@ -0,0 +1,147 @@
+---
+name: release-finalize
+on:
+  pull_request:
+    types: [closed]
+    branches: [misc/prepare-release-*]
+env:
+  ZENML_ANALYTICS_OPT_IN: false
+jobs:
+  fetch-versions:
+    if: github.repository == 'zenml-io/zenml' && github.event.pull_request.merged
+      == true
+    runs-on: ubuntu-latest
+    outputs:
+      old_version: ${{ steps.new-version.outputs.new_version }}
+      new_version: ${{ steps.old-version.outputs.old_version }}
+    steps:
+      # Extract the version
+      - name: Extract version from branch name
+        id: new-version
+        run: |
+          BRANCH_NAME=${GITHUB_REF#refs/heads/}
+          NEW_VERSION=${BRANCH_NAME#misc/prepare-release-}
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+          echo "::set-output name=new_version::${{ env.NEW_VERSION }}"
+      # Checkout main as develop is already changed
+      - name: Checkout code
+        id: checkout-code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: main
+      # Extract the old version
+      - name: Fetch the old version
+        id: old-version
+        run: |
+          OLD_VERSION=$(cat src/zenml/VERSION)
+          echo "OLD_VERSION=$OLD_VERSION" >> $GITHUB_ENV
+          echo "::set-output name=ideal_new_version::${{ env.OLD_VERSION }}"
+  create-release-branch:
+    needs: fetch-versions
+    runs-on: ubuntu-latest
+    steps:
+      # Configure Git
+      - name: Configure git
+        shell: bash
+        run: |
+          git config --global user.email "info@zenml.io"
+          git config --global user.name "ZenML GmbH"
+      # Check out develop
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: develop
+      # Create the release branch
+      - name: Release branch
+        run: |
+          git checkout -b release/${{ needs.fetch-versions.outputs.new_version }}
+          git push --set-upstream origin release/${{ needs.fetch-versions.outputs.new_version }}
+  add-docs-warning-header:
+    needs: fetch-versions
+    runs-on: ubuntu-latest
+    steps:
+      # Configure Git
+      - name: Configure git
+        shell: bash
+        run: |
+          git config --global user.email "info@zenml.io"
+          git config --global user.name "ZenML GmbH"
+      # Check out the previous release branch
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: release/${{ needs.fetch-versions.outputs.old_version }}
+      # Create the docs update PR
+      - name: Create docs update PR
+        shell: bash
+        run: |
+          bash scripts/add-docs-warning.sh ${{ needs.fetch-versions.outputs.old_version }}
+  add-new-version-to-migration-tests:
+    needs: fetch-versions
+    runs-on: ubuntu-latest
+    steps:
+      # Configure Git
+      - name: Configure git
+        shell: bash
+        run: |
+          git config --global user.email "info@zenml.io"
+          git config --global user.name "ZenML GmbH"
+      # Check out develop
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: develop
+      # Create the migration test version if necessary
+      - name: Create docs update PR
+        shell: bash
+        run: |-
+          bash scripts/add-migration-test-version.sh ${{ needs.fetch-versions.outputs.old_version }} ${{ needs.fetch-versions.outputs.new_version }}
+  order-gitbook-release-spaces:
+    needs: fetch-versions
+    runs-on: ubuntu-latest
+    steps:
+      # Check out develop
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: develop
+      # Setting up the Python
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      # Install requests
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+      # Adjust the docs
+      - name: Adjust gitbook docs
+        env:
+          ZENML_NEW_VERSION: ${{ needs.fetch-versions.outputs.new_version }
+          ZENML_OLD_VERSION: ${{ needs.fetch-versions.outputs.old_version }
+          GITBOOK_API_KEY: ${{secrets.GITBOOK_API_KEY}}
+          GITBOOK_ORGANIZATION: ${{secrets.GITBOOK_ORGANIZATION}}
+          GITBOOK_DOCS_COLLECTION: ${{secrets.GITBOOK_DOCS_COLLECTION}}
+          GITBOOK_LEGACY_COLLECTION: ${{secrets.GITBOOK_LEGACY_COLLECTION}}
+        run: python scripts/sync-gitbook-release-spaces.py
+  deprecate-docs-gitbook-legacy:
+    needs: [fetch-versions, order-gitbook-release-spaces]
+    runs-on: ubuntu-latest
+    steps:
+      # Configure Git
+      - name: Configure git
+        shell: bash
+        run: |
+          git config --global user.email "info@zenml.io"
+          git config --global user.name "ZenML GmbH"
+      # Check out legacy docs branch
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+        with:
+          ref: docs/legacy-docs-page
+      # Append new version to the legacy docs table
+      - name: Update legacy docs file
+        shell: bash
+        run: |-
+          bash scripts/deprecate-previous-docs-to-legacy.sh ${{ needs.fetch-versions.outputs.old_version }}
diff --git a/.github/workflows/release_prepare.yml b/.github/workflows/release_prepare.yml
@@ -0,0 +1,214 @@
+---
+name: release-prepare
+on:
+  pull_request:
+    types: [opened]
+    branches: [misc/prepare-release-*]
+env:
+  ZENML_ANALYTICS_OPT_IN: false
+jobs:
+  prepare-changes:
+    if: github.repository == 'zenml-io/zenml'
+    runs-on: ubuntu-latest
+    steps:
+      # Check out the code
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      # Configure Git
+      - name: Configure git
+        shell: bash
+        run: |
+          git config --global user.email "info@zenml.io"
+          git config --global user.name "ZenML GmbH"
+      # Extract the new version form the branch name
+      - name: Extract version from branch name
+        run: |
+          BRANCH_NAME=${GITHUB_REF#refs/heads/}
+          NEW_VERSION=${BRANCH_NAME#misc/prepare-release-}
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+      # Validate the new version
+      - name: Validate new version
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          scripts/validate-new-version.sh ${{ env.NEW_VERSION }}
+      # Send a message to Discord to alert everyone for the release
+      - name: Send message to Discord
+        run: |
+          curl -X POST \
+           -H "Content-Type: application/json" \
+           -d '{
+             "content": "[Prepare Release PR](${{ env.PR_URL }}) opened by @${{ env.PR_AUTHOR }}.\n\n@growth and @product, please do not merge anything to develop until the process is completed.",
+             "thread_name": "Preparing release: ${{ env.NEW_VERSION }}"
+           }' \
+           ${{ secrets.DISCORD_WEBHOOK_RELEASE }}
+      # Set up Python
+      - name: Set up Python
+        uses: actions/setup-python@v5.0.0
+        with:
+          python-version: '3.12'
+      # Install ZenML
+      - name: Install ZenML and dependencies
+        shell: bash
+        run: |
+          scripts/install-zenml-dev.sh --system --integrations "no"
+          uv pip list
+          uv pip check || true
+      # Alembic migration file
+      - name: Run Alembic merge
+        shell: bash
+        run: |
+          alembic merge -m "Release" heads --rev-id ${{ env.NEW_VERSION }}
+          scripts/format.sh
+          git add src/zenml/zen_stores/migrations/versions
+      # Update the README, pyproject.toml, version and helm files
+      - name: Update main files
+        run: |
+          OLD_VERSION=$(cat src/zenml/VERSION)
+          echo "OLD_VERSION=$OLD_VERSION" >> $GITHUB_ENV
+          sed -i "s/${{ env.OLD_VERSION }}/${{ env.NEW_VERSION }}/g" README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md
+          git add README.md pyproject.toml src/zenml/VERSION src/zenml/zen_server/deploy/helm/Chart.yaml src/zenml/zen_server/deploy/helm/README.md
+      # Update the Quickstart references
+      - name: Replace the references in the quickstart example
+        run: |
+          find examples/quickstart -type f \( -name "*.txt" -o -name "*.yaml" -o -name "*.ipynb" \) -print0 | 
+          while IFS= read -r -d '' file; do
+            if [[ "$file" == *.ipynb ]]; then
+              # For .ipynb files, we need to parse JSON
+              jq --arg OLD ${{ env.OLD_VERSION }} --arg NEW ${{ env.NEW_VERSION }} \
+                '(.cells[] | select(.cell_type == "code") | .source) |= map(gsub($OLD; $NEW))' "$file" > "${file}.tmp" && mv "${file}.tmp" "$file"
+            else
+              # For .txt and .yaml files, we can use sed
+              sed -i "s/${{ env.OLD_VERSION }}/${{ env.NEW_VERSION }}/g" "$file"
+            fi
+          done
+          git add examples/quickstart
+      # Generate and append release notes
+      - name: Generate release notes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          RELEASE_NOTES=$(gh api repos/${{ github.repository }}/releases/generate-notes -F tag_name=${{ env.NEW_VERSION }} -F target_commitish=${{ github.sha }} -F previous_tag_name=${{ env.OLD_VERSION }} | jq -r '.body')
+          echo "$RELEASE_NOTES" >> RELEASE_NOTES.md
+          git add RELEASE_NOTES.md
+      # Push the changes
+      - name: Push the changes
+        run: |
+          git commit -m "Adding the new version to the necessary files."
+          git push origin HEAD:${{ github.event.pull_request.head.ref }}
+  build-test-images:
+    runs-on: ubuntu-latest
+    needs: prepare-changes
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      # Check out the prepare-release branch
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      # Extract the new version form the branch name
+      - name: Extract version from branch name
+        run: |
+          BRANCH_NAME=${GITHUB_REF#refs/heads/}
+          NEW_VERSION=${BRANCH_NAME#misc/prepare-release-}
+          echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
+      # Sign in to Google
+      - uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_email: ${{ secrets.GCP_CLOUDBUILD_EMAIL }}
+          service_account_key: ${{ secrets.GCP_CLOUDBUILD_KEY }}
+          project_id: ${{ secrets.GCP_CLOUDBUILD_PROJECT }}
+      # Submit the Cloudbuild job
+      - name: Build docker images
+        run: |
+          gcloud builds submit \
+            --quiet \
+            --config=release-cloudbuild-preparation.yaml \
+            --substitutions=_ZENML_BRANCH=${{ github.event.pull_request.head.ref }} \
+            --substitutions=_NEW_VERSION=${{ env.NEW_VERSION }}
+      # Sign in to AWS
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: arn:aws:iam::715803424590:role/gh-action-role-zenml-quickstart-ecr
+          aws-region: eu-central-1
+      - name: Login to Amazon ECR
+        id: login-ecr
+        run: |
+          aws ecr get-login-password --region eu-central-1 | docker login 715803424590.dkr.ecr.eu-central-1.amazonaws.com --username AWS --password-stdin
+      # Publish the AWS image
+      - name: Pull quickstart image from Dockerhub
+        run: |
+          docker pull zenmldocker/prepare-release:quickstart-aws-${{ env.NEW_VERSION }}
+      - name: Push quickstart image to ECR
+        run: |
+          docker tag zenmldocker/prepare-release:quickstart-aws-${{ env.NEW_VERSION }} 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }}
+          docker push 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }}
+  setup-prep-release-tenant:
+    needs: build-test-images
+    env:
+      ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }}
+      ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }}
+    runs-on: ubuntu-latest
+    steps:
+      # Check out the code
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      # Setting up Python
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      # Install requests
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+      # Deactivate and redeploy the tenant
+      - name: Run tenant management script
+        env:
+          CLOUD_STAGING_CLIENT_ID: ${{ secrets.CLOUD_STAGING_CLIENT_ID }}
+          CLOUD_STAGING_CLIENT_SECRET: ${{ secrets.CLOUD_STAGING_CLIENT_SECRET }}
+          RELEASE_TENANT_ID: ${{ secrets.RELEASE_TENANT_ID }}
+        run: python scripts/redeploy-release-prep-tenant.py
+  run_quickstart_pipelines:
+    needs: setup-prep-release-tenant
+    runs-on: ubuntu-latest
+    env:
+      ZENML_STORE_URL: ${{ secrets.RELEASE_TENANT_URL }}
+      ZENML_STORE_API_KEY: ${{ secrets.RELEASE_TENANT_SERVICE_ACCOUNT_KEY }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - cloud: aws
+            parent_image: 715803424590.dkr.ecr.eu-central-1.amazonaws.com/prepare-release:quickstart-aws-${{ env.NEW_VERSION }}
+          - cloud: azure
+            parent_image: zenmldocker/prepare-release:quickstart-azure${{ env.NEW_VERSION }}-${{
+              env.NEW_VERSION }}
+          - cloud: gcp
+            parent_image: zenmldocker/prepare-release:quickstart-gcp-${{ env.NEW_VERSION }}
+    steps:
+      # Check out the code
+      - name: Checkout code
+        uses: actions/checkout@v4.1.1
+      # Setting up Python
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+      # ZenML Integrations
+      - name: Install ZenML and the required integrations
+        run: |
+          scripts/install-zenml-dev.sh --system --integrations "no"
+      # Run the Quickstart pipeline
+      - name: Run on ${{ matrix.cloud }}
+        run: |-
+          cd examples/quickstart
+          zenml stack set ${{ matrix.cloud }}
+          sed -i "s|parent_image:.*|parent_image: \"${{ matrix.parent_image }}\"|" "configs/training_${{ matrix.cloud }}.yaml"
+          sed -i 's|zenml\[server\]==[^[:space:]]*|git+https://github.com/zenml-io/zenml.git@${{ github.event.pull_request.head.ref }}#egg=zenml[server]|g' requirements_${{ matrix.cloud }}.txt
+          pip install -r requirements_${{ matrix.cloud }}.txt
+          zenml integration install ${{ matrix.cloud }} -y
+          python run.py --model_type=t5-small
diff --git a/docker/zenml-quickstart-dev.Dockerfile b/docker/zenml-quickstart-dev.Dockerfile
@@ -0,0 +1,28 @@
+ARG BASE_IMAGE
+
+FROM $BASE_IMAGE AS base
+
+# Set the working directory
+WORKDIR /app
+
+ARG ZENML_BRANCH
+ARG CLOUD_PROVIDER
+
+# Install the Python requirements
+RUN pip install uv
+
+RUN uv pip install "git+https://github.com/zenml-io/zenml.git@$ZENML_BRANCH" notebook pyarrow datasets transformers transformers[torch] torch sentencepiece
+
+RUN echo "Cloud Provider: $CLOUD_PROVIDER";
+# Install cloud-specific ZenML integrations
+RUN if [ "$CLOUD_PROVIDER" = "aws" ]; then \
+        zenml integration install aws s3 -y; \
+    elif [ "$CLOUD_PROVIDER" = "azure" ]; then \
+        zenml integration install azure -y; \
+    elif [ "$CLOUD_PROVIDER" = "gcp" ]; then \
+        zenml integration install gcp -y; \
+    else \
+        echo "No specific cloud integration installed"; \
+    fi
+
+ENV ZENML_REQUIRES_CODE_DOWNLOAD=True