Nightly builds (#2031)

* v1

* nightly build workflow

* poetry install comes earlier

* refactor to remove duplicate docker build workflow

* update workflow filename

.github/workflows/nightly_build.yml

@@ -0,0 +1,36 @@
+name: Nightly Build
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run this workflow every day at midnight
+    - cron: '0 0 * * *'
+
+jobs:
+    setup-and-test:
+        uses: ./.github/workflows/setup-python-environment.yml
+        with:
+          os: "ubuntu-latest"
+          python-version: "3.8"
+          git-ref: "develop"
+        secrets: inherit
+
+    publish-python-package:
+        needs: setup-and-test
+        uses: ./.github/workflows/publish_to_pypi_nightly.yml
+        secrets: inherit
+
+    wait-for-package-release:
+        runs-on: ubuntu-latest
+        needs: publish-python-package
+        steps:
+          - name: Sleep for 4 minutes
+            run: sleep 240
+            shell: bash
+
+    publish-docker-image:
+        needs: wait-for-package-release
+        uses: ./.github/workflows/publish_docker_image.yml
+        with:
+          - config_file: release-cloudbuild-nightly.yaml
+        secrets: inherit

.github/workflows/publish_docker_image.yml

@@ -1,9 +1,16 @@
-name: Publish Docker image
+name: Publish Docker images
 
-on: workflow_call
+on:
+  workflow_call:
+    inputs:
+      config_file:
+        description: 'YAML config for Google Cloud Build'
+        required: false
+        type: string
+        default: "release-cloudbuild.yaml"
 
 jobs:
-  publish_to_pypi:
+  publish_to_docker:
     name: Publish Docker 🐋 image 🖼️ to Dockerhub
     runs-on: ubuntu-latest
     env:
@@ -30,5 +37,5 @@ jobs:
         run: |-
           gcloud builds submit \
             --quiet \
-            --config=release-cloudbuild.yaml \
+            --config=${{inputs.config_file}} \
             --substitutions=TAG_NAME=${{github.ref_name}} .

.github/workflows/publish_to_pypi_nightly.yml

@@ -0,0 +1,62 @@
+name: Publish Nightly PyPI package
+
+on:
+  workflow_call:
+
+jobs:
+  publish_to_pypi:
+    name: Publish Nightly Python 🐍 package 📦 to PyPI
+    runs-on: ubuntu-latest
+    env:
+      ZENML_DEBUG: 1
+      ZENML_ANALYTICS_OPT_IN: false
+      PYTHONIOENCODING: 'utf-8'
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: 'develop'
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Set nightly version
+        run: |
+            # Extract the current version
+            CURRENT_VERSION=$(poetry version -s)
+
+            # Get the current date in the format of YYYY-MM-DD
+            DATE=$(date +"%Y-%m-%d")
+
+            # Combine the current version with the date to form
+            # the new version string
+            NIGHTLY_VERSION="${CURRENT_VERSION}-${DATE}"
+
+            # Set the nightly version
+            echo "NIGHTLY_VERSION=$NIGHTLY_VERSION" >> $GITHUB_ENV
+
+      - name: Modify pyproject.toml for nightly release
+        run: |
+              # Change the package name to `zenml-nightly`
+              sed -i 's/name = "zenml"/name = "zenml-nightly"/' pyproject.toml
+
+              # Update the version to the nightly version
+              poetry version $NIGHTLY_VERSION
+
+      - name: Include latest dashboard
+        run: bash scripts/install-dashboard.sh
+
+      - name: publish
+        env:
+          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: bash scripts/publish.sh
+

.github/workflows/setup-python-environment.yml

@@ -53,6 +53,11 @@ on:
           - 'before-tests'
         required: false
         default: 'no'
+      git-ref:
+        description: 'Git branch or ref'
+        type: string
+        required: false
+        default: ''
 
 jobs:
   setup-and-test:
@@ -75,6 +80,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.inputs.git-ref || github.ref }}
 
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@main

release-cloudbuild-nightly.yaml

@@ -0,0 +1,75 @@
+steps:
+# build base image - python 3.10
+- name: gcr.io/cloud-builders/docker
+  args:
+  - '-c'
+  - |
+    docker build \
+      --build-arg ZENML_VERSION=$TAG_NAME-nightly \
+      --build-arg PYTHON_VERSION=3.10 \
+      --target base \
+      -f docker/base.Dockerfile . \
+      -t $$USERNAME/zenml:$TAG_NAME-py3.10-nightly
+
+    # no need to check for official release regex, this is for nightly builds
+    docker tag $$USERNAME/zenml:$TAG_NAME-py3.10-nightly $$USERNAME/zenml:py3.10-nightly
+
+  id: build-base-3.10-nightly
+  waitFor: ['-']
+  entrypoint: bash
+  secretEnv:
+  - USERNAME
+
+# build server image - python 3.8 only
+- name: gcr.io/cloud-builders/docker
+  args:
+  - '-c'
+  - "docker build \\\n--build-arg ZENML_VERSION=$TAG_NAME-nightly \\\n--build-arg PYTHON_VERSION=3.8 \\\n-f docker/base.Dockerfile . \\\n-t $$USERNAME/zenml-server:$TAG_NAME-nightly\n  #magic___^_^___line\n# use latest tags only for official releases"
+  id: build-server-nightly
+  waitFor: ['-']
+  entrypoint: bash
+  secretEnv:
+  - USERNAME
+
+# login to Dockerhub
+- name: gcr.io/cloud-builders/docker
+  args:
+  - '-c'
+  - docker login --username=$$USERNAME --password=$$PASSWORD
+  id: docker-login-nightly
+  entrypoint: bash
+  secretEnv:
+  - USERNAME
+  - PASSWORD
+
+# push base images
+- name: gcr.io/cloud-builders/docker
+  args:
+  - '-c'
+  - docker push --all-tags $$USERNAME/zenml
+  id: push-base
+  waitFor:
+  - docker-login
+  entrypoint: bash
+  secretEnv:
+  - USERNAME
+
+# push server images
+- name: gcr.io/cloud-builders/docker
+  args:
+  - '-c'
+  - docker push --all-tags $$USERNAME/zenml-server
+  id: push-server
+  waitFor:
+  - docker-login
+  entrypoint: bash
+  secretEnv:
+  - USERNAME
+
+timeout: 3600s
+availableSecrets:
+  secretManager:
+  - versionName: projects/$PROJECT_ID/secrets/docker-password/versions/1
+    env: PASSWORD
+  - versionName: projects/$PROJECT_ID/secrets/docker-username/versions/1
+    env: USERNAME