trying CI flow for beta releases

.github/workflows/release_engine.yml

@@ -1,7 +1,7 @@
 on:
   push:
     tags:
-      - "engine@v*" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0
+      - "engine@v[0-9]+\.[0-9]+\.[0-9]+" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0
 
   # Allow the workflow to be manually run.
   # When doing so, artifacts will be built but not associated with a release.
@@ -44,15 +44,15 @@ jobs:
       # Determine the version being released (if tagged).
       - name: Determine version
         id: determine_version
-        if: startsWith(github.ref, 'refs/tags/engine@v')
+        if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta')
         run: |
           echo "version=${GITHUB_REF_NAME#engine@}" | tee -a "$GITHUB_OUTPUT"
           echo "tag=${GITHUB_REF_NAME#}" | tee -a "$GITHUB_OUTPUT"
       # Run the release drafter to update the release notes for this tag.
       - name: Draft Engine release
         id: draft_release
         uses: release-drafter/release-drafter@v5
-        if: startsWith(github.ref, 'refs/tags/engine@v')
+        if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta')
         with:
           config-name: release-drafter-engine.yml
           disable-autolabeler: true
@@ -200,7 +200,7 @@ jobs:
             release/cli${{ matrix.exe }}
 
       - name: Add artifacts to release
-        if: startsWith(github.ref, 'refs/tags/engine@v')
+        if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta')
         run: |
           cp release/sparrow-main${{ matrix.exe }} ${ENGINE_ASSET_NAME}
           gh release upload ${TAG} ${ENGINE_ASSET_NAME} --clobber
@@ -223,7 +223,7 @@ jobs:
   release_docker_images:
     name: Build and push Docker images for releases
     runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags/engine@v')
+    if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta')
     needs:
       [
         "release_engine_draft",
@@ -320,7 +320,7 @@ jobs:
   release_engine_release:
     name: Create and upload engine release
     runs-on: ubuntu-20.04
-    if: startsWith(github.ref, 'refs/tags/engine@v')
+    if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta')
     needs: ["release_engine_draft", "release_engine_build"]
     steps:
       - name: Checkout code

.github/workflows/release_engine_beta.yml

@@ -0,0 +1,321 @@
+on:
+  push:
+    tags:
+      - "engine@v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+" # Create tags matching beta version numbers, i.e. engine@v1.0.0-beta.1
+
+  # Allow the workflow to be manually run.
+  # When doing so, artifacts will be built but not associated with a release.
+  workflow_dispatch:
+
+# In cases of concurrent workflows running (consecutive pushes to PR)
+# leave the latest workflow and cancel the other (older) workflows
+# See https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: engine-release-beta
+  cancel-in-progress: true
+
+permissions:
+  # For deploying release artifacts.
+  contents: write
+  pull-requests: read
+  # For pushing Docker images to ghcr.io.
+  packages: write
+  # For creating the release announcement.
+  discussions: write
+
+env:
+  IMAGE_ENGINE: ghcr.io/${{ github.repository }}/engine
+  IMAGE_JUPYTER: ghcr.io/${{ github.repository }}/jupyter
+  DOCKERHUB_ENGINE: kaskadaio/engine
+  DOCKERHUB_JUPYTER: kaskadaio/jupyter
+
+name: Engine Beta Release
+jobs:
+  release_engine_beta_version:
+    name: Engine Release Draft
+    outputs:
+      version: ${{ steps.determine_version.outputs.version }}
+    runs-on: ubuntu-latest
+    steps:
+      # Determine the version being released (if tagged).
+      - name: Determine version
+        id: determine_version
+        if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta')
+        run: |
+          echo "version=${GITHUB_REF_NAME#engine@}" | tee -a "$GITHUB_OUTPUT"
+          echo "tag=${GITHUB_REF_NAME#}" | tee -a "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Diagnostics
+        run: |
+          echo Release Input Version: ${{ steps.determine_version.outputs.version }}
+
+  release_engine_cross_build:
+    name: Cross build release for aarch64 linux
+    needs: ["release_engine_beta_version"]
+    uses: ./.github/workflows/reusable_ci_arm64_release.yml
+
+  release_engine_build:
+    name: Engine Release Build
+    needs: ["release_engine_beta_version"]
+    strategy:
+      matrix:
+        include:
+          - target: x86_64-apple-darwin
+            goos: darwin
+            goarch: amd64
+            host: macos-latest
+          - target: aarch64-apple-darwin
+            goos: darwin
+            goarch: arm64
+            host: macos-latest
+          - target: x86_64-unknown-linux-gnu
+            goos: linux
+            goarch: amd64
+            host: ubuntu-20.04
+          # - target: x86_64-unknown-linux-gnu
+          #   goos: linux
+          #   goarch: amd64
+          #   host: ubuntu-22.04
+          - target: x86_64-pc-windows-msvc
+            goos: windows
+            goarch: amd64
+            host: windows-latest
+            exe: .exe
+    runs-on: ${{ matrix.host }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Wren - Install Buf
+        uses: bufbuild/buf-setup-action@v1.12.0
+        with:
+          version: "1.17.0"
+          buf_user: ${{ secrets.BUF_USER }}
+          buf_api_token: ${{ secrets.BUF_API_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Wren - Install GoLang
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.19"
+          cache: true
+          cache-dependency-path: wren/go.sum
+
+      - name: Wren - Generate Protobufs
+        run: buf generate
+        working-directory: proto
+
+      - name: Wren - Generate Ent
+        run: go generate ./ent
+        working-directory: wren
+
+      - name: Wren - Copy NOTICE
+        run: |
+          cp NOTICE ./wren/
+          cp NOTICE ./clients/cli
+
+      - name: Apt Packages (Linux)
+        if: matrix.goos == 'linux'
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: lld
+          version: 1.0
+
+      - name: Wren - Build
+        run: |
+          mkdir ../release/
+          go build -ldflags="-w -s" -o ../release/wren${{ matrix.exe }} main.go
+        env:
+          GOARCH: ${{ matrix.goarch }}
+          GOOS: ${{ matrix.goos }}
+          CGO_ENABLED: 1
+          CXX: ${{ matrix.cxx }}
+          CC: ${{ matrix.cc }}
+        working-directory: wren
+
+      - name: CLI - Build
+        run: |
+          go build -ldflags="-w -s" -o ../../release/cli${{ matrix.exe }} main.go
+        env:
+          GOARCH: ${{ matrix.goarch }}
+          GOOS: ${{ matrix.goos }}
+          CGO_ENABLED: 1
+          CXX: ${{ matrix.cxx }}
+          CC: ${{ matrix.cc }}
+        working-directory: clients/cli
+
+      - name: Sparrow - Install toolchain
+        id: toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          target: ${{ matrix.target }}
+
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Sparrow - cargo build (Release)
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --all-features --target ${{ matrix.target }} -p sparrow-main
+
+      - name: Organize engine artifacts
+        run: |
+          mv target/${{ matrix.target }}/release/sparrow-main${{ matrix.exe }} release
+
+      - name: Upload engine artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: engine-${{ matrix.goos }}-${{ matrix.goarch }}
+          retention-days: 5
+          path: |
+            release/sparrow-main${{ matrix.exe }}
+            release/wren${{ matrix.exe }}
+            release/cli${{ matrix.exe }}
+
+      - name: Add artifacts to release
+        if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta')
+        run: |
+          gh release create ${TAG} --generate-notes -p -t "Engine BETA : ${TAG}"
+          cp release/sparrow-main${{ matrix.exe }} ${ENGINE_ASSET_NAME}
+          gh release upload ${TAG} ${ENGINE_ASSET_NAME} --clobber
+          cp release/wren${{ matrix.exe }} ${WREN_ASSET_NAME}
+          gh release upload ${TAG} ${WREN_ASSET_NAME} --clobber
+          cp release/cli${{ matrix.exe }} ${CLI_ASSET_NAME}
+          gh release upload ${TAG} ${CLI_ASSET_NAME} --clobber
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ github.ref_name }}
+          # Name of the assets to produce. We don't include the version
+          # so can have a stable link to the latest asset.
+          ENGINE_ASSET_NAME: kaskada-engine-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }}
+          WREN_ASSET_NAME: kaskada-manager-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }}
+          CLI_ASSET_NAME: kaskada-cli-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }}
+
+  # Not building docker image for ubuntu 22.04 (libc 2.35 libssl 3.0) 
+  # See https://github.com/cross-rs/cross/pull/973 on the cross repo 
+  release_docker_images:
+    name: Build and push Docker images for releases
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta')
+    needs:
+      [
+        "release_engine_beta_version",
+        "release_engine_build",
+        "release_engine_cross_build",
+      ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+        # Refs: https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
+      - name: Install Ubuntu packages for muliplatform
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install qemu binfmt-support qemu-user-static
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+          docker run --rm --platform linux/arm64 -t arm64v8/ubuntu uname -m
+          mkdir -p release/linux/amd64
+          mkdir -p release/linux/arm64
+
+      - name: Download AMD64 binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: engine-linux-amd64
+          path: release/linux/amd64
+
+      - name: Download ARM64 binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: engine-linux-aarch64
+          path: release/linux/arm64
+
+      - name: Current Date
+        id: current_date
+        run: |
+          echo "created=$(date -u +"%Y-%m-%d %H:%M:%S%z")" | tee -a "$GITHUB_OUTPUT"
+        shell: bash
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Login to DockerHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          # registry: ghcr.io  default is dockerhub (https://docs.docker.com/build/ci/github-actions/push-multi-registries/)
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker images for Engine
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64, linux/arm64
+          file: Dockerfile.release
+          push: true
+          tags: ${{ env.IMAGE_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }}
+          labels: |
+            org.opencontainers.image.title=kaskada engine 
+            org.opencontainers.image.description=The kaskada engine
+            org.opencontainers.image.url=${{ github.repositoryUrl }}
+            org.opencontainers.image.source=${{ github.repositoryUrl }}
+            org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }}
+            org.opencontainers.image.created=${{ steps.current_date.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.licenses=Apache-2.0
+
+      - name: Build and push Docker images for Jupyter
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          platforms: linux/amd64, linux/arm64
+          file: Dockerfile.jupyter
+          push: true
+          tags: ${{ env.IMAGE_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }}
+          labels: |
+            org.opencontainers.image.title=kaskada engine with Jupyter
+            org.opencontainers.image.description=The kaskada engine with pre-installed Jupyter notebook environment
+            org.opencontainers.image.url=${{ github.repositoryUrl }}
+            org.opencontainers.image.source=${{ github.repositoryUrl }}
+            org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }}
+            org.opencontainers.image.created=${{ steps.current_date.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.licenses=Apache-2.0
+
+  release_engine_release:
+    name: Create and upload engine release
+    runs-on: ubuntu-20.04
+    if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta')
+    needs: ["release_engine_beta_version", "release_engine_build"]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Install Buf
+        uses: bufbuild/buf-setup-action@v1.12.0
+        with:
+          version: "1.17.0"
+          buf_user: ${{ secrets.BUF_USER }}
+          buf_api_token: ${{ secrets.BUF_API_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push Protobufs
+        run: buf push --tag ${{ github.ref_name }} --tag ${{ github.sha }}
+        working-directory: proto
+

.github/workflows/release_python_client.yml

@@ -1,7 +1,7 @@
 on:
   push:
     tags:
-      - "python@v*" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0
+      - "python@v[0-9]+\.[0-9]+\.[0-9]+" # Create tags matching 'python@v*'', i.e. python@v1.0.0
 
   # Allow the workflow to be manually run.
   # When doing so, artifacts will be built but not associated with a release.
@@ -145,7 +145,7 @@ jobs:
         # Upload artifact to GitHub release and remove draft status 
         # DO NOT MARK GITHUB RELEASE AS latest -- only one release can be latest in a GitHub repo and we reserve that for the engine release. 
       - name: Add artifacts to GitHub release
-        if: startsWith(github.ref, 'refs/tags/engine@v')
+        if: startsWith(github.ref, 'refs/tags/python@v') && !contains(github.ref, 'beta')
         run: | 
           gh release upload ${TAG} clients/python/dist/kaskada-*.whl --clobber 
           gh release upload ${TAG} clients/python/dist/kaskada-*.tar.gz --clobber