trying CI flow for beta releases #2
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
on: | ||
push: | ||
tags: | ||
- "engine@v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+" # Create tags matching beta version numbers, i.e. engine@v1.0.0-beta.1 | ||
# Allow the workflow to be manually run. | ||
# When doing so, artifacts will be built but not associated with a release. | ||
workflow_dispatch: | ||
# In cases of concurrent workflows running (consecutive pushes to PR) | ||
# leave the latest workflow and cancel the other (older) workflows | ||
# See https://docs.github.com/en/actions/using-jobs/using-concurrency | ||
concurrency: | ||
group: engine-release-beta | ||
cancel-in-progress: true | ||
permissions: | ||
# For deploying release artifacts. | ||
contents: write | ||
pull-requests: read | ||
# For pushing Docker images to ghcr.io. | ||
packages: write | ||
# For creating the release announcement. | ||
discussions: write | ||
env: | ||
IMAGE_ENGINE: ghcr.io/${{ github.repository }}/engine | ||
IMAGE_JUPYTER: ghcr.io/${{ github.repository }}/jupyter | ||
DOCKERHUB_ENGINE: kaskadaio/engine | ||
DOCKERHUB_JUPYTER: kaskadaio/jupyter | ||
name: Engine Beta Release | ||
jobs: | ||
release_engine_beta_version: | ||
name: Extract Version and Tag | ||
outputs: | ||
version: ${{ steps.determine_version.outputs.version }} | ||
runs-on: ubuntu-latest | ||
steps: | ||
# Determine the version being released (if tagged). | ||
- name: Determine version | ||
id: determine_version | ||
if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') | ||
run: | | ||
echo "version=${GITHUB_REF_NAME#engine@}" | tee -a "$GITHUB_OUTPUT" | ||
echo "tag=${GITHUB_REF_NAME#}" | tee -a "$GITHUB_OUTPUT" | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
- name: Diagnostics | ||
run: | | ||
echo Release Input Version: ${{ steps.determine_version.outputs.version }} | ||
release_engine_cross_build: | ||
name: Cross build release for aarch64 linux | ||
needs: ["release_engine_beta_version"] | ||
uses: ./.github/workflows/reusable_ci_arm64_release.yml | ||
release_engine_build: | ||
name: Engine Release Build | ||
needs: ["release_engine_beta_version"] | ||
strategy: | ||
matrix: | ||
include: | ||
- target: x86_64-apple-darwin | ||
goos: darwin | ||
goarch: amd64 | ||
host: macos-latest | ||
- target: aarch64-apple-darwin | ||
goos: darwin | ||
goarch: arm64 | ||
host: macos-latest | ||
- target: x86_64-unknown-linux-gnu | ||
goos: linux | ||
goarch: amd64 | ||
host: ubuntu-20.04 | ||
# - target: x86_64-unknown-linux-gnu | ||
# goos: linux | ||
# goarch: amd64 | ||
# host: ubuntu-22.04 | ||
- target: x86_64-pc-windows-msvc | ||
goos: windows | ||
goarch: amd64 | ||
host: windows-latest | ||
exe: .exe | ||
runs-on: ${{ matrix.host }} | ||
steps: | ||
- name: Checkout code | ||
uses: actions/checkout@master | ||
- name: Wren - Install Buf | ||
uses: bufbuild/buf-setup-action@v1.12.0 | ||
with: | ||
version: "1.17.0" | ||
buf_user: ${{ secrets.BUF_USER }} | ||
buf_api_token: ${{ secrets.BUF_API_TOKEN }} | ||
github_token: ${{ secrets.GITHUB_TOKEN }} | ||
- name: Wren - Install GoLang | ||
uses: actions/setup-go@v3 | ||
with: | ||
go-version: "1.19" | ||
cache: true | ||
cache-dependency-path: wren/go.sum | ||
- name: Wren - Generate Protobufs | ||
run: buf generate | ||
working-directory: proto | ||
- name: Wren - Generate Ent | ||
run: go generate ./ent | ||
working-directory: wren | ||
- name: Wren - Copy NOTICE | ||
run: | | ||
cp NOTICE ./wren/ | ||
cp NOTICE ./clients/cli | ||
- name: Apt Packages (Linux) | ||
if: matrix.goos == 'linux' | ||
uses: awalsh128/cache-apt-pkgs-action@latest | ||
with: | ||
packages: lld | ||
version: 1.0 | ||
- name: Wren - Build | ||
run: | | ||
mkdir ../release/ | ||
go build -ldflags="-w -s" -o ../release/wren${{ matrix.exe }} main.go | ||
env: | ||
GOARCH: ${{ matrix.goarch }} | ||
GOOS: ${{ matrix.goos }} | ||
CGO_ENABLED: 1 | ||
CXX: ${{ matrix.cxx }} | ||
CC: ${{ matrix.cc }} | ||
working-directory: wren | ||
- name: CLI - Build | ||
run: | | ||
go build -ldflags="-w -s" -o ../../release/cli${{ matrix.exe }} main.go | ||
env: | ||
GOARCH: ${{ matrix.goarch }} | ||
GOOS: ${{ matrix.goos }} | ||
CGO_ENABLED: 1 | ||
CXX: ${{ matrix.cxx }} | ||
CC: ${{ matrix.cc }} | ||
working-directory: clients/cli | ||
- name: Sparrow - Install toolchain | ||
id: toolchain | ||
uses: actions-rs/toolchain@v1 | ||
with: | ||
profile: minimal | ||
toolchain: stable | ||
override: true | ||
target: ${{ matrix.target }} | ||
- name: Install Protoc | ||
uses: arduino/setup-protoc@v1 | ||
with: | ||
repo-token: ${{ secrets.GITHUB_TOKEN }} | ||
- name: Sparrow - cargo build (Release) | ||
uses: actions-rs/cargo@v1 | ||
with: | ||
command: build | ||
args: --release --all-features --target ${{ matrix.target }} -p sparrow-main | ||
- name: Organize engine artifacts | ||
run: | | ||
mv target/${{ matrix.target }}/release/sparrow-main${{ matrix.exe }} release | ||
- name: Upload engine artifacts | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: engine-${{ matrix.goos }}-${{ matrix.goarch }} | ||
retention-days: 5 | ||
path: | | ||
release/sparrow-main${{ matrix.exe }} | ||
release/wren${{ matrix.exe }} | ||
release/cli${{ matrix.exe }} | ||
- name: Add artifacts to release | ||
if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') | ||
run: | | ||
gh release create ${TAG} --generate-notes -p -t "Engine BETA : ${TAG}" | ||
cp release/sparrow-main${{ matrix.exe }} ${ENGINE_ASSET_NAME} | ||
gh release upload ${TAG} ${ENGINE_ASSET_NAME} --clobber | ||
cp release/wren${{ matrix.exe }} ${WREN_ASSET_NAME} | ||
gh release upload ${TAG} ${WREN_ASSET_NAME} --clobber | ||
cp release/cli${{ matrix.exe }} ${CLI_ASSET_NAME} | ||
gh release upload ${TAG} ${CLI_ASSET_NAME} --clobber | ||
shell: bash | ||
env: | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
TAG: ${{ github.ref_name }} | ||
# Name of the assets to produce. We don't include the version | ||
# so can have a stable link to the latest asset. | ||
ENGINE_ASSET_NAME: kaskada-engine-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} | ||
WREN_ASSET_NAME: kaskada-manager-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} | ||
CLI_ASSET_NAME: kaskada-cli-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} | ||
# Not building docker image for ubuntu 22.04 (libc 2.35 libssl 3.0) | ||
# See https://github.com/cross-rs/cross/pull/973 on the cross repo | ||
release_docker_images: | ||
name: Build and push Docker images for releases | ||
runs-on: ubuntu-latest | ||
if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') | ||
needs: | ||
[ | ||
"release_engine_beta_version", | ||
"release_engine_build", | ||
"release_engine_cross_build", | ||
] | ||
steps: | ||
- name: Checkout code | ||
uses: actions/checkout@master | ||
# Refs: https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/ | ||
- name: Install Ubuntu packages for muliplatform | ||
shell: bash | ||
run: | | ||
sudo apt-get update | ||
sudo apt-get install qemu binfmt-support qemu-user-static | ||
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes | ||
docker run --rm --platform linux/arm64 -t arm64v8/ubuntu uname -m | ||
mkdir -p release/linux/amd64 | ||
mkdir -p release/linux/arm64 | ||
- name: Download AMD64 binaries | ||
uses: actions/download-artifact@v3 | ||
with: | ||
name: engine-linux-amd64 | ||
path: release/linux/amd64 | ||
- name: Download ARM64 binaries | ||
uses: actions/download-artifact@v3 | ||
with: | ||
name: engine-linux-aarch64 | ||
path: release/linux/arm64 | ||
- name: Current Date | ||
id: current_date | ||
run: | | ||
echo "created=$(date -u +"%Y-%m-%d %H:%M:%S%z")" | tee -a "$GITHUB_OUTPUT" | ||
shell: bash | ||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v2 | ||
- name: Login to GitHub Container Registry | ||
uses: docker/login-action@v2 | ||
with: | ||
registry: ghcr.io | ||
username: ${{ github.actor }} | ||
password: ${{ secrets.GITHUB_TOKEN }} | ||
- name: Login to DockerHub Container Registry | ||
uses: docker/login-action@v2 | ||
with: | ||
# registry: ghcr.io default is dockerhub (https://docs.docker.com/build/ci/github-actions/push-multi-registries/) | ||
username: ${{ secrets.DOCKERHUB_USERNAME }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
- name: Build and push Docker images for Engine | ||
uses: docker/build-push-action@v4 | ||
with: | ||
context: . | ||
platforms: linux/amd64, linux/arm64 | ||
file: Dockerfile.release | ||
push: true | ||
tags: ${{ env.IMAGE_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }} | ||
labels: | | ||
org.opencontainers.image.title=kaskada engine | ||
org.opencontainers.image.description=The kaskada engine | ||
org.opencontainers.image.url=${{ github.repositoryUrl }} | ||
org.opencontainers.image.source=${{ github.repositoryUrl }} | ||
org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }} | ||
org.opencontainers.image.created=${{ steps.current_date.outputs.created }} | ||
org.opencontainers.image.revision=${{ github.sha }} | ||
org.opencontainers.image.licenses=Apache-2.0 | ||
- name: Build and push Docker images for Jupyter | ||
uses: docker/build-push-action@v4 | ||
with: | ||
context: . | ||
platforms: linux/amd64, linux/arm64 | ||
file: Dockerfile.jupyter | ||
push: true | ||
tags: ${{ env.IMAGE_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }} | ||
labels: | | ||
org.opencontainers.image.title=kaskada engine with Jupyter | ||
org.opencontainers.image.description=The kaskada engine with pre-installed Jupyter notebook environment | ||
org.opencontainers.image.url=${{ github.repositoryUrl }} | ||
org.opencontainers.image.source=${{ github.repositoryUrl }} | ||
org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }} | ||
org.opencontainers.image.created=${{ steps.current_date.outputs.created }} | ||
org.opencontainers.image.revision=${{ github.sha }} | ||
org.opencontainers.image.licenses=Apache-2.0 | ||
release_engine_release: | ||
name: Create and upload engine release | ||
runs-on: ubuntu-20.04 | ||
if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') | ||
needs: ["release_engine_beta_version", "release_engine_build"] | ||
steps: | ||
- name: Checkout code | ||
uses: actions/checkout@master | ||
- name: Install Buf | ||
uses: bufbuild/buf-setup-action@v1.12.0 | ||
with: | ||
version: "1.17.0" | ||
buf_user: ${{ secrets.BUF_USER }} | ||
buf_api_token: ${{ secrets.BUF_API_TOKEN }} | ||
github_token: ${{ secrets.GITHUB_TOKEN }} | ||
- name: Push Protobufs | ||
run: buf push --tag ${{ github.ref_name }} --tag ${{ github.sha }} | ||
working-directory: proto | ||