diff --git a/.github/workflows/release_engine.yml b/.github/workflows/release_engine.yml index e49865c95..55045d3ba 100644 --- a/.github/workflows/release_engine.yml +++ b/.github/workflows/release_engine.yml @@ -1,7 +1,7 @@ on: push: tags: - - "engine@v*" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0 + - "engine@v[0-9]+\.[0-9]+\.[0-9]+" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0 # Allow the workflow to be manually run. # When doing so, artifacts will be built but not associated with a release. @@ -44,7 +44,7 @@ jobs: # Determine the version being released (if tagged). - name: Determine version id: determine_version - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta') run: | echo "version=${GITHUB_REF_NAME#engine@}" | tee -a "$GITHUB_OUTPUT" echo "tag=${GITHUB_REF_NAME#}" | tee -a "$GITHUB_OUTPUT" @@ -52,7 +52,7 @@ jobs: - name: Draft Engine release id: draft_release uses: release-drafter/release-drafter@v5 - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta') with: config-name: release-drafter-engine.yml disable-autolabeler: true @@ -200,7 +200,7 @@ jobs: release/cli${{ matrix.exe }} - name: Add artifacts to release - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta') run: | cp release/sparrow-main${{ matrix.exe }} ${ENGINE_ASSET_NAME} gh release upload ${TAG} ${ENGINE_ASSET_NAME} --clobber @@ -223,7 +223,7 @@ jobs: release_docker_images: name: Build and push Docker images for releases runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta') needs: [ "release_engine_draft", @@ -320,7 +320,7 @@ jobs: release_engine_release: name: Create and upload engine release runs-on: ubuntu-20.04 - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/engine@v') && !contains(github.ref, 'beta') needs: ["release_engine_draft", "release_engine_build"] steps: - name: Checkout code diff --git a/.github/workflows/release_engine_beta.yml b/.github/workflows/release_engine_beta.yml new file mode 100644 index 000000000..489cc415f --- /dev/null +++ b/.github/workflows/release_engine_beta.yml @@ -0,0 +1,321 @@ +on: + push: + tags: + - "engine@v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+" # Create tags matching beta version numbers, i.e. engine@v1.0.0-beta.1 + + # Allow the workflow to be manually run. + # When doing so, artifacts will be built but not associated with a release. + workflow_dispatch: + +# In cases of concurrent workflows running (consecutive pushes to PR) +# leave the latest workflow and cancel the other (older) workflows +# See https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + group: engine-release-beta + cancel-in-progress: true + +permissions: + # For deploying release artifacts. + contents: write + pull-requests: read + # For pushing Docker images to ghcr.io. + packages: write + # For creating the release announcement. + discussions: write + +env: + IMAGE_ENGINE: ghcr.io/${{ github.repository }}/engine + IMAGE_JUPYTER: ghcr.io/${{ github.repository }}/jupyter + DOCKERHUB_ENGINE: kaskadaio/engine + DOCKERHUB_JUPYTER: kaskadaio/jupyter + +name: Engine Beta Release +jobs: + release_engine_beta_version: + name: Engine Release Draft + outputs: + version: ${{ steps.determine_version.outputs.version }} + runs-on: ubuntu-latest + steps: + # Determine the version being released (if tagged). + - name: Determine version + id: determine_version + if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') + run: | + echo "version=${GITHUB_REF_NAME#engine@}" | tee -a "$GITHUB_OUTPUT" + echo "tag=${GITHUB_REF_NAME#}" | tee -a "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Diagnostics + run: | + echo Release Input Version: ${{ steps.determine_version.outputs.version }} + + release_engine_cross_build: + name: Cross build release for aarch64 linux + needs: ["release_engine_beta_version"] + uses: ./.github/workflows/reusable_ci_arm64_release.yml + + release_engine_build: + name: Engine Release Build + needs: ["release_engine_beta_version"] + strategy: + matrix: + include: + - target: x86_64-apple-darwin + goos: darwin + goarch: amd64 + host: macos-latest + - target: aarch64-apple-darwin + goos: darwin + goarch: arm64 + host: macos-latest + - target: x86_64-unknown-linux-gnu + goos: linux + goarch: amd64 + host: ubuntu-20.04 + # - target: x86_64-unknown-linux-gnu + # goos: linux + # goarch: amd64 + # host: ubuntu-22.04 + - target: x86_64-pc-windows-msvc + goos: windows + goarch: amd64 + host: windows-latest + exe: .exe + runs-on: ${{ matrix.host }} + steps: + - name: Checkout code + uses: actions/checkout@master + + - name: Wren - Install Buf + uses: bufbuild/buf-setup-action@v1.12.0 + with: + version: "1.17.0" + buf_user: ${{ secrets.BUF_USER }} + buf_api_token: ${{ secrets.BUF_API_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Wren - Install GoLang + uses: actions/setup-go@v3 + with: + go-version: "1.19" + cache: true + cache-dependency-path: wren/go.sum + + - name: Wren - Generate Protobufs + run: buf generate + working-directory: proto + + - name: Wren - Generate Ent + run: go generate ./ent + working-directory: wren + + - name: Wren - Copy NOTICE + run: | + cp NOTICE ./wren/ + cp NOTICE ./clients/cli + + - name: Apt Packages (Linux) + if: matrix.goos == 'linux' + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: lld + version: 1.0 + + - name: Wren - Build + run: | + mkdir ../release/ + go build -ldflags="-w -s" -o ../release/wren${{ matrix.exe }} main.go + env: + GOARCH: ${{ matrix.goarch }} + GOOS: ${{ matrix.goos }} + CGO_ENABLED: 1 + CXX: ${{ matrix.cxx }} + CC: ${{ matrix.cc }} + working-directory: wren + + - name: CLI - Build + run: | + go build -ldflags="-w -s" -o ../../release/cli${{ matrix.exe }} main.go + env: + GOARCH: ${{ matrix.goarch }} + GOOS: ${{ matrix.goos }} + CGO_ENABLED: 1 + CXX: ${{ matrix.cxx }} + CC: ${{ matrix.cc }} + working-directory: clients/cli + + - name: Sparrow - Install toolchain + id: toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + target: ${{ matrix.target }} + + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Sparrow - cargo build (Release) + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --all-features --target ${{ matrix.target }} -p sparrow-main + + - name: Organize engine artifacts + run: | + mv target/${{ matrix.target }}/release/sparrow-main${{ matrix.exe }} release + + - name: Upload engine artifacts + uses: actions/upload-artifact@v3 + with: + name: engine-${{ matrix.goos }}-${{ matrix.goarch }} + retention-days: 5 + path: | + release/sparrow-main${{ matrix.exe }} + release/wren${{ matrix.exe }} + release/cli${{ matrix.exe }} + + - name: Add artifacts to release + if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') + run: | + gh release create ${TAG} --generate-notes -p -t "Engine BETA : ${TAG}" + cp release/sparrow-main${{ matrix.exe }} ${ENGINE_ASSET_NAME} + gh release upload ${TAG} ${ENGINE_ASSET_NAME} --clobber + cp release/wren${{ matrix.exe }} ${WREN_ASSET_NAME} + gh release upload ${TAG} ${WREN_ASSET_NAME} --clobber + cp release/cli${{ matrix.exe }} ${CLI_ASSET_NAME} + gh release upload ${TAG} ${CLI_ASSET_NAME} --clobber + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + # Name of the assets to produce. We don't include the version + # so can have a stable link to the latest asset. + ENGINE_ASSET_NAME: kaskada-engine-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} + WREN_ASSET_NAME: kaskada-manager-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} + CLI_ASSET_NAME: kaskada-cli-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.exe }} + + # Not building docker image for ubuntu 22.04 (libc 2.35 libssl 3.0) + # See https://github.com/cross-rs/cross/pull/973 on the cross repo + release_docker_images: + name: Build and push Docker images for releases + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') + needs: + [ + "release_engine_beta_version", + "release_engine_build", + "release_engine_cross_build", + ] + + steps: + - name: Checkout code + uses: actions/checkout@master + + # Refs: https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/ + - name: Install Ubuntu packages for muliplatform + shell: bash + run: | + sudo apt-get update + sudo apt-get install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm --platform linux/arm64 -t arm64v8/ubuntu uname -m + mkdir -p release/linux/amd64 + mkdir -p release/linux/arm64 + + - name: Download AMD64 binaries + uses: actions/download-artifact@v3 + with: + name: engine-linux-amd64 + path: release/linux/amd64 + + - name: Download ARM64 binaries + uses: actions/download-artifact@v3 + with: + name: engine-linux-aarch64 + path: release/linux/arm64 + + - name: Current Date + id: current_date + run: | + echo "created=$(date -u +"%Y-%m-%d %H:%M:%S%z")" | tee -a "$GITHUB_OUTPUT" + shell: bash + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to DockerHub Container Registry + uses: docker/login-action@v2 + with: + # registry: ghcr.io default is dockerhub (https://docs.docker.com/build/ci/github-actions/push-multi-registries/) + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker images for Engine + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64, linux/arm64 + file: Dockerfile.release + push: true + tags: ${{ env.IMAGE_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_ENGINE }}:${{ needs.release_engine_beta_version.outputs.version }} + labels: | + org.opencontainers.image.title=kaskada engine + org.opencontainers.image.description=The kaskada engine + org.opencontainers.image.url=${{ github.repositoryUrl }} + org.opencontainers.image.source=${{ github.repositoryUrl }} + org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }} + org.opencontainers.image.created=${{ steps.current_date.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=Apache-2.0 + + - name: Build and push Docker images for Jupyter + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64, linux/arm64 + file: Dockerfile.jupyter + push: true + tags: ${{ env.IMAGE_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }}, ${{ env.DOCKERHUB_JUPYTER }}:${{ needs.release_engine_beta_version.outputs.version }} + labels: | + org.opencontainers.image.title=kaskada engine with Jupyter + org.opencontainers.image.description=The kaskada engine with pre-installed Jupyter notebook environment + org.opencontainers.image.url=${{ github.repositoryUrl }} + org.opencontainers.image.source=${{ github.repositoryUrl }} + org.opencontainers.image.version=${{ needs.release_engine_beta_version.outputs.version }} + org.opencontainers.image.created=${{ steps.current_date.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.licenses=Apache-2.0 + + release_engine_release: + name: Create and upload engine release + runs-on: ubuntu-20.04 + if: startsWith(github.ref, 'refs/tags/engine@v') && contains(github.ref, 'beta') + needs: ["release_engine_beta_version", "release_engine_build"] + steps: + - name: Checkout code + uses: actions/checkout@master + + - name: Install Buf + uses: bufbuild/buf-setup-action@v1.12.0 + with: + version: "1.17.0" + buf_user: ${{ secrets.BUF_USER }} + buf_api_token: ${{ secrets.BUF_API_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Push Protobufs + run: buf push --tag ${{ github.ref_name }} --tag ${{ github.sha }} + working-directory: proto + diff --git a/.github/workflows/release_python_client.yml b/.github/workflows/release_python_client.yml index 7e44b77b9..7f0e419c1 100644 --- a/.github/workflows/release_python_client.yml +++ b/.github/workflows/release_python_client.yml @@ -1,7 +1,7 @@ on: push: tags: - - "python@v*" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0 + - "python@v[0-9]+\.[0-9]+\.[0-9]+" # Create tags matching 'python@v*'', i.e. python@v1.0.0 # Allow the workflow to be manually run. # When doing so, artifacts will be built but not associated with a release. @@ -145,7 +145,7 @@ jobs: # Upload artifact to GitHub release and remove draft status # DO NOT MARK GITHUB RELEASE AS latest -- only one release can be latest in a GitHub repo and we reserve that for the engine release. - name: Add artifacts to GitHub release - if: startsWith(github.ref, 'refs/tags/engine@v') + if: startsWith(github.ref, 'refs/tags/python@v') && !contains(github.ref, 'beta') run: | gh release upload ${TAG} clients/python/dist/kaskada-*.whl --clobber gh release upload ${TAG} clients/python/dist/kaskada-*.tar.gz --clobber diff --git a/.github/workflows/release_python_client_beta.yml b/.github/workflows/release_python_client_beta.yml new file mode 100644 index 000000000..c02ab8288 --- /dev/null +++ b/.github/workflows/release_python_client_beta.yml @@ -0,0 +1,147 @@ +on: + push: + tags: + - "python@v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+" # Create tags matching 'engine@v*'', i.e. engine@v1.0.0 + + # Allow the workflow to be manually run. + # When doing so, artifacts will be built but not associated with a release. + workflow_dispatch: + +# In cases of concurrent workflows running (consecutive pushes to PR) +# leave the latest workflow and cancel the other (older) workflows +# See https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + group: python-client-beta + cancel-in-progress: true + + +permissions: + # For deploying release artifacts. + contents: write + pull-requests: read + +name: Python Client Beta Release + +jobs: + python_protos: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Buf + uses: bufbuild/buf-setup-action@v1.12.0 + with: + version: "1.17.0" + buf_user: ${{ secrets.BUF_USER }} + buf_api_token: ${{ secrets.BUF_API_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate Protos + run: buf generate + working-directory: proto + + - name: Upload Python Proto Artifact + uses: actions/upload-artifact@v3 + with: + name: python-protos + # Python gRPC code is generated in src/kaskada/kaskada + path: clients/python/src/kaskada/kaskada + + python_client_build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.8", "3.9"] + + needs: [python_protos] + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ./clients/python + + steps: + + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Download Python Proto Artifact + uses: actions/download-artifact@v3 + with: + name: python-protos + path: clients/python/src/kaskada/kaskada + + - name: Run image + uses: abatilo/actions-poetry@v2.0.0 + with: + poetry-version: 1.3.2 + + - name: Install project + run: poetry install + + - name: Python Client Style + run: poetry run poe style + + - name: Python Client Types + run: poetry run poe types + + - name: Python Client Lint + run: poetry run poe lint + + - name: Python Client Test + run: poetry run poe test + + - name: Python Client build package + run: poetry build + + # Used in publish_to_PyPi to verify the package + - name: Generate requirements.txt + run: poetry export -f requirements.txt --output requirements.txt + + - name: Upload Python Client Artifacts + uses: actions/upload-artifact@v3 + with: + name: python-client-${{ matrix.os }}-${{ matrix.python-version }} + path: | + tests/integration/pythonClient/test_package.py + clients/python/requirements.txt + clients/python/dist + + publish_to_PyPi: + needs: [python_client_build] + runs-on: ubuntu-latest + steps: + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Download Python Client Artifacts + uses: actions/download-artifact@v3 + with: + name: python-client-ubuntu-latest-3.8 + # path: verify/ + + - name: Verify python package build artifact + run: | + pip install -r clients/python/requirements.txt + rm clients/python/requirements.txt + pip install clients/python/dist/kaskada-*.whl + python tests/integration/pythonClient/test_package.py + rm tests/integration/pythonClient/test_package.py + + - name: Create GitHub release + if: startsWith(github.ref, 'refs/tags/python@v') && contains(github.ref, 'beta') + run: | + gh release create ${TAG} --generate-notes -p -t "Python Client BETA: ${TAG}" + gh release upload ${TAG} clients/python/dist/kaskada-*.whl --clobber + gh release upload ${TAG} clients/python/dist/kaskada-*.tar.gz --clobber + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }}