Speed Up Publish OSS Artifacts for Cloud Workflow

.github/actions/build-branch/action.yml

@@ -4,6 +4,10 @@ inputs:
   branch_version_tag:
     description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
     required: false
+  build_docker_images:
+    description: 'Build docker images'
+    default: 'true'
+    required: false
 outputs:
   branch_version_tag:
     description: "Tag used for jars and docker images. Either user specified or auto generated as `dev-<commit_hash>`"
@@ -20,35 +24,17 @@ runs:
         [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "branch_version_tag=${{ inputs.branch_version_tag }}" >> $GITHUB_OUTPUT \
           || { short_hash=$(git rev-parse --short=10 HEAD); echo "branch_version_tag=dev-$short_hash" >> $GITHUB_OUTPUT ; }
 
-    - uses: actions/setup-java@v3
-      with:
-        distribution: "zulu"
-        java-version: "17"
-
-    - uses: actions/setup-node@v3
-      with:
-        node-version: "lts/gallium"
-
-    - uses: actions/setup-python@v4
-      with:
-        python-version: "3.9"
+    - name: Prepare Runner for Building
+      uses: ./.github/actions/runner-prepare-for-build
 
-    - name: Set up CI Gradle Properties
-      run: |
-        mkdir -p ~/.gradle/
-        cat > ~/.gradle/gradle.properties <<EOF
-        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-        org.gradle.workers.max=8
-        org.gradle.vfs.watch=false
-        EOF
+    - name: Build with Docker Images
+      if: inputs.build_docker_images == 'true'
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
       shell: bash
 
-    - name: Build
-      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
+    - name: Build without Docker Images
+      if: inputs.build_docker_images != 'true'
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew test --scan
       shell: bash
 
     - name: Publish to Maven Local

.github/actions/runner-prepare-for-build/action.yml

@@ -0,0 +1,47 @@
+name: 'Prepare Runner for Building'
+description: 'Prepare Runner for Building project'
+inputs:
+  install_java:
+    description: ''
+    required: false
+    default: 'true'
+  install_node:
+    description: ''
+    required: false
+    default: 'true'
+  install_python:
+    description: ''
+    required: false
+    default: 'true'
+runs:
+  using: "composite"
+  steps:
+    - if: inputs.install_java == 'true'
+      uses: actions/setup-java@v3
+      with:
+        distribution: "zulu"
+        java-version: "17"
+
+    - if: inputs.install_node == 'true'
+      uses: actions/setup-node@v3
+      with:
+        node-version: "lts/gallium"
+
+    - if: inputs.install_python == 'true'
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.9"
+
+    - if: inputs.install_java == 'true'
+      name: Set up CI Gradle Properties
+      run: |
+        mkdir -p ~/.gradle/
+        cat > ~/.gradle/gradle.properties <<EOF
+        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        org.gradle.vfs.watch=false
+        EOF
+      shell: bash

.github/workflows/publish-oss-for-cloud.yml

@@ -1,6 +1,8 @@
 name: Publish OSS Artifacts for Cloud
-concurrency:
-  group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+
+env:
+  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
 
 on:
   workflow_dispatch:
@@ -64,6 +66,12 @@ jobs:
           cat $GITHUB_OUTPUT || true # for the sake of investigation
 
   oss-branch-build:
+    concurrency:
+      # only allow one workflow run at a time for a given SHA
+      # to prevent multiple runs from pushing artifacts for the same SHA at the same time
+      # note: using inputs in the group expression only works when specifying concurrency at the job level
+      group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+      cancel-in-progress: false
     name: "Gradle Build and Publish"
     needs:
       - start-runner
@@ -80,6 +88,7 @@ jobs:
         uses: ./.github/actions/build-branch
         with:
           branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }}
+          build_docker_images: 'false'
 
       - name: Publish Dev Jars
         env:
@@ -97,6 +106,12 @@ jobs:
         shell: bash
 
   docker-push:
+    concurrency:
+      # only allow one workflow run at a time for a given SHA
+      # to prevent multiple runs from pushing artifacts for the same SHA at the same time
+      # note: using inputs in the group expression only works when specifying concurrency at the job level
+      group: ${{ github.workflow }}-${{ inputs.oss_ref || github.sha }}
+      cancel-in-progress: false
     name: "Push Docker Images"
     needs:
       - start-runner
@@ -123,6 +138,15 @@ jobs:
           echo "GIT_REVISION=${GIT_REVISION}" >> $GITHUB_ENV
         shell: bash
 
+      - name: Prepare Runner for Building
+        uses: ./.github/actions/runner-prepare-for-build
+
+        # Put tars/artifacts in the correct build directories so docker buildx can find the artifacts it needs when building docker images
+      - name: Prepare Docker context
+        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew copyGeneratedTar airbyte-db:db-lib:copyInitSql
+        shell: bash
+
+        # Build docker images using docker buildx (for multi platform)
       - name: Push Docker Images
         env:
           VERSION: ${{ needs.generate-tags.outputs.dev_tag }}
@@ -133,3 +157,37 @@ jobs:
       - name: Cleanup Docker buildx
         run: docker buildx rm oss-buildx
         shell: bash
+
+  stop-runner:
+    name: "Stop Build EC2 Runner"
+    timeout-minutes: 10
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - docker-push # wait until all publish steps are done
+    runs-on: ubuntu-latest
+    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+    # Thus, we check for skipped here.
+    if: ${{ always() && needs.start-runner.result != 'skipped'}}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+      - name: Stop EC2 runner
+        uses: airbytehq/ec2-github-runner@base64v1.1.0
+        with:
+          mode: stop
+          github-token: ${{ env.PAT }}
+          label: ${{ needs.start-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}