HDDS-6717. Allow running flaky-test-check with specific Ratis commit (a…

…pache#6675)

.github/workflows/build-ratis.yml

@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This workflow can be called by other workflows to build Ratis.
+#
+# Inputs:
+# - Ratis repo
+# - the commit to build
+# Outputs:
+# - various version numbers that need to be provided to the Ozone build process.
+# - Ratis repository is uploaded as an artifact named `ratis-jars`
+#
+# See `intermittent-test-check.yml` as an example use of this workflow.
+
+name: build-ratis
+on:
+  workflow_call:
+    inputs:
+      repo:
+        description: Ratis repository
+        default: apache/ratis
+        required: true
+        type: string
+      ref:
+        description: Ratis ref (branch, tag or commit SHA)
+        default: master
+        required: true
+        type: string
+    outputs:
+      ratis-version:
+        description: "Ratis Version"
+        value: ${{ jobs.ratis.outputs.ratis-version }}
+      thirdparty-version:
+        description: "Ratis Third-Party Version"
+        value: ${{ jobs.ratis.outputs.thirdparty-version }}
+      grpc-version:
+        description: "gRPC Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.grpc-version }}
+      netty-version:
+        description: "Netty Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.netty-version }}
+      protobuf-version:
+        description: "Protobuf Version"
+        value: ${{ jobs.ratis-thirdparty.outputs.protobuf-version }}
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+jobs:
+  ratis:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    outputs:
+      ratis-version: ${{ steps.versions.outputs.ratis }}
+      thirdparty-version: ${{ steps.versions.outputs.thirdparty }}
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo }}
+          ref: ${{ inputs.ref }}
+      - name: Cache for maven dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ratis
+          key: ratis-dependencies-${{ hashFiles('**/pom.xml') }}
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Get component versions
+        id: versions
+        run: |
+          thirdparty_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=ratis.thirdparty.version)"
+          echo "thirdparty=${thirdparty_version}" >> $GITHUB_OUTPUT
+
+          ratis_sha=$(git rev-parse --short HEAD)
+          ratis_version="$(mvn help:evaluate -N -q -DforceStdout -Dexpression=project.version | sed -e "s/-SNAPSHOT/-${ratis_sha}-SNAPSHOT/")"
+          echo "ratis=${ratis_version}" >> $GITHUB_OUTPUT
+      - name: Run a full build
+        run: |
+          mvn versions:set -DnewVersion=${{ steps.versions.outputs.ratis }}
+          dev-support/checks/build.sh
+      - name: Store Maven repo for tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+          retention-days: 1
+  ratis-thirdparty:
+    runs-on: ubuntu-20.04
+    needs:
+      - ratis
+    timeout-minutes: 30
+    outputs:
+      grpc-version: ${{ steps.versions.outputs.grpc }}
+      netty-version: ${{ steps.versions.outputs.netty }}
+      protobuf-version: ${{ steps.versions.outputs.protobuf }}
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          repository: apache/ratis-thirdparty
+          ref: ${{ needs.ratis.outputs.thirdparty-version }}
+      - name: Get component versions
+        id: versions
+        run: |
+          echo "grpc=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.grpc.version)" >> $GITHUB_OUTPUT
+          echo "netty=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.netty.version)" >> $GITHUB_OUTPUT
+          echo "protobuf=$(mvn help:evaluate -N -q -DforceStdout -Dexpression=shaded.protobuf.version)" >> $GITHUB_OUTPUT
+  debug:
+    runs-on: ubuntu-20.04
+    needs:
+      - ratis
+      - ratis-thirdparty
+    steps:
+      - name: Print versions
+        run: |
+          echo ${{ needs.ratis.outputs.ratis-version }}
+          echo ${{ needs.ratis.outputs.thirdparty-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.grpc-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.netty-version }}
+          echo ${{ needs.ratis-thirdparty.outputs.protobuf-version }}

.github/workflows/intermittent-test-check.yml

@@ -40,12 +40,22 @@ on:
         description: Stop after first failure
         default: false
         required: true
+      ratis-repo:
+        description: Ratis repository
+        default: ''
+        required: false
+      ratis-ref:
+        description: Ratis ref (branch, tag or commit SHA)
+        default: ''
+        required: false
 env:
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
   TEST_CLASS: ${{ github.event.inputs.test-class}}
   TEST_METHOD: ${{ github.event.inputs.test-name }}
   ITERATIONS: ${{ github.event.inputs.iterations }}
   FAIL_FAST: ${{ github.event.inputs.fail-fast }}
+  RATIS_REPO: ${{ github.event.inputs.ratis-repo }}
+  RATIS_VERSION: ${{ github.event.inputs.ratis-ref }}
 run-name: ${{ github.event_name == 'workflow_dispatch' && format('{0}#{1}[{2}]-{3}x{4}', inputs.test-class, inputs.test-name, inputs.ref, inputs.splits, inputs.iterations) || '' }}
 jobs:
   prepare-job:
@@ -66,9 +76,17 @@ jobs:
           printf -v x "%s," "${splits[@]}"
           split_matrix="[${x%,}]"
           echo "matrix=$split_matrix" >> $GITHUB_OUTPUT
+  ratis:
+    uses: ./.github/workflows/build-ratis.yml
+    if: ${{ github.event.inputs.ratis-ref != '' }}
+    with:
+      repo: ${{ github.event.inputs.ratis-repo || format('{0}/ratis', github.repository_owner) }}
+      ref: ${{ github.event.inputs.ratis-ref }}
   build:
+    if: ${{ always() }}
     needs:
       - prepare-job
+      - ratis
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     steps:
@@ -83,13 +101,30 @@ jobs:
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
+      - name: Download Ratis repo
+        if: ${{ github.event.inputs.ratis-ref != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
       - name: Setup java
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Build (most) of Ozone
-        run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx -DskipShade
+        run: |
+          args="-Dskip.npx -Dskip.installnpx -DskipShade"
+          if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
+            args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
+            args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"
+            args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }}"
+            args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version }}"
+            args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
+          fi
+
+          hadoop-ozone/dev-support/checks/build.sh $args
       - name: Store Maven repo for tests
         uses: actions/upload-artifact@v4
         with:
@@ -98,8 +133,10 @@ jobs:
             ~/.m2/repository/org/apache/ozone
           retention-days: 1
   run-test:
+    if: ${{ always() }}
     needs:
       - prepare-job
+      - ratis
       - build
     name: Run-Split
     runs-on: ubuntu-20.04
@@ -120,6 +157,13 @@ jobs:
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
+      - name: Download Ratis repo
+        if: ${{ github.event.inputs.ratis-ref != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
       - name: Download Ozone repo
         id: download-ozone-repo
         uses: actions/download-artifact@v4
@@ -140,6 +184,14 @@ jobs:
           fi
 
           args="-DexcludedGroups=native|slow|unhealthy"
+          if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
+            args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
+            args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"
+            args="$args -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }}"
+            args="$args -Dnetty.version=${{ needs.ratis.outputs.netty-version }}"
+            args="$args -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
+          fi
+
           if [ "$TEST_METHOD" = "ALL" ]; then
             echo "Running all tests from $TEST_CLASS"
             set -x