HDDS-11718. Some CI checks passing despite error

.github/workflows/ci.yml

@@ -272,12 +272,11 @@ jobs:
           java-version: 8
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -320,12 +319,11 @@ jobs:
           java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -400,10 +398,9 @@ jobs:
       - name: Execute tests
         run: |
           hadoop-ozone/dev-support/checks/${{ github.job }}.sh
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -451,10 +448,9 @@ jobs:
       - name: Execute tests
         run: |
           hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Install diffoscope
         run: |
           sudo apt update -q
@@ -509,10 +505,9 @@ jobs:
           KEEP_IMAGE: false
           OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
           OZONE_VOLUME_OWNER: 1000
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -549,10 +544,9 @@ jobs:
           sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
           popd
           ./hadoop-ozone/dev-support/checks/kubernetes.sh
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -608,7 +602,6 @@ jobs:
           distribution: 'temurin'
           java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
-        continue-on-error: true
         run: |
           args="${{ inputs.ratis_args }}"
           if [[ "${{ matrix.profile }}" == "flaky" ]]; then
@@ -627,7 +620,7 @@ jobs:
             cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
           fi
           hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()

hadoop-ozone/dev-support/checks/_post_process.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script does common processing after Maven-based checks.
+#
+# - ensures Maven error is reported as failure
+# - writes number of failures into file
+# - exits with the correct code
+
+# Prerequisites:
+# - $rc should be set to Maven exit code
+# - $REPORT_DIR should be defined
+# - $REPORT_FILE should be defined
+# - Maven output should be saved in $REPORT_DIR/output.log
+
+# script failed, but report file is empty (does not reflect failure)
+if [[ ${rc} -ne 0 ]] && [[ ! -s "${REPORT_FILE}" ]]; then
+  # do we know what to look for?
+  if [[ -n "${ERROR_PATTERN:-}" ]]; then
+    grep -m25 "${ERROR_PATTERN}" "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
+  fi
+  if [[ ! -s "${REPORT_FILE}" ]]; then
+    echo "Unknown failure, check output.log" > "${REPORT_FILE}"
+  fi
+fi
+
+# number of failures = number of lines in report, unless file already created with custom count
+if [[ ! -s "${REPORT_DIR}/failures" ]]; then
+  wc -l "$REPORT_FILE" | awk '{ print $1 }' > "$REPORT_DIR/failures"
+fi
+
+# exit with failure if report is not empty
+if [[ -s "${REPORT_FILE}" ]]; then
+  rc=1
+fi
+
+exit ${rc}

hadoop-ozone/dev-support/checks/acceptance.sh

@@ -30,6 +30,7 @@ OZONE_ROOT=$(pwd -P)
 source "${DIR}/_lib.sh"
 
 REPORT_DIR=${OUTPUT_DIR:-"${OZONE_ROOT}/target/acceptance"}
+REPORT_FILE="$REPORT_DIR/summary.txt"
 
 OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout -Dscan=false)
 DIST_DIR="${OZONE_ROOT}/hadoop-ozone/dist/target/ozone-$OZONE_VERSION"
@@ -50,7 +51,7 @@ if [[ "${OZONE_ACCEPTANCE_SUITE}" == "s3a" ]]; then
   fi
 
   if ! download_hadoop_aws "${HADOOP_AWS_DIR}"; then
-    echo "Failed to download Hadoop ${HADOOP_VERSION}" > "${REPORT_DIR}/summary.txt"
+    echo "Failed to download Hadoop ${HADOOP_VERSION}" > "${REPORT_FILE}"
     exit 1
   fi
 fi
@@ -59,22 +60,21 @@ export OZONE_ACCEPTANCE_SUITE OZONE_ACCEPTANCE_TEST_TYPE
 
 cd "$DIST_DIR/compose" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"
-RES=$?
+rc=$?
 
 if [[ "${OZONE_ACCEPTANCE_TEST_TYPE}" == "maven" ]]; then
   pushd result
   source "${DIR}/_mvn_unit_report.sh"
   find . -name junit -print0 | xargs -r -0 rm -frv
   cp -rv * "${REPORT_DIR}"/
   popd
+  ERROR_PATTERN="\[ERROR\]"
 else
   cp -rv result/* "$REPORT_DIR/"
-  if [[ -f "${REPORT_DIR}/log.html" ]]; then
-    cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
-  fi
-  grep -A1 FAIL "${REPORT_DIR}/output.log" | grep -v '^Output' > "${REPORT_DIR}/summary.txt"
+  grep -A1 FAIL "${REPORT_DIR}/output.log" | grep -v '^Output' > "${REPORT_FILE}"
+  ERROR_PATTERN="FAIL"
 fi
 
 find "$REPORT_DIR" -type f -empty -not -name summary.txt -print0 | xargs -0 rm -v
 
-exit $RES
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/author.sh

@@ -16,17 +16,19 @@
 
 #checks:basic
 
+set -u -o pipefail
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
 REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/author"}
 mkdir -p "$REPORT_DIR"
 REPORT_FILE="$REPORT_DIR/summary.txt"
 
-grep -r --include="*.java" "@author" . | tee "$REPORT_FILE"
-
-wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
+rc=0
+if grep -r --include="*.java" "@author" . | tee "$REPORT_FILE"; then
+  rc=1
 fi
+
+ERROR_PATTERN=""
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/bats.sh

@@ -16,6 +16,8 @@
 
 #checks:basic
 
+set -u -o pipefail
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "${DIR}/../../.." || exit 1
 
@@ -39,11 +41,11 @@ find * \( \
     \) -print0 \
   | xargs -0 -n1 bats --formatter tap \
   | tee -a "${REPORT_DIR}/output.log"
+rc=$?
 
 grep '^\(not ok\|#\)' "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
 
 grep -c '^not ok' "${REPORT_FILE}" > "${REPORT_DIR}/failures"
 
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
+ERROR_PATTERN=""
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/checkstyle.sh

@@ -53,15 +53,8 @@ find "." -name checkstyle-errors.xml -print0 \
       -e "s/>/>/g" \
   | tee "$REPORT_FILE"
 
-# check if Maven failed due to some error other than checkstyle violation
-if [[ ${rc} -ne 0 ]] && [[ ! -s "${REPORT_FILE}" ]]; then
-  grep -m1 -F '[ERROR]' "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
-fi
-
 ## generate counter
 grep -c ':' "$REPORT_FILE" > "$REPORT_DIR/failures"
 
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
-exit ${rc}
+ERROR_PATTERN="\[ERROR\]"
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/docs.sh

@@ -33,10 +33,5 @@ rc=$?
 
 grep -o 'ERROR.*' "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
 
-wc -l "${REPORT_FILE}" | awk '{ print $1 }' > "${REPORT_DIR}/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
-
-exit ${rc}
+ERROR_PATTERN=""
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/findbugs.sh

@@ -16,6 +16,8 @@
 
 #checks:basic
 
+set -u -o pipefail
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "$DIR/../../.." || exit 1
 
@@ -25,30 +27,25 @@ source "${DIR}/_lib.sh"
 
 install_spotbugs
 
+REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
+mkdir -p "$REPORT_DIR"
+REPORT_FILE="$REPORT_DIR/summary.txt"
+
 MAVEN_OPTIONS='-B -fae -DskipRecon --no-transfer-progress'
 
 if [[ "${OZONE_WITH_COVERAGE}" != "true" ]]; then
   MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
 fi
 
 #shellcheck disable=SC2086
-mvn ${MAVEN_OPTIONS} test-compile spotbugs:spotbugs "$@"
+mvn ${MAVEN_OPTIONS} test-compile spotbugs:spotbugs "$@" | tee "${REPORT_DIR}/output.log"
 rc=$?
 
-REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
-mkdir -p "$REPORT_DIR"
-REPORT_FILE="$REPORT_DIR/summary.txt"
-
 touch "$REPORT_FILE"
 
 find hadoop-hdds hadoop-ozone -name spotbugsXml.xml -print0 | xargs -0 unionBugs -output "${REPORT_DIR}"/summary.xml
 convertXmlToText "${REPORT_DIR}"/summary.xml | tee -a "${REPORT_FILE}"
 convertXmlToText -html:fancy-hist.xsl "${REPORT_DIR}"/summary.xml "${REPORT_DIR}"/summary.html
 
-wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
-
-exit ${rc}
+ERROR_PATTERN="\[ERROR\]"
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/junit.sh

@@ -100,14 +100,10 @@ for i in $(seq 1 ${ITERATIONS}); do
   fi
 done
 
-# check if Maven failed due to some error other than test failure
-if [[ ${rc} -ne 0 ]] && [[ ! -s "${REPORT_FILE}" ]]; then
-  grep -m1 -F '[ERROR]' "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
-fi
-
 if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
   #Archive combined jacoco records
   mvn -B -N jacoco:merge -Djacoco.destFile=$REPORT_DIR/jacoco-combined.exec -Dscan=false
 fi
 
-exit ${rc}
+ERROR_PATTERN="\[ERROR\]"
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/kubernetes.sh

@@ -35,6 +35,7 @@ else
 fi
 
 REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/kubernetes"}
+REPORT_FILE="$REPORT_DIR/summary.txt"
 
 OZONE_VERSION=$(mvn help:evaluate -Dexpression=ozone.version -q -DforceStdout -Dscan=false)
 DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
@@ -48,10 +49,10 @@ mkdir -p "$REPORT_DIR"
 
 cd "$DIST_DIR/kubernetes/examples" || exit 1
 ./test-all.sh 2>&1 | tee "${REPORT_DIR}/output.log"
-RES=$?
+rc=$?
 cp -r result/* "$REPORT_DIR/"
-cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
 
-grep -A1 FAIL "${REPORT_DIR}/output.log" > "${REPORT_DIR}/summary.txt"
+grep -A1 FAIL "${REPORT_DIR}/output.log" > "${REPORT_FILE}"
 
-exit $RES
+ERROR_PATTERN="FAIL"
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/license.sh

@@ -67,9 +67,7 @@ grep '(' ${src} \
     || true ) \
   | sort -u \
   | tee "${REPORT_FILE}"
+rc=$?
 
-wc -l "${REPORT_FILE}" | awk '{ print $1 }' > "${REPORT_DIR}/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
+ERROR_PATTERN=""
+source "${DIR}/_post_process.sh"

hadoop-ozone/dev-support/checks/rat.sh

@@ -28,9 +28,5 @@ mvn -B --no-transfer-progress -fn org.apache.rat:apache-rat-plugin:check "$@"
 
 grep -r --include=rat.txt "!????" $dirs | tee "$REPORT_FILE"
 
-wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
-   exit 1
-fi
-
+ERROR_PATTERN="\[ERROR\]"
+source "${DIR}/_post_process.sh"