VAULT-17592 Extract failed Go test results across runners (#21625)

hashicorp · Jul 7, 2023 · 5defa54 · 5defa54
1 parent c7d3f8c
commit 5defa54
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 10 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -288,7 +288,6 @@ jobs:
     steps:
       - run: |
           tr -d '\n' <<< '${{ toJSON(needs.*.result) }}' | grep -q -v -E '(failure|cancelled)'
-
   notify-tests-completed-failures-oss:
     if: ${{ always() && github.repository == 'hashicorp/vault' && needs.tests-completed.result == 'failure' && (github.ref_name == 'main' || startsWith(github.ref_name, 'release/')) }}
     runs-on: ubuntu-latest
@@ -342,3 +341,74 @@ jobs:
           payload: |
             {"text":"Enterprise test failures on ${{ github.ref_name }}","blocks":[{"type":"header","text":{"type":"plain_text","text":":rotating_light: Enterprise test failures :rotating_light:","emoji":true}},{"type":"divider"},{"type":"section","text":{"type":"mrkdwn","text":"test(s) failed on ${{ github.ref_name }}"},"accessory":{"type":"button","text":{"type":"plain_text","text":"View Failing Workflow","emoji":true},"url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}}]}
     
+  test-summary:
+    name: Go test failures
+    runs-on: ubuntu-latest
+    if: success() || failure() || needs.tests-completed.result == 'skipped'
+    needs:
+      - test-go
+      - test-go-fips
+      - test-go-race
+      - tests-completed
+    steps:
+      - name: Download failure summary
+        uses: actions/download-artifact@v3
+        with:
+          name: failure-summary
+      - name: Prepare failure summary
+        run: |
+          # We will store the jq query results in a temp file
+          temp_file_name=temp-$(date +%s)
+
+          # The 'jq' command below filters and formats JSON data from input files to generate a failure summary report.
+          # The query is a bit of a nightmare, though, so let's have a closer look at it.
+          #
+          # The command takes input from files matching the pattern "failure-summary-*.json".
+          # The input files should contain streams of JSON objects(with no commas in between),
+          # one per line, representing test results.
+          # Each object should have the "Action" and "Package" keys.
+          #
+          # We invoke the command with two flags:
+          # - '-r' specifies that the output should be in raw format,
+          #   without any JSON formatting. (I.e. no quotes).
+          # - '-n' tells 'jq' not to read any input from the command line.
+          #   It is used when input is provided through the 'inputs' function or other methods.
+          #
+          # 'inputs':
+          # Read JSON objects from the input files specified after the 'jq' command.
+          # We assume that the input files contain one JSON object per line.
+          #
+          # 'select(.Action == "fail") | select(.Test != null)':
+          # Filter JSON array to contain only objects where the value of "Action" is "fail"
+          # and the value of "Test" key is not null.
+          #
+          # The remaining part of the query constructs a formatted string for each filtered JSON object`:
+          # - '\(.Package)' and '\(.Test)' insert the values of the "Package" and "Test" keys into the string,
+          #   respectively.
+          # - 'input_filename' is a special variable in 'jq' that represents the name of the input file being processed.
+          # - 'split("-")' splits the input filename on the hyphen ("-") character and returns an array of the
+          #   resulting parts.
+          # - '.sub(".json"; "")' removes the ".json" extension from the string.
+          # - The third part of the filename is extracted using '.split("-") | .[2]'.
+          # - If the fourth part of the filename exists, it contains the test type.
+          #   Otherwise, the default value "normal" is used.
+          # - The '.sub(".json"; "")' removes the ".json" extension from the string.
+          #
+          # The filtered and formatted data is outputted as rows of a Markdown table, like this:
+          # | pkg1 | test1 | 1 | normal |
+          # | pkg2 | test2 | 4 | race |
+          # | pkg3 | test3 | 6 | fips |
+
+          jq -r -n 'inputs | select(.Action == "fail") | select(.Test != null) | "| \(.Package) | \(.Test) | \(input_filename | split("-") | .[2] | sub(".json"; "")) | \(input_filename | split("-") | .[3] // "normal" | sub(".json";"") )"' failure-summary-*.json | sort >> "$temp_file_name"
+
+          # if there are test failures, present them in a format of a GH Markdown table
+          if [ -s "$temp_file_name" ]; then
+            # shellcheck disable=SC2129
+            # Here we create the headings for the summary table
+            echo "### Go test failures" >> "$GITHUB_STEP_SUMMARY"
+            echo "| Package | Test | Runner Index | Test Type |" >> "$GITHUB_STEP_SUMMARY"
+            echo "| ------- | ---- | ------------ | --------- |" >> "$GITHUB_STEP_SUMMARY"
+            cat "$temp_file_name" >>  "$GITHUB_STEP_SUMMARY"
+          else
+            echo "### All Go tests passed! :white_check_mark:" >> "$GITHUB_STEP_SUMMARY"
+          fi
diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml
@@ -230,6 +230,7 @@ jobs:
             go run gotest.tools/gotestsum --format=short-verbose \
               --junitfile test-results/go-test/results-${{ matrix.runner-index }}.xml \
               --jsonfile test-results/go-test/results-${{ matrix.runner-index }}.json \
+              --jsonfile-timing-events failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json \
               -- \
               -tags "${{ inputs.go-build-tags }}" \
               -timeout=${{ env.TIMEOUT_IN_MINUTES }}m \
@@ -258,10 +259,10 @@ jobs:
         with:
           name: test-results${{ inputs.name }}
           path: test-results/
-        if: success() || failure() 
-      - name: Create a summary of tests
-        uses: test-summary/action@62bc5c68de2a6a0d02039763b8c754569df99e3f   # TSCCR: no entry for repository "test-summary/action"
+        if: success() || failure()
+      - name: Upload failure summary
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        if: success() || failure()
         with:
-          paths: "test-results/go-test/results-${{ matrix.runner-index }}.xml"
-          show: "fail"
-        if: success() || failure() 
+          name: failure-summary
+          path: failure-summary-${{ matrix.runner-index }}${{inputs.name}}.json
diff --git a/go.mod b/go.mod
@@ -219,7 +219,7 @@ require (
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/ory-am/dockertest.v3 v3.3.4
-	gotest.tools/gotestsum v1.9.0
+	gotest.tools/gotestsum v1.10.0
 	honnef.co/go/tools v0.4.3
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	layeh.com/radius v0.0.0-20190322222518-890bc1058917

diff --git a/go.sum b/go.sum
@@ -3854,8 +3854,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
-gotest.tools/gotestsum v1.9.0 h1:Jbo/0k/sIOXIJu51IZxEAt27n77xspFEfL6SqKUR72A=
-gotest.tools/gotestsum v1.9.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
+gotest.tools/gotestsum v1.10.0 h1:lVO4uQJoxdsJb7jgmr1fg8QW7zGQ/tuqvsq5fHKyoHQ=
+gotest.tools/gotestsum v1.10.0/go.mod h1:6JHCiN6TEjA7Kaz23q1bH0e2Dc3YJjDUZ0DmctFZf+w=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=