Improve test hygiene and parallelize tool tests (#432)

A number of fixes/improvements: 1. Tool tests were running twice on PRs needlessly 2. Extract tool tests into their own jobs. This could run into some unfortunate "throttling" in the case of multiple PRs at once, but it should be better for the general case 3. Add ci-debugger to the tool tests invocations on non-Windows 4. Use more sane timeouts 5. Some better logging 6. Makes some tests run on Windows and mark others to skip Windows [Successful run Windows](https://github.com/trunk-io/plugins/actions/runs/5836160966/job/15829177813)
trunk-io · Aug 11, 2023 · 814ac5a · 814ac5a
1 parent adf11ee
commit 814ac5a
Show file tree

Hide file tree

Showing 27 changed files with 842 additions and 43 deletions.
diff --git a/.github/actions/linter_tests/action.yaml b/.github/actions/linter_tests/action.yaml
@@ -74,6 +74,7 @@ runs:
 
     - name: Run plugin tests
       if: runner.os != 'Windows'
+      # trunk-ignore(semgrep/yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha)
       uses: trunk-io/breakpoint@v1
       with:
         breakpoint-id: trunk-plugins-linter-tests

diff --git a/.github/actions/tool_tests/action.yaml b/.github/actions/tool_tests/action.yaml
@@ -16,6 +16,9 @@ inputs:
     description: Additional args to append to the test invocation
     required: false
     default: tools
+  trunk-token:
+    description: CI debugger api token
+    required: true
 
 runs:
   # TODO(Tyler): See if this can be converted to a js action
@@ -37,9 +40,25 @@ runs:
       working-directory: ${{ inputs.path }}
 
     - name: Run plugin tests
+      if: runner.os == 'Windows'
       run: npm test --ci ${{ inputs.append-args }}
       shell: bash
       working-directory: ${{ inputs.path }}
       env:
         PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
         PLUGINS_TEST_CLI_PATH: ${{ inputs.cli-path }}
+
+    - name: Run plugin tests
+      if: runner.os != 'Windows'
+      # trunk-ignore(semgrep/yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha)
+      uses: trunk-io/breakpoint@v1
+      with:
+        breakpoint-id: trunk-plugins-tool-tests
+        shell: bash
+        working-directory: ${{ inputs.path }}
+        trunk-token: ${{ inputs.trunk-token  }}
+        org: trunk-staging-org
+        run: npm test --ci ${{ inputs.append-args }}
+      env:
+        PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
+        PLUGINS_TEST_CLI_PATH: ${{ inputs.cli-path }}
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -58,7 +58,7 @@ jobs:
   linter_tests_release:
     name: Plugin Tests Release
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 120
+    timeout-minutes: 150
     strategy:
       fail-fast: false
       matrix:
@@ -156,7 +156,10 @@ jobs:
       # TODO(lauri): For now this just runs on the hardcoded versions. We should configure this
       # akin to the linter_tests job.
       - name: Tool Tests
+        if: failure() || success()
         uses: ./.github/actions/tool_tests
+        with:
+          trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 
       - name: Upload Test Outputs for Upload Job
         # Only upload results from latest. Always run, except when cancelled.

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -83,16 +83,13 @@ jobs:
 
   # Run tests against all linters for known_good_version and latest version
   linter_tests:
-    name: Plugin Tests
+    name: Linter Tests
     runs-on: ${{ matrix.os }}
     needs: detect_changes
-    permissions:
-      pull-requests: write
     if:
       needs.detect_changes.outputs.linters == 'true' || needs.detect_changes.outputs.all-linters ==
-      'linters' || needs.detect_changes.outputs.tools == 'true' ||
-      needs.detect_changes.outputs.all-tools == 'tools'
-    timeout-minutes: 120
+      'linters'
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
@@ -131,12 +128,35 @@ jobs:
         with:
           linter-version: Latest
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
-          # TODO(Tyler): We need downloads to work with known_bad_versions
           append-args: ${{ needs.detect_changes.outputs.linters-files }}
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 
+  tool_tests:
+    name: Tool Tests
+    runs-on: ${{ matrix.os }}
+    needs: detect_changes
+    if:
+      needs.detect_changes.outputs.tools == 'true' || needs.detect_changes.outputs.all-tools ==
+      'tools'
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Cache tool downloads
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          path: /tmp/plugins_testing_download_cache
+          # No need to key on trunk version unless we change how we store downloads.
+          key: trunk-${{ runner.os }}
+
       - name: Tool Tests
         # Run tests using KnownGoodVersion with any modified tools and conditionally all tools. Don't run when cancelled.
+        # TODO(Tyler): Wire up Latest tests and ReleaseVersionService
         uses: ./.github/actions/tool_tests
         if:
           (failure() || success()) && needs.detect_changes.outputs.tools == 'true' ||
@@ -145,15 +165,7 @@ jobs:
           append-args:
             ${{ needs.detect_changes.outputs.all-tools }} ${{
             needs.detect_changes.outputs.tools-files }}
-
-      - name: Tool Tests Latest
-        # Run tests using Latest with any modified tools (see filters.yaml). Don't run when cancelled.
-        if:
-          (failure() || success()) && needs.detect_changes.outputs.tools == 'true' &&
-          needs.detect_changes.outputs.tools-files != ''
-        uses: ./.github/actions/tool_tests
-        with:
-          append-args: ${{ needs.detect_changes.outputs.tools-files }}
+          trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 
   # Run repo healthcheck tests
   repo_tests:
@@ -166,14 +178,18 @@ jobs:
     if: ${{ always() }}
     runs-on: ubuntu-latest
     name: Aggregate Test Results
-    needs: [linter_tests, repo_tests]
+    needs: [linter_tests, tool_tests, repo_tests]
     steps:
       - run: |
           linters="${{ needs.linter_tests.result }}"
           repos="${{ needs.repo_tests.result }}"
+          tools="${{ needs.tool_tests.result }}"
           if [[ $linters != "success" && $linters != "skipped" ]]; then
             echo "Detected failure in linter tests"
             exit 1
+          elif [[ $tools != "success" && $tools != "skipped" ]]; then
+            echo "Detected failure in tool tests"
+            exit 1
           elif [[ $repos != "success" && $repos != "skipped" ]]; then
             echo "Detected failure in repo tests"
             exit 1

diff --git a/.github/workflows/windows_nightly.yaml b/.github/workflows/windows_nightly.yaml
@@ -5,7 +5,7 @@ on:
     - cron: 0 8 * * 1-5
   # Only enable on pull requests for testing
   # pull_request:
-  # paths: [.github/workflows/windows_nightly.yaml]
+  #   paths: [.github/workflows/windows_nightly.yaml]
   workflow_dispatch: {}
 
 permissions: read-all
@@ -14,7 +14,7 @@ jobs:
   # Run tests against all linters for snapshots and latest version as they exist on main
   # This job is used to diagnose plugin config health in advance of a release
   linter_tests_main:
-    name: Plugin Tests Main
+    name: Linter Tests Main
     runs-on: windows-latest
     timeout-minutes: 90
     strategy:
@@ -56,8 +56,41 @@ jobs:
           # CI Debugger is not yet supported on Windows
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 
-      # # TODO(lauri): For now this just runs on the hardcoded versions. We should configure this
-      # # akin to the linter_tests job.
-      # - name: Tool Tests
-      #   if: failure() || success()
-      #   uses: ./.github/actions/tool_tests
+  tool_tests_main:
+    name: Tool Tests Main
+    runs-on: windows-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+
+      - name: Cache tool downloads
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          path: /tmp/plugins_testing_download_cache
+          # No need to key on trunk version unless we change how we store downloads.
+          key: trunk-${{ runner.os }}
+
+      - name: Delete cache
+        # For now, avoid deleting cache on pull request changes to nightly. This improves PR experience.
+        if: env.TRIGGER != 'pull_request'
+        run: |
+          if [ -d "/tmp/plugins_testing_download_cache" ]
+          then
+            tmp_dir=/tmp/${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}
+            mv "/tmp/plugins_testing_download_cache" ${tmp_dir}
+            chmod -R u+w ${tmp_dir}
+            rm -rf ${tmp_dir}
+          fi
+        shell: bash
+
+      # TODO(lauri): For now this just runs on the hardcoded versions. We should configure this
+      # akin to the linter_tests job.
+      - name: Tool Tests
+        if: failure() || success()
+        uses: ./.github/actions/tool_tests
+        with:
+          cli-path: ${{ github.workspace }}\trunk.ps1
+          append-args: tools -- --maxWorkers=5
+          # CI Debugger is not yet supported on Windows
+          trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml
@@ -2,7 +2,7 @@ version: 0.1
 
 # version used for local trunk runs and testing
 cli:
-  version: 1.13.1-beta.21
+  version: 1.13.1-beta.24
 
 api:
   address: api.trunk-staging.io:8443