test.yml

name: Test

on:
  pull_request:
  push:
    branches:
      - master
      - release/*
  workflow_dispatch:

defaults:
  run:
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
  contents: read

jobs:
  discover:
    name: Discover Test Groups
    runs-on: ubuntu-latest
    outputs:
      groups: ${{ steps.test.outputs.groups }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'
          fetch-depth: 0
      - id: test
        env:
          # Unit test groups other than unit-rest
          utests: |
            [
              {"name": "unit-cli", "packages": ["./cli/...", "./cmd/...", "./api/..."]},
              {"name": "unit-storage", "packages": ["./storage/...", "./extern/..."]},
              {"name": "unit-node", "packages": ["./node/..."]}
            ]
          # Other tests that require special configuration
          otests: |
            [
              {
                "name": "multicore-sdr",
                "packages": ["./storage/sealer/ffiwrapper"],
                "go_test_flags": "-run=TestMulticoreSDR",
                "test_rustproofs_logs": "1"
              }, {
                "name": "conformance",
                "packages": ["./conformance"],
                "go_test_flags": "-run=TestConformance",
                "skip_conformance": "0"
              }
            ]
          # Mapping from test group names to custom runner labels
          # The jobs default to running on the default hosted runners (4 CPU, 16 RAM).
          # We use self-hosted xlarge (4 CPU, 8 RAM; and large - 2 CPU, 4 RAM) runners
          #   to extend the available runner capacity (60 default hosted runners).
          # We use self-hosted 4xlarge (16 CPU, 32 RAM; and 2xlarge - 8 CPU, 16 RAM) self-hosted
          #   to support resource intensive jobs.
          runners: |
            {
              "itest-niporep_manual": ["self-hosted", "linux", "x64", "4xlarge"],
              "itest-sector_pledge": ["self-hosted", "linux", "x64", "4xlarge"],
              "itest-worker": ["self-hosted", "linux", "x64", "4xlarge"],
              "itest-manual_onboarding": ["self-hosted", "linux", "x64", "4xlarge"],

              "itest-gateway": ["self-hosted", "linux", "x64", "2xlarge"],
              "itest-sector_import_full": ["self-hosted", "linux", "x64", "2xlarge"],
              "itest-sector_import_simple": ["self-hosted", "linux", "x64", "2xlarge"],
              "itest-wdpost": ["self-hosted", "linux", "x64", "2xlarge"],
              "unit-storage": ["self-hosted", "linux", "x64", "2xlarge"],

              "itest-cli": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-deals_invalid_utf8_label": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-decode_params": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-dup_mpool_messages": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_account_abstraction": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_api": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_balance": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_bytecode": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_config": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_conformance": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_deploy": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_fee_history": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-eth_transactions": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-fevm_address": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-fevm_events": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-gas_estimation": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-get_messages_in_ts": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-lite_migration": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-lookup_robust_address": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-mempool": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-mpool_msg_uuid": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-mpool_push_with_uuid": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-msgindex": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-multisig": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-net": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-nonce": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-path_detach_redeclare": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-pending_deal_allocation": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-remove_verifreg_datacap": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-sector_miner_collateral": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-sector_numassign": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-self_sent_txn": ["self-hosted", "linux", "x64", "xlarge"],
              "itest-verifreg": ["self-hosted", "linux", "x64", "xlarge"],
              "multicore-sdr": ["self-hosted", "linux", "x64", "xlarge"],
              "unit-node": ["self-hosted", "linux", "x64", "xlarge"]
            }
          # A list of test groups that require Proof Parameters to be fetched
          parameters: |
            [
              "conformance",
              "itest-api",
              "itest-direct_data_onboard_verified",
              "itest-direct_data_onboard",
              "itest-manual_onboarding",
              "itest-niporep_manual",
              "itest-net",
              "itest-path_detach_redeclare",
              "itest-sealing_resources",
              "itest-sector_import_full",
              "itest-sector_import_simple",
              "itest-sector_pledge",
              "itest-sector_unseal",
              "itest-wdpost_no_miner_storage",
              "itest-wdpost_worker_config",
              "itest-wdpost",
              "itest-worker",
              "multicore-sdr",
              "unit-cli",
              "unit-storage"
            ]
        run: |
          # Create a list of integration test groups
          itests="$(
            find ./itests -name "*_test.go" | \
              jq -R '{
                "name": "itest-\(. | split("/") | .[2] | sub("_test.go$";""))",
                "packages": [.]
              }' | jq -s
            )"

          # Create a list of packages that are covered by the integration and unit tests
          packages="$(jq -n --argjson utests "$utests" '$utests | map(.packages) | flatten | . + ["./itests/..."]')"

          # Create a new group for the unit tests that are not yet covered
          rest="$(
            find . -name "*_test.go" | cut -d/ -f2 | sort | uniq | \
              jq -R '"./\(.)/..."' | \
              jq -s --argjson p "$packages" '{"name": "unit-rest", "packages": (. - $p)}'
          )"

          # Combine the groups for integration tests, unit tests, the new unit-rest group, and the other tests
          groups="$(jq -n --argjson i "$itests" --argjson u "$utests" --argjson r "$rest" --argjson o "$otests" '$i + $u + [$r] + $o')"

          # Apply custom runner labels to the groups
          groups="$(jq -n --argjson g "$groups" --argjson r "$runners" '$g | map(. + {"runner": (.name as $n | $r | .[$n]) })')"

          # Apply the needs_parameters flag to the groups
          groups="$(jq -n --argjson g "$groups" --argjson p "$parameters" '$g | map(. + {"needs_parameters": ([.name] | inside($p)) })')"

          # Output the groups
          echo "groups=$groups"
          echo "groups=$(jq -nc --argjson g "$groups" '$g')" >> $GITHUB_OUTPUT
  cache:
    name: Cache Dependencies
    runs-on: ubuntu-latest
    outputs:
      fetch_params_key: ${{ steps.fetch_params.outputs.key }}
      fetch_params_path: ${{ steps.fetch_params.outputs.path }}
      make_deps_key: ${{ steps.make_deps.outputs.key }}
      make_deps_path: ${{ steps.make_deps.outputs.path }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'
          fetch-depth: 0
      - id: fetch_params
        env:
          CACHE_KEY: fetch-params-${{ hashFiles('./extern/filecoin-ffi/parameters.json') }}
          CACHE_PATH: |
            /var/tmp/filecoin-proof-parameters/
        run: |
          echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT
          echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT
      - id: make_deps
        env:
          CACHE_KEY: ${{ runner.os }}-${{ runner.arch }}-make-deps-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}-p
          CACHE_PATH: |
            ./extern/filecoin-ffi/filcrypto.h
            ./extern/filecoin-ffi/libfilcrypto.a
            ./extern/filecoin-ffi/filcrypto.pc
        run: |
          echo -e "key=$CACHE_KEY" | tee -a $GITHUB_OUTPUT
          echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT
      - id: restore_fetch_params
        uses: actions/cache/restore@v4
        with:
          key: ${{ steps.fetch_params.outputs.key }}
          path: ${{ steps.fetch_params.outputs.path }}
          lookup-only: true
      - id: restore_make_deps
        uses: actions/cache/restore@v4
        with:
          key: ${{ steps.make_deps.outputs.key }}
          path: ${{ steps.make_deps.outputs.path }}
          lookup-only: true
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
        uses: ./.github/actions/install-system-dependencies
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
        uses: ./.github/actions/install-go
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: FFI_PORTABLE=1 make deps
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
        run: make lotus
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
        run: ./lotus fetch-params 2048
      - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          key: ${{ steps.fetch_params.outputs.key }}
          path: ${{ steps.fetch_params.outputs.path }}
      - if: steps.restore_make_deps.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          key: ${{ steps.make_deps.outputs.key }}
          path: ${{ steps.make_deps.outputs.path }}
  test:
    needs: [discover, cache]
    name: Test (${{ matrix.name }})
    runs-on: ${{ github.repository_owner == 'filecoin-project' && matrix.runner || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.discover.outputs.groups) }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'
          fetch-depth: 0
      - uses: ./.github/actions/install-system-dependencies
      - uses: ./.github/actions/install-go
      - name: Install gotestsum
        run: go install gotest.tools/gotestsum@latest
      - name: Restore cached make deps outputs
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.cache.outputs.make_deps_key }}
          path: ${{ needs.cache.outputs.make_deps_path }}
          fail-on-cache-miss: true
      - if: ${{ matrix.needs_parameters }}
        name: Restore cached fetch params outputs
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.cache.outputs.fetch_params_key }}
          path: ${{ needs.cache.outputs.fetch_params_path }}
          fail-on-cache-miss: true
      # TODO: Install statediff (used to be used for conformance)
      - name: Create temporary directory for reports
        id: reports
        run: mktemp -d | xargs -0 -I{} echo "path={}" | tee -a $GITHUB_OUTPUT
      # TODO: Track coverage (used to be tracked for conformance)
      - name: Run tests
        env:
          NAME: ${{ matrix.name }}
          LOTUS_SRC_DIR: ${{ github.workspace }}
          REPORTS_PATH: ${{ steps.reports.outputs.path }}
          SKIP_CONFORMANCE: ${{ matrix.skip_conformance || '1' }}
          TEST_RUSTPROOFS_LOGS: ${{ matrix.test_rustproofs_logs || '0' }}
          FORMAT: ${{ matrix.format || 'standard-verbose' }}
          PACKAGES: ${{ join(matrix.packages, ' ') }}
        run: |
          gotestsum \
            --format "$FORMAT" \
            --junitfile "$REPORTS_PATH/$NAME.xml" \
            --jsonfile "$REPORTS_PATH/$NAME.json" \
            --packages="$PACKAGES" \
            -- ${{ matrix.go_test_flags || '' }}
      - name: Modify junit.xml for BuildPulse
        env:
          NAME: ${{ matrix.name }}
          REPORTS_PATH: ${{ steps.reports.outputs.path }}
          PACKAGES: ${{ join(matrix.packages, ' ') }}
        if: always()
        run: |
          # Modify test suite name and classname attributes in JUnit XML for better grouping
          # in BuildPulse. itests are run with go test ./itests/file_test.go and therefore Go
          # assigns the name and classname attributes to "command-line-arguments". Others get the
          # package name for both.
          if [[ "${{ matrix.name }}" == itest-* ]]; then
            PACKAGE_NAME=$(basename "$PACKAGES" .go)
            sed -i 's/ name="command-line-arguments"/ name="itests"/g' "$REPORTS_PATH/$NAME.xml"
            sed -i 's/classname="command-line-arguments"/classname="'"$PACKAGE_NAME"'"/g' "$REPORTS_PATH/$NAME.xml"
          else
            sed -i 's# name="github.com/filecoin-project/lotus/\(.*\)"# name="'${{ matrix.name }}':\1"#g' "$REPORTS_PATH/$NAME.xml"
          fi
          cat "$REPORTS_PATH/$NAME.xml"
      - if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.name }}
          path: |
            ${{ steps.reports.outputs.path }}/${{ matrix.name }}.xml
            ${{ steps.reports.outputs.path }}/${{ matrix.name }}.json
        continue-on-error: true