Integration tests #101
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| on: | |
| # Use the following to explicitly start this workflow. | |
| # packages/deployment/scripts/start-github-integration-test.sh <BRANCH-OR-TAG> | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| - 'release-*' | |
| - 'dev-*' | |
| - beta | |
| tags: | |
| - '@agoric/sdk@*' | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - converted_to_draft | |
| - ready_for_review | |
| - labeled | |
| - unlabeled | |
| - auto_merge_enabled | |
| - auto_merge_disabled | |
| merge_group: | |
| schedule: | |
| - cron: '17 6 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre_check: | |
| uses: ./.github/workflows/pre-check-integration.yml | |
| # This job is meant to emulate what developers working with the Agoric platform will experience | |
| # It should be kept in sync with https://agoric.com/documentation/getting-started/ | |
| getting-started: | |
| needs: pre_check | |
| if: needs.pre_check.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cli: [link-cli/yarn, registry/yarn, registry/npm, registry/npx] | |
| timeout-minutes: 40 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Reconfigure git to use HTTP authentication | |
| run: git config --global url."https://github.com/".insteadOf ssh://git@github.com/ | |
| shell: bash | |
| # Prerequisites | |
| - uses: ./.github/actions/restore-node | |
| with: | |
| node-version: 18.18 | |
| keep-endo: 'true' | |
| # Select a branch on dapp to test against by adding text to the body of the | |
| # pull request. For example: #getting-started-branch: zoe-release-0.7.0 | |
| # The default is 'main' | |
| - name: Get the appropriate dapp branch | |
| id: get-branch | |
| uses: actions/github-script@v7 | |
| with: | |
| result-encoding: string | |
| script: | | |
| let branch = 'main'; | |
| if (context.payload.pull_request) { | |
| const { body } = context.payload.pull_request; | |
| const regex = /^\#getting-started-branch:\s+(\S+)/m; | |
| const result = regex.exec(body); | |
| if (result) { | |
| branch = result[1]; | |
| } | |
| } | |
| console.log('getting-started dapp branch: ' + branch); | |
| return branch; | |
| - name: Start local NPM registry | |
| if: ${{ startsWith(matrix.cli, 'registry') }} | |
| run: | | |
| set -xe | |
| git reset --hard HEAD | |
| git clean -f | |
| git config user.email "noreply@agoric.com" | |
| REGISTRY_PUBLISH_WORKSPACES="$HOME/endo" | |
| timeout 900 scripts/registry.sh bg-publish ${{ matrix.cli }} | |
| - name: run agoric-cli integration-test | |
| # These integration tests can be flaky so retry automatically | |
| uses: nick-fields/retry@v3 | |
| with: | |
| # This step usually takes <4 minutes (after 6m setup). The | |
| # deployment-test running in parallel takes 35 minutes so 20 minutes | |
| # won't increase the time for this workflow to complete. | |
| timeout_minutes: 20 | |
| command: scripts/registry.sh test ${{ matrix.cli }} ${{steps.get-branch.outputs.result}} | |
| on_retry_command: rm -f $HOME/bin/agoric | |
| - name: notify on failure | |
| if: > | |
| failure() && github.event_name != 'pull_request' && | |
| github.repository_owner == 'agoric' | |
| uses: ./.github/actions/notify-status | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| from: ${{ secrets.NOTIFY_EMAIL_FROM }} | |
| to: ${{ secrets.NOTIFY_EMAIL_TO }} | |
| password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }} | |
| deployment-test: | |
| needs: pre_check | |
| if: needs.pre_check.outputs.should_run == 'true' | |
| runs-on: ubuntu-22.04 # jammy (LTS) | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| path: ./agoric-sdk | |
| - run: sudo packages/deployment/scripts/install-deps.sh | |
| working-directory: ./agoric-sdk | |
| - uses: ./agoric-sdk/.github/actions/restore-golang | |
| with: | |
| go-version: '1.20' | |
| path: ./agoric-sdk | |
| - uses: ./agoric-sdk/.github/actions/restore-node | |
| with: | |
| # XXX loadgen not compatible with 18.19 https://github.com/Agoric/agoric-sdk/pull/8365#issuecomment-1848003597 | |
| node-version: 18.18 | |
| path: ./agoric-sdk | |
| # Forces xsnap to initialize all memory to random data, which increases | |
| # the chances the content of snapshots may deviate between validators | |
| xsnap-random-init: '1' | |
| # Select a branch on loadgen to test against by adding text to the body of the | |
| # pull request. For example: #loadgen-branch: user-123-update-foo | |
| # The default is 'main' | |
| - name: Get the appropriate loadgen branch | |
| id: get-loadgen-branch | |
| uses: actions/github-script@v7 | |
| with: | |
| result-encoding: string | |
| script: | | |
| let branch = 'main'; | |
| if (context.payload.pull_request) { | |
| const { body } = context.payload.pull_request; | |
| const regex = /^\#loadgen-branch:\s+(\S+)/m; | |
| const result = regex.exec(body); | |
| if (result) { | |
| branch = result[1]; | |
| } | |
| } | |
| console.log('loadgen branch: ' + branch); | |
| return branch; | |
| - name: Check out loadgen | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Agoric/testnet-load-generator | |
| path: ./testnet-load-generator | |
| ref: ${{steps.get-loadgen-branch.outputs.result}} | |
| - name: Build cosmic-swingset dependencies | |
| working-directory: ./agoric-sdk | |
| run: | | |
| set -e | |
| cd packages/cosmic-swingset | |
| make install | |
| - name: Make networks directory | |
| run: | | |
| set -e | |
| mkdir networks | |
| - name: Run integration test | |
| working-directory: ./networks | |
| run: | | |
| set -xe | |
| DOCKER_VOLUMES="$PWD/../agoric-sdk:/usr/src/agoric-sdk" \ | |
| LOADGEN=1 \ | |
| ../agoric-sdk/packages/deployment/scripts/integration-test.sh | |
| timeout-minutes: 90 | |
| env: | |
| NETWORK_NAME: chaintest | |
| - name: capture results | |
| if: always() | |
| working-directory: ./networks | |
| run: | | |
| NOW=$(date -u +%Y%m%dT%H%M%S) | |
| echo "NOW=$NOW" >> "$GITHUB_ENV" | |
| # Stop the chain from running. | |
| ../agoric-sdk/packages/deployment/scripts/setup.sh play stop || true | |
| # Get the results. | |
| ../agoric-sdk/packages/deployment/scripts/capture-integration-results.sh "${{ job.status == 'failure' }}" | |
| # Tear down the nodes. | |
| echo yes | ../agoric-sdk/packages/deployment/scripts/setup.sh destroy || true | |
| env: | |
| NETWORK_NAME: chaintest | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: deployment-test-results-${{ env.NOW }} | |
| path: ./networks/chaintest/results | |
| - name: notify on failure | |
| if: failure() && github.event_name != 'pull_request' | |
| uses: ./agoric-sdk/.github/actions/notify-status | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| from: ${{ secrets.NOTIFY_EMAIL_FROM }} | |
| to: ${{ secrets.NOTIFY_EMAIL_TO }} | |
| password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }} | |
| test-docker-build: | |
| needs: pre_check | |
| if: needs.pre_check.outputs.should_run == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - name: free up disk space | |
| run: | | |
| # Workaround to provide additional free space for testing. | |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| # If this turns out not to be enough, maybe look instead at | |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-1540506686 | |
| df -h | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| echo "=== After cleanup:" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - name: docker build (sdk) | |
| # Produces ghcr.io/agoric/agoric-sdk:unreleased used in the following upgrade test. | |
| # run: cd packages/deployment && ./scripts/test-docker-build.sh | $TEST_COLLECT | |
| # XXX skip TAP test output and collection for now; it hides the output from the logs | |
| run: make docker-build-sdk | |
| working-directory: packages/deployment | |
| - uses: ./.github/actions/restore-node | |
| with: | |
| node-version: '18.x' | |
| # Rebuilding the SDK image with resolved endo packages is not currently supported | |
| # and since we can't build core eval submissions from the SDK using a newer endo, | |
| # simply ignore any endo branch integration (this means we don't have full coverage) | |
| ignore-endo-branch: 'true' | |
| id: restore-node | |
| - name: setup a3p-integration | |
| run: yarn install | |
| working-directory: a3p-integration | |
| - name: verify SDK image didn't change | |
| # In the future when we can rebuild the SDK image with resolved endo packages, it would | |
| # be expected that the SDK image previously built has changed | |
| if: steps.restore-node.outputs.endo-branch == 'NOPE' | |
| run: | | |
| original=$(docker inspect --format "{{.ID}}" ghcr.io/agoric/agoric-sdk:unreleased) | |
| yarn build:sdk | |
| new=$(docker inspect --format "{{.ID}}" ghcr.io/agoric/agoric-sdk:unreleased) | |
| if [ "$original" != "$new" ]; then | |
| echo "New SDK docker image ($new) changed after restore-node (original $original)" 1>&2 | |
| exit 1 | |
| fi | |
| working-directory: a3p-integration | |
| - name: build proposals tests | |
| run: yarn build | |
| working-directory: a3p-integration | |
| - name: run proposals tests | |
| run: yarn test | |
| working-directory: a3p-integration | |
| - name: collect all core eval scripts | |
| # Core eval scripts will be copied under /tmp/core_eval_scripts directory | |
| # colons in the parent directory name will be replaced with dashes | |
| run: | | |
| find . -type d -path "./a3p-integration/proposals/*/submission" | while read -r dir; do | |
| # Get the parent directory name | |
| parent_dir=$(basename "$(dirname "$dir")") | |
| # Replace colons with dashes in the parent directory name | |
| sanitized_parent_dir=${parent_dir//:/-} | |
| # Create the destination directory under /tmp/core_eval_scripts if it doesn't exist | |
| destination_dir="/tmp/core_eval_scripts/$sanitized_parent_dir" | |
| mkdir -p "$destination_dir" | |
| # Copy everything from the $dir to the destination directory | |
| cp -r "$dir"/* "$destination_dir/" | |
| echo "Copied contents of $dir to $destination_dir" | |
| done | |
| - name: archive core eval scripts | |
| # The core eval scripts can be found at the bottom of `Summary` page of | |
| # `Integration tests` workflow once the workflow is completed. | |
| # Ref: https://github.com/actions/upload-artifact?tab=readme-ov-file#where-does-the-upload-go | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: core-eval-scripts | |
| path: /tmp/core_eval_scripts | |
| - name: notify on failure | |
| if: failure() && github.event_name != 'pull_request' | |
| uses: ./.github/actions/notify-status | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| from: ${{ secrets.NOTIFY_EMAIL_FROM }} | |
| to: ${{ secrets.NOTIFY_EMAIL_TO }} | |
| password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }} | |
| - uses: ./.github/actions/post-test | |
| if: (success() || failure()) | |
| continue-on-error: true | |
| timeout-minutes: 4 | |
| with: | |
| datadog-token: ${{ secrets.DATADOG_API_KEY }} | |
| test-multichain-e2e: | |
| needs: pre_check | |
| if: needs.pre_check.outputs.should_run == 'true' | |
| uses: ./.github/workflows/multichain-e2e.yml | |
| finalize-integration-result: | |
| needs: | |
| - pre_check | |
| - getting-started | |
| - deployment-test | |
| - test-docker-build | |
| - test-multichain-e2e | |
| if: >- | |
| always() && | |
| needs.pre_check.result == 'success' && | |
| needs.getting-started.result != 'cancelled' && | |
| needs.deployment-test.result != 'cancelled' && | |
| needs.test-docker-build.result != 'cancelled' && | |
| ( | |
| needs.pre_check.outputs.should_run == 'true' || | |
| needs.pre_check.outputs.previous_success == 'true' | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Final integration-test-result | |
| if: always() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const previousSuccess = ${{ needs.pre_check.outputs.previous_success }}; | |
| const gettingStartedTestSuccess = "${{ needs.getting-started.result }}" === "success"; | |
| const deploymentTestSuccess = "${{ needs.deployment-test.result }}" === "success"; | |
| const testDockerBuildSuccess = "${{ needs.test-docker-build.result }}" === "success"; | |
| const conclusion = previousSuccess || (gettingStartedTestSuccess && deploymentTestSuccess && testDockerBuildSuccess) ? | |
| 'success' : 'failure'; | |
| console.log(`Finishing with ${conclusion}`) | |
| if (conclusion === 'failure') { | |
| core.setFailed('Integration tests failed') | |
| } |