Merge pull request #8159 from Agoric/mhofman/rebase-for-upgrade11

merge a subset of trunk changes, which are relevant for upgrade11, into the `release-mainnet1B` branch, for use in the upcoming `agoric-upgrade-11-rc0` release

.dockerignore

@@ -3,14 +3,15 @@
 docker
 packages/cosmic-swingset/t[0-9]
 packages/cosmic-swingset/t[0-9].*
-packages/cosmic-swingset/lib/lib*.h
-packages/cosmic-swingset/lib/lib*.so
+golang/cosmos/build
+packages/deployment
 packages/swingset-runner
 packages/stat-logger
 **/deployment.json
 **/vars.tf
 **/*.log
 **/build
+**/bundles
 **/__pycache__
 **/*.egg-info
 **/swingset-kernel-state

.github/actions/restore-golang/action.yml

@@ -5,6 +5,10 @@ inputs:
   go-version:
     description: 'The version of Go to use'
     required: true
+  path:
+    description: 'The relative path to the agoric-sdk directory'
+    required: false
+    default: '.'
 
 runs:
   using: composite
@@ -14,10 +18,12 @@ runs:
       shell: bash
     - uses: actions/checkout@v3
       with:
+        path: ${{ inputs.path }}
         clean: 'false'
         submodules: 'true'
     - uses: actions/setup-go@v4
       with:
+        cache-dependency-path: ${{ inputs.path }}/golang/cosmos/go.sum
         go-version: ${{ inputs.go-version }}
     - uses: kenchan0130/actions-system-info@master
       id: system-info
@@ -26,10 +32,11 @@ runs:
       uses: actions/cache@v3
       with:
         path: ${{ env.GOPATH }}/pkg/mod
-        key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-built-${{ hashFiles('go.sum') }}
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-built-${{ hashFiles('golang/**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-${{ runner.arch }}-${{ steps.system-info.outputs.release }}-go-${{ inputs.go-version }}-built-
     - name: go mod download
+      working-directory: ${{ inputs.path }}/golang/cosmos
       run: go mod download
       shell: bash
       if: steps.cache.outputs.cache-hit != 'true'

.github/workflows/deployment-test.yml

@@ -25,13 +25,17 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: 'true'
+          path: ./agoric-sdk
       - run: sudo packages/deployment/scripts/install-deps.sh
-      - uses: ./.github/actions/restore-golang
+        working-directory: ./agoric-sdk
+      - uses: ./agoric-sdk/.github/actions/restore-golang
         with:
           go-version: '1.20'
-      - uses: ./.github/actions/restore-node
+          path: ./agoric-sdk
+      - uses: ./agoric-sdk/.github/actions/restore-node
         with:
           node-version: 18.x
+          path: ./agoric-sdk
           # Forces xsnap to initialize all memory to random data, which increases
           # the chances the content of snapshots may deviate between validators
           xsnap-random-init: '1'
@@ -61,54 +65,55 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: Agoric/testnet-load-generator
-          path: testnet-load-generator
+          path: ./testnet-load-generator
           ref: ${{steps.get-loadgen-branch.outputs.result}}
 
-      - name: Put repos under /usr/src where scripts expect them
-        run: |
-          set -e
-          sudo mv "$GITHUB_WORKSPACE/testnet-load-generator" /usr/src/testnet-load-generator
-          sudo cp -a "$GITHUB_WORKSPACE" /usr/src/agoric-sdk
-          ln -s /usr/src/agoric-sdk/packages/deployment/bin/ag-setup-cosmos /usr/local/bin/ag-setup-cosmos
-        working-directory: /
-
       - name: Build cosmic-swingset dependencies
+        working-directory: ./agoric-sdk
         run: |
           set -e
           cd packages/cosmic-swingset
           make install
-        working-directory: /usr/src/agoric-sdk
-      - run: /usr/src/agoric-sdk/packages/deployment/scripts/integration-test.sh
+      - name: Make networks directory
+        run: |
+          set -e
+          mkdir networks
+      - name: Run integration test
+        working-directory: ./networks
+        run: |
+          set -xe
+          DOCKER_VOLUMES="$PWD/../agoric-sdk:/usr/src/agoric-sdk" \
+          LOADGEN=1 \
+          ../agoric-sdk/packages/deployment/scripts/integration-test.sh
         timeout-minutes: 90
-        working-directory: /usr/src/agoric-sdk
         env:
           NETWORK_NAME: chaintest
       - name: capture results
         if: always()
+        working-directory: ./networks
         run: |
           NOW=$(date -u +%Y%m%dT%H%M%S)
           echo "NOW=$NOW" >> "$GITHUB_ENV"
 
           # Stop the chain from running.
-          packages/deployment/scripts/setup.sh play stop || true
+          ../agoric-sdk/packages/deployment/scripts/setup.sh play stop || true
 
           # Get the results.
-          packages/deployment/scripts/capture-integration-results.sh "${{ job.status == 'failure' }}"
+          ../agoric-sdk/packages/deployment/scripts/capture-integration-results.sh "${{ job.status == 'failure' }}"
 
           # Tear down the nodes.
-          echo yes | packages/deployment/scripts/setup.sh destroy || true
-        working-directory: /usr/src/agoric-sdk
+          echo yes | ../agoric-sdk/packages/deployment/scripts/setup.sh destroy || true
         env:
           NETWORK_NAME: chaintest
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: deployment-test-results-${{ env.NOW }}
-          path: /usr/src/agoric-sdk/chaintest/results
+          path: ./networks/chaintest/results
 
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
-        uses: ./.github/actions/notify-status
+        uses: ./agoric-sdk/.github/actions/notify-status
         with:
           webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
           from: ${{ secrets.NOTIFY_EMAIL_FROM }}

.github/workflows/golangci-lint.yml

@@ -24,15 +24,22 @@ jobs:
           go-version: '>=1.20'
           cache: false
           check-latest: true
+        # https://github.com/golangci/golangci-lint/issues/3862#issuecomment-1572973588
+      - run: echo "GOROOT=$(go env GOROOT)" >> $GITHUB_ENV
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          # golangci-lint version and command line arguments
-          # v1.52.3 has a bug that causes it to fail depguard with:
-          # level=error msg="[linter] depguard: create analyzer:
-          # couldn't expand $gostd: could not read GOROOT directory:
-          # open src: no such file or directory\nmust have an Allow and/or Deny package list"
-          version: 'v1.52.2' # FIXME: change back to latest when it is fixed
+          version: latest
           args: --timeout=3m
           # for pull requests, show only new issues
-          only-new-issues: true
+          # Too bad it's incompatible with working-directory.
+          # only-new-issues: true
+          only-new-issues: false
+          working-directory: ./golang/cosmos
+      - name: forbid %w error-wrapping format specifier
+        run: |
+          set -e
+          if find ./golang/cosmos -name '*.go' ! -name '*_test.go' -print0 | xargs -0 grep '%w'; then
+            echo "Found %w in ./golang/cosmos; please use %s instead."
+            exit 1
+          fi

.github/workflows/test-all-packages.yml

@@ -569,11 +569,22 @@ jobs:
       matrix:
         bootstrap-version: ['test', 'main']
     steps:
+      - name: free up additional worker space
+        run: |
+          # Workaround to provide additional free space for testing.
+          #   https://github.com/actions/virtual-environments/issues/2840
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          df -h
+
       - uses: actions/checkout@v3
       - name: docker build (sdk)
         run: cd packages/deployment && ./scripts/test-docker-build.sh | $TEST_COLLECT
       - name: docker build upgrade test
-        run: cd packages/deployment/upgrade-test && docker build --build-arg BOOTSTRAP_MODE=${{ matrix.bootstrap-version }} -t docker-upgrade-test:latest -f Dockerfile upgrade-test-scripts
+        run: cd packages/deployment/upgrade-test && docker build --build-arg BOOTSTRAP_MODE=${{ matrix.bootstrap-version }} --build-arg DEST_IMAGE=ghcr.io/agoric/agoric-sdk:latest -t docker-upgrade-test:latest -f Dockerfile upgrade-test-scripts
       - name: docker run upgrade final stage
         run: docker run --env "DEST=0" docker-upgrade-test:latest
       - uses: ./.github/actions/post-test

.github/workflows/test-golang.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           go-version: '1.20'
       - name: go test
-        run: cd golang && go test -coverprofile=coverage.txt -covermode=atomic ./...
+        run: cd golang/cosmos && go test -coverprofile=coverage.txt -covermode=atomic ./...
       - uses: ./.github/actions/post-test
         if: (success() || failure())
         continue-on-error: true

bin/agd

@@ -148,7 +148,7 @@ fi
     print=()
   fi
   print+=( -print )
-  src=$(find go.* "$GOLANG_DIR" \( -name '*.go' -o -name 'go.*' \) "${print[@]}" | head -1 || true)
+  src=$(find "$GOLANG_DIR" \( -name '*.go' -o -name 'go.*' \) "${print[@]}" | head -1 || true)
   test -z "$src" || {
     echo "At least $src is newer than $stamp"
 
@@ -184,21 +184,22 @@ fi
   }
 )
 
-if $BUILD_ONLY; then
-  echo "Build complete." 1>&2
-  exit 0
-fi
-
 # the xsnap binary lives in a platform-specific directory
 unameOut="$(uname -s)"
 case "${unameOut}" in
     Linux*)     platform=lin;;
     Darwin*)    platform=mac;;
     *)          platform=win;;
 esac
+
 # check the xsnap version against our baked-in notion of what version we should be using
 xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n)
-[[ "${xsnap_version}" == "agoric-upgrade-10" ]] || fatal "xsnap out of date"
+[[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}"
+
+if $BUILD_ONLY; then
+  echo "Build complete." 1>&2
+  exit 0
+fi
 
 # Run the built Cosmos daemon.
 # shellcheck disable=SC2031