Merge pull request #8886 from Agoric/mhofman/clarify-a3p

Clarify a3p-integration

.dockerignore

@@ -1,6 +1,8 @@
 **/node_modules
 **/.git
 docker
+a3p-integration
+coverage
 packages/cosmic-swingset/t[0-9]
 packages/cosmic-swingset/t[0-9].*
 golang/cosmos/build
@@ -10,13 +12,17 @@ packages/stat-logger
 **/deployment.json
 **/vars.tf
 **/*.log
+**/dist
+!packages/cosmic-proto/dist
+**/compiled
 **/build
 **/bundles
 **/__pycache__
 **/*.egg-info
 **/swingset-kernel-state
 **/_agstate
 .vagrant
+endo-sha.txt
 # When changing/adding entries here, make sure to search the whole project for
 # `@@AGORIC_DOCKER_SUBMODULES@@`
 packages/xsnap/moddable

.github/actions/restore-node/action.yml

@@ -13,6 +13,10 @@ inputs:
     description: 'Keep Endo repository after installing from it'
     required: false
     default: 'false'
+  ignore-endo-branch:
+    description: 'Ignore external Endo branch configuration'
+    required: false
+    default: 'false'
   xsnap-random-init:
     description: Build xsnap binary with random memory init
     required: false
@@ -46,7 +50,9 @@ runs:
         result-encoding: string
         script: |-
           let branch = 'NOPE';
-          if (context.eventName === 'schedule') {
+          if (${{ inputs.ignore-endo-branch }}) {
+            // Skip endo branch
+          } else  if (context.eventName === 'schedule') {
             branch = 'master';
           } else if (context.payload.pull_request) {
             const { body } = context.payload.pull_request;

.github/workflows/integration.yml

@@ -237,11 +237,29 @@ jobs:
       - uses: ./.github/actions/restore-node
         with:
           node-version: '18.x'
+          # Rebuilding the SDK image with resolved endo packages is not currently supported
+          # and since we can't build core eval submissions form the SDK using a newer endo,
+          # simply ignore any endo branch integration (this means we don't have full coverage)
+          ignore-endo-branch: 'true'
+        id: restore-node
       - name: setup a3p-integration
         run: |
           corepack enable
           yarn install
         working-directory: a3p-integration
+      - name: verify SDK image didn't change
+        # In the future when we can rebuild the SDK image with resolved endo packages, it would
+        # be expected that the SDK image previously built has changed
+        if: steps.restore-node.outputs.endo-branch == 'NOPE'
+        run: |
+          original=$(docker inspect --format "{{.ID}}" ghcr.io/agoric/agoric-sdk:unreleased)
+          yarn build:sdk
+          new=$(docker inspect --format "{{.ID}}" ghcr.io/agoric/agoric-sdk:unreleased)
+          if [ "$original" != "$new" ]; then
+            echo "New SDK docker image ($new) changed after restore-node (original $original)" 1>&2
+            exit 1
+          fi
+        working-directory: a3p-integration
       - name: build proposals tests
         run: yarn build
         working-directory: a3p-integration

a3p-integration/README.md

@@ -1,38 +1,123 @@
-# Integration with agoric-3 synthetic test chain
+# Overview
 
-The test runner is `@agoric/synthetic-chain`. This package depends on that so that you can run,
+This folder contains an end-to-end integration test executed against a synthetic agoric-3 chain. The test performs a chain software upgrade to the software contained in the enclosing `agoric-sdk` repository, then executes a series of functional tests verifying the upgrade accomplished its goal.
+
+# How to run
+
+The synthetic chain testing infrastructure relies on Docker, Docker Buildx extended build capabilities, and the experimental Buildx Bake extension. Make sure you have a recent Docker engine installed for your system.
+
+## Install test dependencies
+
+This directory hierarchy, while it contains packages, is not part of the `agoric-sdk` root Yarn project. This is to isolate it from tooling that expects a public package published to NPM. For example, it doesn't run in the CI jobs for the `/packages` packages.
+
+The end-to-end integration test relies on the [@agoric/synthetic-chain](https://www.npmjs.com/package/@agoric/synthetic-chain) test tool (published to NPM from the [`agoric-3-proposals` repository](https://github.com/Agoric/agoric-3-proposals/tree/main/packages/synthetic-chain)), and uses Yarn 4 (configured through corepack).
+
+First time setup (or after updates to `@agoric/synthetic-chain`):
+```sh
+corepack enable
+yarn install
 ```
-yarn synthetic-chain build
-yarn synthetic-chain test
-yarn synthetic-chain test --debug
+
+## Run the integration test
+
+The `@agoric/synthetic-chain` test runner is invoked through NPM scripts.
+
+To build the agoric-3 synthetic chain images with the unreleased chain software upgrade:
+```sh
+yarn build
 ```
 
-# Package management
+To run the unit tests:
+```sh
+yarn test
+# or for an interactive session into a given proposal package
+yarn test --debug -m "proposal-name"
+```
+
+# Proposal structure
+
+## Package layering
+
+Proposals are packages under the `/a3p-integration/proposals` folder, and are separate from each other, from the `a3p-integration` test environment, and from the enclosing `agoric-sdk` repository. They run inside the docker image, and cannot access SDK code. The are lower case, and executed in lexical order.
+
+In the release branches, the end-to-end `a3p-integration` test usually only has a single proposal package named `a:upgrade-NN`, which performs a chain software upgrade proposal to the corresponding upgrade plan. In the `master` branch, there may also be core-eval proposal packages, either before or after the chain software upgrade proposal.
+
+The details of a proposal package are configured through the `agoricProposal` field of its `package.json`.
+
+More details about synthetic-chain proposals can be found in the [`agoric-3-proposals` README](https://github.com/Agoric/agoric-3-proposals/blob/main/README.md)
+
+### Chain software upgrade proposal
+
+For a chain software upgrade proposal, the `type` is `"Software Upgrade Proposal"`, and the relevant fields are `sdkImageTag`, `planName` and `upgradeInfo`.
+
+- `sdkImageTag` is the docker image tag to use that contains the upgraded chain software. It has a value of `unreleased`, which is the tag for the image that is built from the enclosing `agoric-sdk` repository.
+- `planName` is the "upgrade name" included in the proposal which must match the value in the upgraded chain software. In the `master` branch its value is `UNRELEASED_UPGRADE`. In the release branches, it's `agoric-upgrade-NN`.
+- `upgradeInfo` contains other details passed to the governance proposal. In particular, the info can have a `coreProposal` field which instruct the chain software to run other core proposals besides the one already configured in the chain software's upgrade handler (see `CoreProposalSteps` in `/golang/cosmos/app/app.go`). This field is likely not relevant for release branches.
+
+### Core-eval proposal
+
+The `type` of a core-eval proposal is `"/agoric.swingset.CoreEvalProposal"`, and content is submitted from a `submission` subfolder.
 
-This directory hierarchy, while it contains packages, is not part of the agoric-sdk workspace. This is to isolate it from tooling that expects a public package published to NPM. For example, it doesn't run in the CI jobs for the `/packages` packages.
+If the proposal is planned to be executed after the chain software upgrade, and the source of the proposal is in `agoric-sdk`, it's recommended to not check-in the `submission` content in source control and instead generate it automatically when testing. Since proposals cannot access SDK code, a script can be used to generate the `submission` content. Until there is [native support for build scripts in the `synthetic-chain` tool](https://github.com/Agoric/agoric-3-proposals/issues/87), `a3p-integration`'s `build:submission` step invokes `/script/generate-a3p-submission.sh` in `agoric-sdk` before starting the upgrade test.
 
-For each proposal, their package.json is also separate and can't access the SDK code. There is an issue to automatically build proposals from scripts declared in the proposal package.json: https://github.com/Agoric/agoric-3-proposals/issues/87 . Until that is resolved, use `agoric run` on the proposal and copy the outputs to a `submission` directory within the proposals package, to be checked in.
+For core eval proposals executing before the chain software upgrade, the `submission` should be checked in, since bundles built from newer software may not be compatible with older chains.
+
+## Hooks
+
+<!-- TODO: Move this section over to synthetic-chain docs -->
+
+Each proposal must have a `test.sh` file, which is executed during a `synthetic-chain test` invocation. In general this simply executes `yarn ava` to run any `*.test.js`. This step can perform any chain action, including submitting bespoke core-evals.
+
+A proposal can also have a `use.sh` file to perform any persisted action after the proposal has passed. The effects of these actions can be used in the test phase, or in subsequent proposal packages.
+
+A chain software upgrade proposal can also have a `prepare.sh` file which is executed before the proposal is submitted to the chain. The actions taken are similarly persisted.
+
+Instead of relying on an automatic `submission` folder, a core-eval proposal can define an `eval.sh` file to generate and submit the core eval (or perform any standalone actions).
+
+# Build details
+
+The `yarn build` script automates 3 steps:
+- Building the `unreleased` SDK image
+- Generating the `submission` folders in core proposal packages
+- Building the synthetic-chain images using the proposals
+
+## Generate a docker image with the `agoric-sdk` chain software
+
+The chain software upgrade proposal contained in this end-to-end integration test performs an upgrade of the agoric-3 synthetic chain to an `UNRELEASED_UPGRADE` plan name (or the corresponding upgrade plan name for release branches). It loads the docker image `ghcr.io/agoric/agoric-sdk:unreleased` for the software implementing that upgrade (both in the `master` branch or in release branches).
+
+The upgrade handler is implemented by the code in the enclosing `agoric-sdk` repository. After any change to the chain software or vat code upgraded through core proposals, the image must be regenerated. This is automatically done by the `build:sdk` script, but can also be performed manually using:
+
+```sh
+make -C ../packages/deployment docker-build-sdk
+```
+
+## Generating core-eval submissions
+
+Some core-eval proposals `submission` content are generated from the `agoric-sdk` code, and must be rebuilt every time there is a change. The `/scripts/generate-a3p-submission.sh` script contains commands to generate the core-eval content and move it to the expected proposal package's `submission` folder. It is executed as part of `a3p-integration`'s `build:submission` step.
+
+## Building synthetic-chain images
+
+Synthetic-chain tests run inside generated proposal Docker images. If any changes are made to the proposals or their tests, these images must be rebuilt. This is performed during the `build:synthetic-chain` step.
 
 # Troubleshooting
 
 ## no match for platform
 
 If you get an error like this,
 ```
-ERROR: failed to solve: ghcr.io/agoric/agoric-3-proposals:main: no match for platform in manifest sha256:83321abda66fa94915f1ae20d651b66870f2d1aac17b71449c04ecd46b6b1b96: not found
+ERROR: failed to solve: ghcr.io/agoric/agoric-3-proposals:latest: no match for platform in manifest sha256:14e22b6f75b568a5d32f7a74b701d978b7656ba4d33e2ec7ad2ff0611e7c2530: not found
 ```
-it's because our CI only builds x64 yet and you're on some other machine, probably a Mac.
+it's because you're on an architecture for which we don't generate an image. We currently generate `linux/amd64` and `linux/arm64` images.
 
-There is some effort to make CI build multiplatform: https://github.com/Agoric/agoric-3-proposals/pull/32
-
-Meanwhile you can build the `main` image locally:
+You can build the `latest` image locally:
 
 ```sh
 cd agoric-3-proposals
-./node_modules/.bin/synthetic-chain build
+yarn install
+yarn synthetic-chain build
 
 # build the default entrypoint and tag it so the `append` command finds it
-docker buildx build --tag ghcr.io/agoric/agoric-3-proposals:main .
+docker buildx build --tag ghcr.io/agoric/agoric-3-proposals:latest .
 ```
 
 ## missing "unreleased" image
@@ -42,15 +127,13 @@ If you get an error like,
 ERROR: failed to solve: ghcr.io/agoric/agoric-sdk:unreleased: ghcr.io/agoric/agoric-sdk:unreleased: not found
 ```
 
-That's probably because you don't have that image built locally. To build it,
-```
-cd packages/deployment
-make docker-build-sdk
-```
+That's because you didn't create an image from the local `agoric-sdk`. Run `yarn build:sdk`.
+
+## UPGRADE NEEDED
 
-### UPGRADE NEEDED
+If you get an error like,
 ```
 panic: UPGRADE "UNRELEASED_UPGRADE" NEEDED at height: 1101: {"coreProposals":["@agoric/builders/scripts/vats/init-network.js"]}
 ```
 
-Means your SDK image is different than the one expected by the upgrade proposal. The remedy also to rebuild it as in the case it's missing.
+Means your SDK image is different than the one expected by the upgrade proposal. To build the correct image, run `yarn build:sdk`.

a3p-integration/package.json

@@ -4,7 +4,10 @@
         "fromTag": "use-upgrade-13"
     },
     "scripts": {
-        "build": "../scripts/generate-a3p-submission.sh && yarn synthetic-chain build",
+        "build": "yarn run build:sdk && yarn run build:submission && yarn run build:synthetic-chain",
+        "build:sdk": "make -C ../packages/deployment docker-build-sdk",
+        "build:submission": "../scripts/generate-a3p-submission.sh",
+        "build:synthetic-chain": "yarn synthetic-chain build",
         "test": "yarn synthetic-chain test",
         "doctor": "yarn synthetic-chain doctor"
     },

a3p-integration/proposals/a:upgrade-14/prepare.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Exit when any command fails
+set -e
+
+# Place here any actions that should happen before the upgrade is proposed. The
+# actions are executed in the previous chain software, and the effects are
+# persisted so they can be used in the steps after the upgrade is complete,
+# such as in the "use" or "test" steps, or further proposal layers.

a3p-integration/proposals/a:upgrade-14/test.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
-source /usr/src/upgrade-test-scripts/env_setup.sh
+
+# Place here any test that should be executed using the executed proposal.
+# The effects of this step are not persisted in further proposal layers.
 
 yarn ava

a3p-integration/proposals/a:upgrade-14/use.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
-# UNTIl this is upstream https://github.com/Agoric/agoric-3-proposals/issues/40
-# Set to zero so tests don't have to pay gas (we're not testing that)
-sed --in-place=.bak s/'minimum-gas-prices = ""'/'minimum-gas-prices = "0ubld,0uist"'/ ~/.agoric/config/app.toml
+# Place here any actions that should happen after the upgrade has executed. The
+# actions are executed in the upgraded chain software and the effects are
+# persisted in the generated image for the upgrade, so they can be used in
+# later steps, such as the "test" step, or further proposal layers.