Upgrade-16 cherry-picks round 2 (#9625)

Rebase todo

```
# Branch fix-vow-include-vat-js-in-package-files-9607-
label base-fix-vow-include-vat-js-in-package-files-9607-
pick b6ffa6f fix(vow): include vat.js in package files
label fix-vow-include-vat-js-in-package-files-9607-
reset base-fix-vow-include-vat-js-in-package-files-9607-
merge -C a3826e9 fix-vow-include-vat-js-in-package-files-9607- # fix(vow): include vat.js in package files (#9607)

# Pull Request #9601
pick 6bc363b fix(cosmos): only allow snapshot export at latest height (#9601)

# Branch Force-xsnap-rebuild-9618-
label base-Force-xsnap-rebuild-9618-
pick 467435a fix(xsnap): force rebuild if build config changes
pick a22772e fix(agd): check xsnap was rebuilt
pick 2b53896 feat(xsnap): force rebuild if binary version mismatch
label Force-xsnap-rebuild-9618-
reset base-Force-xsnap-rebuild-9618-
merge -C 78b6fec Force-xsnap-rebuild-9618- # Force xsnap rebuild (#9618)

# Branch agd-enforce-Node-js-version-9623-
label base-agd-enforce-Node-js-version-9623-
#pick 4b35caf revert fix: typescript-estree does not support Node.js LTS (#9619)
pick 5f01bef fix(agd): force own node.js version check
label agd-enforce-Node-js-version-9623-
reset base-agd-enforce-Node-js-version-9623-
merge -C 4f70e66 agd-enforce-Node-js-version-9623- # agd enforce Node.js version (#9623)

# Branch gibson-9623-followup
label base-gibson-9623-followup
pick 6102eb9 fix: Disallow Node.js major version >20
label gibson-9623-followup
reset base-gibson-9623-followup
merge -C caaec05 gibson-9623-followup # (upstream/master) fix: Disallow Node.js major version >20 (#9630)
```

.dockerignore

@@ -22,6 +22,7 @@ packages/stat-logger
 **/_agstate
 .vagrant
 endo-sha.txt
+packages/xsnap/build.config.env
 # When changing/adding entries here, make sure to search the whole project for
 # `@@AGORIC_DOCKER_SUBMODULES@@`
 packages/xsnap/moddable

golang/cosmos/daemon/cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -452,7 +453,7 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) {
 	replacedRunE := func(cmd *cobra.Command, args []string) error {
 		ctx := server.GetServerContextFromCmd(cmd)
 
-		height, err := cmd.Flags().GetInt64("height")
+		heightFlag, err := cmd.Flags().GetInt64("height")
 		if err != nil {
 			return err
 		}
@@ -467,13 +468,15 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) {
 		app := ac.newSnapshotsApp(ctx.Logger, db, nil, ctx.Viper)
 		gaiaApp := app.(*gaia.GaiaApp)
 
-		if height == 0 {
-			height = app.CommitMultiStore().LastCommitID().Version
+		latestHeight := app.CommitMultiStore().LastCommitID().Version
+
+		if heightFlag != 0 && latestHeight != heightFlag {
+			return fmt.Errorf("cannot export at height %d, only latest height %d is supported", heightFlag, latestHeight)
 		}
 
-		cmd.Printf("Exporting snapshot for height %d\n", height)
+		cmd.Printf("Exporting snapshot for height %d\n", latestHeight)
 
-		err = gaiaApp.SwingSetSnapshotter.InitiateSnapshot(height)
+		err = gaiaApp.SwingSetSnapshotter.InitiateSnapshot(latestHeight)
 		if err != nil {
 			return err
 		}
@@ -488,7 +491,7 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) {
 			return err
 		}
 
-		snapshotHeight := uint64(height)
+		snapshotHeight := uint64(latestHeight)
 
 		for _, snapshot := range snapshotList {
 			if snapshot.Height == snapshotHeight {

packages/vow/package.json

@@ -45,6 +45,7 @@
   "author": "Agoric",
   "license": "Apache-2.0",
   "files": [
+    "*.js",
     "src"
   ],
   "publishConfig": {

packages/xsnap/.gitignore

@@ -1,4 +1,5 @@
 build
+build.config.env
 dist
 test/fixture-snap-pool/
 test/fixture-snap-shot.xss

packages/xsnap/build.env

@@ -1,4 +1,4 @@
 MODDABLE_URL=https://github.com/agoric-labs/moddable.git
 MODDABLE_COMMIT_HASH=f6c5951fc055e4ca592b9166b9ae3cbb9cca6bf0
 XSNAP_NATIVE_URL=https://github.com/agoric-labs/xsnap-pub
-XSNAP_NATIVE_COMMIT_HASH=2d8ccb76b8508e490d9e03972bb4c64f402d5135
+XSNAP_NATIVE_COMMIT_HASH=eef9b67da5517ed18ff9e0073b842db20924eae3

packages/xsnap/package.json

@@ -16,6 +16,7 @@
     "build:env": "node src/build.js --show-env > build.env",
     "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env",
     "build": "yarn build:bin && yarn build:env",
+    "check-version": "xsnap_version=$(./scripts/get_xsnap_version.sh); if test \"${npm_package_version}\" != \"${xsnap_version}\"; then echo \"xsnap version mismatch; expected '${npm_package_version}', got '${xsnap_version}'\"; exit 1; fi",
     "postinstall": "npm run build:from-env",
     "clean": "rm -rf xsnap-native/xsnap/build",
     "lint": "run-s --continue-on-error lint:*",
@@ -56,6 +57,7 @@
     "moddable/xs/makefiles",
     "moddable/xs/platforms/*.h",
     "moddable/xs/sources",
+    "scripts",
     "src",
     "xsnap-native/xsnap/makefiles",
     "xsnap-native/xsnap/sources"

packages/xsnap/scripts/get_xsnap_version.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -ueo pipefail
+
+# the xsnap binary lives in a platform-specific directory
+unameOut="$(uname -s)"
+case "${unameOut}" in
+  Linux*) platform=lin ;;
+  Darwin*) platform=mac ;;
+  *) platform=win ;;
+esac
+
+# extract the xsnap package version from the long version printed by xsnap-worker
+"./xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -v | sed -e 's/^xsnap \([^ ]*\) (XS [^)]*)$/\1/g'

packages/xsnap/src/build.js

@@ -220,16 +220,34 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => {
  *     existsSync: typeof import('fs').existsSync,
  *     rmdirSync: typeof import('fs').rmdirSync,
  *     readFile: typeof import('fs').promises.readFile,
+ *     writeFile: typeof import('fs').promises.writeFile,
  *   },
  *   os: {
  *     type: typeof import('os').type,
  *   }
  * }} io
+ * @param {object} [options]
+ * @param {boolean} [options.forceBuild]
  */
-const makeXsnap = async ({ spawn, fs, os }) => {
+const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => {
   const pjson = await fs.readFile(asset('../package.json'), 'utf-8');
   const pkg = JSON.parse(pjson);
 
+  const configEnvs = [
+    `XSNAP_VERSION=${pkg.version}`,
+    `CC=cc "-D__has_builtin(x)=1"`,
+  ];
+
+  const configEnvFile = asset('../build.config.env');
+  const existingConfigEnvs = fs.existsSync(configEnvFile)
+    ? await fs.readFile(configEnvFile, 'utf-8')
+    : '';
+
+  const expectedConfigEnvs = configEnvs.concat('').join('\n');
+  if (forceBuild || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) {
+    await fs.writeFile(configEnvFile, expectedConfigEnvs);
+  }
+
   const platform = ModdableSDK.platforms[os.type()];
   if (!platform) {
     throw Error(`Unsupported OS found: ${os.type()}`);
@@ -241,8 +259,10 @@ const makeXsnap = async ({ spawn, fs, os }) => {
       [
         `MODDABLE=${ModdableSDK.MODDABLE}`,
         `GOAL=${goal}`,
-        `XSNAP_VERSION=${pkg.version}`,
-        `CC=cc "-D__has_builtin(x)=1"`,
+        // Any other configuration variables that affect the build output
+        // should be placed in `configEnvs` to force a rebuild if they change
+        ...configEnvs,
+        `EXTRA_DEPS=${configEnvFile}`,
         '-f',
         'xsnap-worker.mk',
       ],
@@ -263,6 +283,7 @@ const makeXsnap = async ({ spawn, fs, os }) => {
  *     existsSync: typeof import('fs').existsSync,
  *     rmdirSync: typeof import('fs').rmdirSync,
  *     readFile: typeof import('fs').promises.readFile,
+ *     writeFile: typeof import('fs').promises.writeFile,
  *   },
  *   os: {
  *     type: typeof import('os').type,
@@ -328,7 +349,18 @@ async function main(args, { env, stdout, spawn, fs, os }) {
 
   if (!showEnv) {
     if (hasSource) {
-      await makeXsnap({ spawn, fs, os });
+      // Force a rebuild if for some reason the binary is out of date
+      // Since the make checks may not always detect that situation
+      let forceBuild = !hasBin;
+      if (hasBin) {
+        const npm = makeCLI('npm', { spawn });
+        await npm
+          .run(['run', '-s', 'check-version'], { cwd: asset('..') })
+          .catch(() => {
+            forceBuild = true;
+          });
+      }
+      await makeXsnap({ spawn, fs, os }, { forceBuild });
     } else if (!hasBin) {
       throw new Error(
         'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?',
@@ -344,6 +376,7 @@ const run = () =>
     spawn: childProcessTop.spawn,
     fs: {
       readFile: fsTop.promises.readFile,
+      writeFile: fsTop.promises.writeFile,
       existsSync: fsTop.existsSync,
       rmdirSync: fsTop.rmdirSync,
     },

repoconfig.sh

@@ -1,18 +1,27 @@
 #! /bin/sh
 # shellcheck disable=SC2034
-NODEJS_VERSION=v16
+NODEJS_VERSION=v20
 GOLANG_VERSION=1.20.3
 GOLANG_DIR=golang/cosmos
 GOLANG_DAEMON=$GOLANG_DIR/build/agd
-XSNAP_VERSION=agoric-upgrade-10
 
 # Args are major, minor and patch version numbers
 golang_version_check() {
   {
     [ "$1" -eq 1 ] && [ "$2" -eq 20 ] && [ "$3" -ge 2 ] && return 0
     [ "$1" -eq 1 ] && [ "$2" -ge 21 ] && return 0
     [ "$1" -ge 2 ] && return 0
-  } 2>/dev/null
+  } 2> /dev/null
   echo 1>&2 "need go version 1.20.2+, 1.21+, or 2+"
   return 1
 }
+
+# Args are major, minor and patch version numbers
+nodejs_version_check() {
+  {
+    [ "$1" -eq 18 ] && [ "$2" -ge 12 ] && return 0
+    [ "$1" -eq 20 ] && [ "$2" -ge 9 ] && return 0
+  } 2> /dev/null
+  echo 1>&2 "need Node.js LTS version ^18.12 or ^20.9, found $1.$2.$3"
+  return 1
+}

scripts/agd-builder.sh

@@ -106,6 +106,12 @@ if $need_nodejs; then
       } 1>&2
       ;;
   esac
+
+  if nodeversion=$(node --version 2> /dev/null); then
+    noderegexp='v([0-9]+)\.([0-9]+)\.([0-9]+)'
+    [[ "$nodeversion" =~ $noderegexp ]] || fatal "illegible node version '$nodeversion'"
+    nodejs_version_check "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" || exit 1
+  fi
 fi
 
 $do_not_build || (
@@ -194,7 +200,9 @@ $do_not_build || (
     test -z "$src" || {
       echo "At least $src is newer than node_modules"
       rm -f "$STAMPS/yarn-built"
-      lazy_yarn install
+      # Ignore engines since we already checked officially supported versions above
+      # UNTIL https://github.com/Agoric/agoric-sdk/issues/9622
+      lazy_yarn install --ignore-engines
     }
 
     stamp=$STAMPS/yarn-built
@@ -219,21 +227,12 @@ $do_not_build || (
       echo "At least $src is newer than gyp bindings"
       (cd "$GOLANG_DIR" && lazy_yarn build:gyp)
     }
+
+    # check the built xsnap version against the package version it should be using
+    (cd "${thisdir}/../packages/xsnap" && npm run -s check-version) || exit 1
   fi
 )
 
-# the xsnap binary lives in a platform-specific directory
-unameOut="$(uname -s)"
-case "${unameOut}" in
-  Linux*) platform=lin ;;
-  Darwin*) platform=mac ;;
-  *) platform=win ;;
-esac
-
-# check the xsnap version against our baked-in notion of what version we should be using
-xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n)
-[[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}"
-
 if $only_build; then
   echo "Build complete." 1>&2
   exit 0