feat(agoric-cli): allow NPM-based agoric install to work with dapps

.github/workflows/integration.yml

@@ -10,6 +10,9 @@ jobs:
   # It should be kept in sync with https://agoric.com/documentation/getting-started/
   getting-started:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        cli: [ link-cli, local-npm ]
     steps:
     - uses: actions/checkout@v2
 
@@ -51,12 +54,46 @@ jobs:
     # cross-package symlinks
     - run: yarn install
     - run: yarn build
-    - run: yarn link-cli ~/bin/agoric
+
+    - name: Link Agoric CLI from SDK
+      if: ${{ matrix.cli == 'link-cli' }}
+      run: |
+        yarn link-cli "$HOME/bin/agoric"
+        echo "AGORIC_CMD=[\"$HOME/bin/agoric\"]" >> $GITHUB_ENV
+
+    - name: Start local NPM registry
+      if: ${{ matrix.cli == 'local-npm' }}
+      run: |
+        tmp_registry_log=`mktemp`
+        nohup npx verdaccio@^5.4.0 &>$tmp_registry_log &
+        # Wait for `verdaccio` to boot
+        grep -q 'http address' <(tail -f $tmp_registry_log)
+
+        # Set registry to local registry
+        npm set registry http://localhost:4873
+        yarn config set registry http://localhost:4873
+
+        # Login so we can publish packages
+        npx npm-cli-login@^1.0.0 -u user -p password -e user@example.com \
+          -r http://localhost:4873 --quotes
+
+        npm whoami
+
+        yarn lerna publish --conventional-prerelease --canary --exact \
+          --dist-tag=blah --preid=dev-$(git rev-parse --short=7 HEAD) \
+          --no-push --no-verify-access --yes
+
+        yarn global add agoric@blah </dev/null
+
+        # Use the locally-installed dist-tag.
+        echo 'AGORIC_INSTALL_OPTIONS=["blah"]' >> $GITHUB_ENV
+        echo 'AGORIC_START_OPTIONS=["--rebuild"]' >> $GITHUB_ENV
+        echo 'AGORIC_CMD=["agoric"]' >> $GITHUB_ENV
 
     - name: run agoric-cli integration-test
       working-directory: ./packages/agoric-cli
-      run: PATH="$PATH:$HOME/bin" yarn integration-test
+      run: yarn integration-test
       env:
         AGORIC_INIT_OPTIONS: '["--dapp-branch=${{steps.get-branch.outputs.result}}"]'
         # Try to avoid hitting a pessimal Actions output rate-limitation.
-        SOLO_MAX_DEBUG_LENGTH: '1024'
+        SOLO_MAX_DEBUG_LENGTH: '1024'

packages/agoric-cli/package.json

@@ -4,13 +4,13 @@
   "description": "Manage the Agoric Javascript smart contract platform",
   "type": "module",
   "main": "src/main.js",
-  "bin": "bin/agoric",
+  "bin": "src/entrypoint.js",
   "files": [
-    "lib",
+    "src",
     "exported.js"
   ],
   "scripts": {
-    "build": "exit 0",
+    "build": "./scripts/get-sdk-package-names.js > src/sdk-package-names.js",
     "test": "ava",
     "test:c8": "c8 $C8_OPTIONS ava --config=ava-nesm.config.js",
     "test:xs": "exit 0",

packages/agoric-cli/scripts/get-sdk-package-names.js

@@ -0,0 +1,34 @@
+#! /usr/bin/env node
+/* global process, Buffer */
+import { spawn } from 'child_process';
+import { basename } from 'path';
+
+const ps = spawn('yarn', ['workspaces', '--silent', 'info'], {
+  stdio: ['ignore', 'pipe', 'inherit'],
+});
+
+// Get Buffers of output.
+const chunks = [];
+ps.stdout.on('data', data => chunks.push(data));
+
+// Wait for the process to exit.
+ps.on('close', code => {
+  if (code !== 0) {
+    throw Error(`yarn info exited with code ${code}`);
+  }
+
+  // Get the output.
+  const json = Buffer.concat(chunks).toString('utf8');
+  // process.stderr.write(json);
+
+  // Write the module.
+  const workspaces = Object.keys(JSON.parse(json)).sort();
+  process.stdout.write(`\
+// DO NOT EDIT - automatically generated by ${basename(
+    new URL(import.meta.url).pathname,
+  )}
+/* eslint-disable comma-dangle,quotes */
+// prettier-ignore
+export default ${JSON.stringify(workspaces, null, 2)};
+`);
+});

packages/agoric-cli/src/helpers.js

@@ -3,13 +3,21 @@
 
 /** @typedef {import('child_process').ChildProcess} ChildProcess */
 
-export const getSDKBinaries = () => {
+export const getSDKBinaries = ({
+  jsPfx = '../..',
+  goPfx = `${jsPfx}/../golang`,
+} = {}) => {
   const myUrl = import.meta.url;
+  const cosmosBuild = ['make', '-C', `${goPfx}/cosmos`, 'all'];
+  const xsnap = new URL(`${jsPfx}/xsnap`, myUrl).pathname;
   return {
-    agSolo: new URL('../../solo/src/entrypoint.js', myUrl).pathname,
-    cosmosChain: new URL('../../cosmic-swingset/bin/ag-chain-cosmos', myUrl)
+    agSolo: new URL(`${jsPfx}/solo/src/entrypoint.js`, myUrl).pathname,
+    agSoloBuild: ['yarn', '--cwd', xsnap, `build:from-env`],
+    cosmosChain: new URL(`${jsPfx}/cosmic-swingset/bin/ag-chain-cosmos`, myUrl)
       .pathname,
-    cosmosHelper: new URL('../../../golang/cosmos/build/agd', myUrl).pathname,
+    cosmosChainBuild: cosmosBuild,
+    cosmosClientBuild: cosmosBuild,
+    cosmosHelper: new URL(`${goPfx}/cosmos/build/agd`, myUrl).pathname,
   };
 };
 

packages/agoric-cli/src/install.js

@@ -2,6 +2,12 @@
 import path from 'path';
 import chalk from 'chalk';
 import { makePspawn } from './helpers.js';
+import DEFAULT_SDK_PACKAGE_NAMES from './sdk-package-names.js';
+
+const REQUIRED_AGORIC_START_PACKAGES = [
+  '@agoric/solo',
+  '@agoric/cosmic-swingset',
+];
 
 const filename = new URL(import.meta.url).pathname;
 const dirname = path.dirname(filename);
@@ -24,7 +30,7 @@ export default async function installMain(progname, rawArgs, powers, opts) {
 
   const rimraf = file => pspawn('rm', ['-rf', file]);
 
-  async function workspaceDirectories(cwd = '.') {
+  async function getWorktreePackagePaths(cwd = '.', map = new Map()) {
     // run `yarn workspaces info` to get the list of directories to
     // use, instead of a hard-coded list
     const p = pspawn('yarn', ['workspaces', '--silent', 'info'], {
@@ -35,14 +41,17 @@ export default async function installMain(progname, rawArgs, powers, opts) {
     p.childProcess.stdout.on('data', out => stdout.push(out));
     await p;
     const d = JSON.parse(Buffer.concat(stdout).toString('utf-8'));
-    const subdirs = Object.values(d).map(v => v.location);
-    return subdirs;
+    Object.entries(d).forEach(([name, { location }]) =>
+      map.set(name, path.resolve(cwd, location)),
+    );
+    return map;
   }
 
   let subdirs;
   const workTrees = ['.'];
   let sdkWorktree;
-  const versions = new Map();
+  /** @type {Map<string, string>} */
+  const sdkPackageToPath = new Map();
   const linkFolder = path.resolve(`_agstate/yarn-links`);
   const linkFlags = [];
   if (opts.sdk) {
@@ -95,7 +104,7 @@ export default async function installMain(progname, rawArgs, powers, opts) {
             const prunedDeps = { ...updatedDeps };
             prunedPackageDescriptor[depSection] = prunedDeps;
             for (const pkg of Object.keys(updatedDeps)) {
-              if (versions.has(pkg)) {
+              if (sdkPackageToPath.has(pkg)) {
                 if (updatedDeps[pkg] === forceVersion) {
                   // We need to remove the old package and `yarn install` to
                   // prune the old dependencies both from disk and also the
@@ -109,6 +118,18 @@ export default async function installMain(progname, rawArgs, powers, opts) {
             }
           }
         }
+        // Ensure the top-level package has the `agoric start` dependencies.
+        if (subdir === '.') {
+          for (const pkg of REQUIRED_AGORIC_START_PACKAGES) {
+            const updatedDevDeps =
+              updatedPackageDescriptor.devDependencies || {};
+            if (!updatedDevDeps[pkg]) {
+              updatedDevDeps[pkg] = forceVersion;
+              updatedPackageDescriptor.devDependencies = updatedDevDeps;
+            }
+          }
+        }
+
         // Ensure we update the package.json before exiting.
         const updatePackageJson = async () => {
           // Don't update on exit anymore.
@@ -187,13 +208,19 @@ export default async function installMain(progname, rawArgs, powers, opts) {
       throw err;
     });
   if (dappPackageJSON.useWorkspaces) {
-    const workdirs = await workspaceDirectories();
+    const workdirs = await getWorktreePackagePaths().then(pp => [
+      ...pp.values(),
+    ]);
     sdkWorktree = workdirs.find(subd => subd === 'agoric-sdk');
     subdirs = ['.', ...workdirs.filter(subd => subd !== 'agoric-sdk')];
 
     // Add 'ui' directory by default, if it exists.
     if (!subdirs.find(subd => subd === 'ui')) {
-      if (await fs.stat(`ui/package.json`).catch(_ => false)) {
+      const uiPackageJSON = await fs.readFile(`ui/package.json`, 'utf-8').then(
+        data => JSON.parse(data),
+        _err => ({}),
+      );
+      if (uiPackageJSON.name) {
         subdirs.push('ui');
         workTrees.push('ui');
       }
@@ -212,34 +239,10 @@ export default async function installMain(progname, rawArgs, powers, opts) {
     subdirs = existingSubdirs.filter(subd => subd);
   }
 
-  /** @type {Map<string, string>} */
-  const packages = new Map();
-
-  const sdkRoot = path.resolve(dirname, `../../..`);
-  const sdkDirs = await workspaceDirectories(sdkRoot);
-  await Promise.all(
-    sdkDirs.map(async location => {
-      const dir = `${sdkRoot}/${location}`;
-      const packageJSON = await fs
-        .readFile(`${dir}/package.json`, 'utf-8')
-        .catch(err => log('error reading', `${dir}/package.json`, err));
-      if (!packageJSON) {
-        return;
-      }
-
-      const pj = JSON.parse(packageJSON);
-      if (pj.private) {
-        log('not linking private package', pj.name);
-        return;
-      }
-
-      // Save our metadata.
-      packages.set(dir, pj.name);
-      versions.set(pj.name, pj.version);
-    }),
-  );
-
   if (opts.sdk) {
+    const sdkRoot = path.resolve(dirname, `../../..`);
+    await getWorktreePackagePaths(sdkRoot, sdkPackageToPath);
+
     // We remove all the links to prevent `yarn install` below from corrupting
     // them.
     log('removing', linkFolder);
@@ -261,12 +264,14 @@ export default async function installMain(progname, rawArgs, powers, opts) {
         // This is needed to prevent yarn errors when installing new versions of
         // linked modules (like `@agoric/zoe`).
         await Promise.all(
-          [...packages.values()].map(async pjName =>
+          [...sdkPackageToPath.keys()].map(async pjName =>
             fs.unlink(`${nm}/${pjName}`).catch(_ => {}),
           ),
         );
       }),
     );
+  } else {
+    DEFAULT_SDK_PACKAGE_NAMES.forEach(name => sdkPackageToPath.set(name, null));
   }
 
   if (forceSdkVersion !== undefined) {
@@ -283,7 +288,7 @@ export default async function installMain(progname, rawArgs, powers, opts) {
 
   // Create symlinks to the SDK packages.
   await Promise.all(
-    [...packages.entries()].map(async ([dir, pjName]) => {
+    [...sdkPackageToPath.entries()].map(async ([pjName, dir]) => {
       const SUBOPTIMAL = false;
       if (SUBOPTIMAL) {
         // This use of yarn is noisy and slow.
@@ -304,7 +309,7 @@ export default async function installMain(progname, rawArgs, powers, opts) {
     }),
   );
 
-  const sdkPackages = [...packages.values()].sort();
+  const sdkPackages = [...sdkPackageToPath.keys()].sort();
   for (const subdir of subdirs) {
     // eslint-disable-next-line no-await-in-loop
     const exists = await fs.stat(`${subdir}/package.json`).catch(_ => false);

packages/agoric-cli/src/main.js

@@ -52,11 +52,7 @@ const main = async (progname, rawArgs, powers) => {
   program
     .option('--sdk', 'use the Agoric SDK containing this program')
     .option('--no-sdk', 'do not use the Agoric SDK containing this program')
-    .option(
-      '--docker-tag <tag>',
-      'image tag to use for Docker containers',
-      'latest',
-    )
+    .option('--docker-tag <tag>', 'image tag to use for Docker containers')
     .option(
       '-v, --verbose',
       'verbosity that can be increased',
@@ -201,6 +197,7 @@ const main = async (progname, rawArgs, powers) => {
     .option('--reset', 'clear all VM state before starting')
     .option('--no-restart', 'do not actually start the VM')
     .option('--pull', 'for Docker-based VM, pull the image before running')
+    .option('--rebuild', 'rebuild VM dependencies before running')
     .option(
       '--delay [seconds]',
       'delay for simulated chain to process messages',

packages/agoric-cli/src/sdk-package-names.js

@@ -0,0 +1,47 @@
+// DO NOT EDIT - automatically generated by get-sdk-package-names.js
+/* eslint-disable comma-dangle,quotes */
+// prettier-ignore
+export default [
+  "@agoric/access-token",
+  "@agoric/assert",
+  "@agoric/bundle-source",
+  "@agoric/captp",
+  "@agoric/cosmic-swingset",
+  "@agoric/cosmos",
+  "@agoric/deploy-script-support",
+  "@agoric/deployment",
+  "@agoric/ertp",
+  "@agoric/eslint-config",
+  "@agoric/eventual-send",
+  "@agoric/far",
+  "@agoric/governance",
+  "@agoric/import-bundle",
+  "@agoric/import-manager",
+  "@agoric/install-ses",
+  "@agoric/lockdown",
+  "@agoric/marshal",
+  "@agoric/notifier",
+  "@agoric/pegasus",
+  "@agoric/promise-kit",
+  "@agoric/run-protocol",
+  "@agoric/same-structure",
+  "@agoric/sharing-service",
+  "@agoric/solo",
+  "@agoric/sparse-ints",
+  "@agoric/spawner",
+  "@agoric/stat-logger",
+  "@agoric/store",
+  "@agoric/swing-store",
+  "@agoric/swingset-runner",
+  "@agoric/swingset-vat",
+  "@agoric/ui-components",
+  "@agoric/vats",
+  "@agoric/wallet",
+  "@agoric/wallet-backend",
+  "@agoric/wallet-connection",
+  "@agoric/wallet-ui",
+  "@agoric/web-components",
+  "@agoric/xsnap",
+  "@agoric/zoe",
+  "agoric"
+];