From 746e48cffbb755407d4b4e9485bcd5d78bbfe4cf Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sun, 2 Aug 2020 16:12:32 -0500 Subject: [PATCH 01/20] test(xs-vat-worker): prototype package with 1 integration test vatWorker runs a vat in its own process. We have drivers for both node and xs, though only the node driver is working as of this commit. We also have a kernelSimulator that takes a vat transcript (as from swingset-runner/bin/kerneldump) and simulates a swingset kernel vat manager. Earlier vatWorker work was done in a gist: https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7 This commit is based on: 2020-07-16 b17fae8 thru 2020-08-02 a32f30e simple transcript runs as integration test The xs driver was working, to some extent, in: 2020-08-01 5b80235 agoric-sdk catch-up See also https://github.com/Agoric/agoric-sdk/issues/1299 --- package.json | 3 +- packages/xs-vat-worker/.eslintignore | 4 + packages/xs-vat-worker/.eslintrc.js | 27 +++ packages/xs-vat-worker/.npmignore | 11 ++ packages/xs-vat-worker/.prettierrc.json | 4 + packages/xs-vat-worker/bin/node-vat-worker | 2 + packages/xs-vat-worker/manifest.json | 48 ++++++ packages/xs-vat-worker/package.json | 57 +++++++ packages/xs-vat-worker/src-native/fdchan.c | 74 ++++++++ packages/xs-vat-worker/src-native/fdchan.js | 9 + packages/xs-vat-worker/src/console.js | 25 +++ packages/xs-vat-worker/src/index.js | 111 ++++++++++++ packages/xs-vat-worker/src/main.js | 26 +++ packages/xs-vat-worker/src/netstring.js | 51 ++++++ packages/xs-vat-worker/src/timer-ticks.js | 17 ++ packages/xs-vat-worker/src/vatWorker.js | 159 ++++++++++++++++++ packages/xs-vat-worker/test/Makefile | 29 ++++ packages/xs-vat-worker/test/bootstrap.js | 34 ++++ .../xs-vat-worker/test/kernelSimulator.js | 134 +++++++++++++++ .../xs-vat-worker/test/sort_transcript.py | 5 + packages/xs-vat-worker/test/test-vat1.js | 28 +++ packages/xs-vat-worker/test/transcript.txt | 6 + packages/xs-vat-worker/test/vat-target.js | 55 ++++++ .../@agoric/install-ses/install-ses.js | 17 ++ 24 files changed, 935 insertions(+), 1 deletion(-) create mode 100644 packages/xs-vat-worker/.eslintignore create mode 100644 packages/xs-vat-worker/.eslintrc.js create mode 100644 packages/xs-vat-worker/.npmignore create mode 100644 packages/xs-vat-worker/.prettierrc.json create mode 100755 packages/xs-vat-worker/bin/node-vat-worker create mode 100644 packages/xs-vat-worker/manifest.json create mode 100644 packages/xs-vat-worker/package.json create mode 100644 packages/xs-vat-worker/src-native/fdchan.c create mode 100644 packages/xs-vat-worker/src-native/fdchan.js create mode 100644 packages/xs-vat-worker/src/console.js create mode 100755 packages/xs-vat-worker/src/index.js create mode 100644 packages/xs-vat-worker/src/main.js create mode 100644 packages/xs-vat-worker/src/netstring.js create mode 100644 packages/xs-vat-worker/src/timer-ticks.js create mode 100644 packages/xs-vat-worker/src/vatWorker.js create mode 100644 packages/xs-vat-worker/test/Makefile create mode 100644 packages/xs-vat-worker/test/bootstrap.js create mode 100644 packages/xs-vat-worker/test/kernelSimulator.js create mode 100644 packages/xs-vat-worker/test/sort_transcript.py create mode 100644 packages/xs-vat-worker/test/test-vat1.js create mode 100644 packages/xs-vat-worker/test/transcript.txt create mode 100644 packages/xs-vat-worker/test/vat-target.js create mode 100644 packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js diff --git a/package.json b/package.json index b73b3b4d16f..68620c702ed 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "packages/cosmic-swingset", "packages/agoric-cli", "packages/deployment", - "packages/notifier" + "packages/notifier", + "packages/xs-vat-worker" ], "devDependencies": { "ava": "^3.11.1", diff --git a/packages/xs-vat-worker/.eslintignore b/packages/xs-vat-worker/.eslintignore new file mode 100644 index 00000000000..ea65306b6ba --- /dev/null +++ b/packages/xs-vat-worker/.eslintignore @@ -0,0 +1,4 @@ +/dist/ +/scripts/ +/xs_modules/ +/swingset/ diff --git a/packages/xs-vat-worker/.eslintrc.js b/packages/xs-vat-worker/.eslintrc.js new file mode 100644 index 00000000000..3f42e1d90c7 --- /dev/null +++ b/packages/xs-vat-worker/.eslintrc.js @@ -0,0 +1,27 @@ +/* global module */ +module.exports = { + // parser: "babel-eslint", + extends: ['airbnb-base', 'plugin:prettier/recommended'], + env: { + es6: true, // supports new ES6 globals (e.g., new types such as Set) + }, + globals: { + "harden": "readonly", + }, + rules: { + 'implicit-arrow-linebreak': 'off', + 'function-paren-newline': 'off', + 'arrow-parens': 'off', + strict: 'off', + 'prefer-destructuring': 'off', + 'no-else-return': 'off', + 'no-console': 'off', + 'no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + 'no-return-assign': 'off', + 'no-param-reassign': 'off', + 'no-restricted-syntax': ['off', 'ForOfStatement'], + 'no-unused-expressions': 'off', + 'no-loop-func': 'off', + 'import/prefer-default-export': 'off', // contrary to Agoric standard + }, +}; diff --git a/packages/xs-vat-worker/.npmignore b/packages/xs-vat-worker/.npmignore new file mode 100644 index 00000000000..cd75c6170bf --- /dev/null +++ b/packages/xs-vat-worker/.npmignore @@ -0,0 +1,11 @@ +# Demo +demo + +# scripts +scripts + +# test +test + +# Travis CI +.travis.yml diff --git a/packages/xs-vat-worker/.prettierrc.json b/packages/xs-vat-worker/.prettierrc.json new file mode 100644 index 00000000000..6e778b4fb9c --- /dev/null +++ b/packages/xs-vat-worker/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "trailingComma": "all", + "singleQuote": true +} diff --git a/packages/xs-vat-worker/bin/node-vat-worker b/packages/xs-vat-worker/bin/node-vat-worker new file mode 100755 index 00000000000..9461d5dbf47 --- /dev/null +++ b/packages/xs-vat-worker/bin/node-vat-worker @@ -0,0 +1,2 @@ +#!/bin/sh +node -r esm ./src/index.js diff --git a/packages/xs-vat-worker/manifest.json b/packages/xs-vat-worker/manifest.json new file mode 100644 index 00000000000..411a331daf3 --- /dev/null +++ b/packages/xs-vat-worker/manifest.json @@ -0,0 +1,48 @@ +{ + "include": [ + "$(MODDABLE)/examples/manifest_base.json" + ], + "$note": [ + "horrible @agoric/E KLUDGE below", + "liveSlots is a bit KLUDGy too" + ], + "creation": { + "keys": { + "available": 4096 + }, + "stack": 4096, + "parser": { + "buffer": 32768 + } + }, + "strip": [], + "modules": { + "files": [ "$(MODULES)/files/file/*", "$(MODULES)/files/file/lin/*" ], + "fdchan": [ "./fdchan" ], + "timer": [ + "$(MODULES)/base/timer/timer", + "$(MODULES)/base/timer/lin/*", + ], + "@agoric/nat": "xs_modules/@agoric/nat/nat.esm", + "@agoric/install-ses": "xs_modules/@agoric/install-ses/install-ses", + "@agoric/import-bundle": "xs_modules/@agoric/import-bundle/index", + "@agoric/compartment-wrapper": "xs_modules/@agoric/import-bundle/compartment-wrapper", + "@agoric/eventual-send": [ + "xs_modules/@agoric/eventual-send/index" + ], + "@agoric/E": "xs_modules/@agoric/eventual-send/E", + "@agoric/marshal": "xs_modules/@agoric/marshal/marshal", + "@agoric/produce-promise": "xs_modules/@agoric/produce-promise/producePromise", + "@agoric/assert": "xs_modules/@agoric/assert/assert", + "@agoric/types": "xs_modules/@agoric/assert/types", + "swingset/capdata": "./swingset/capdata", + "swingset/parseVatSlots": "./swingset/parseVatSlots", + "swingset/kernel/liveSlots": "./swingset/kernel/liveSlots", + "*": [ + "./main", + "./console", + "./timer-ticks", + "./vatWorker" + ] + } +} diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json new file mode 100644 index 00000000000..66624812561 --- /dev/null +++ b/packages/xs-vat-worker/package.json @@ -0,0 +1,57 @@ +{ + "name": "@agoric/xs-vat-worker", + "version": "0.0.1", + "description": "???", + "main": "dist/vat-worker.cjs.js", + "module": "dist/vat-worker.esm.js", + "browser": "dist/vat-worker.umd.js", + "scripts": { + "xs-build": "mkdir -p build; mcconfig -o build -p x-cli-lin -m -d", + "xs-run": "yarn xs-build && ./build/bin/lin/debug/make-vat-transcript", + "xs-build-release": "mkdir -p build; mcconfig -o build -p x-cli-lin -m", + "xs-run-release": "yarn xs-build-release && ./build/bin/lin/release/make-vat-transcript", + "test": "tape -r esm 'test/**/test-*.js'", + "build": "exit 0", + "lint-fix": "eslint --fix '**/*.{js,jsx}'", + "lint-check": "eslint '**/*.{js,jsx}'", + "lint-fix-jessie": "eslint -c '.eslintrc-jessie.js' --fix '**/*.{js,jsx}'", + "lint-check-jessie": "eslint -c '.eslintrc-jessie.js' '**/*.{js,jsx}'" + }, + "devDependencies": { + "eslint": "^6.1.0", + "eslint-config-airbnb-base": "^14.0.0", + "eslint-config-jessie": "0.0.3", + "eslint-config-prettier": "^6.0.0", + "eslint-plugin-import": "^2.18.2", + "eslint-plugin-prettier": "^3.1.0", + "prettier": "^1.18.2", + "tap-spec": "^5.0.0", + "tape": "^4.11.0", + "tape-promise": "^4.0.0" + }, + "dependencies": { + "@agoric/assert": "^0.0.8", + "@agoric/bundle-source": "^1.1.6", + "@agoric/eventual-send": "^0.9.3", + "@agoric/import-bundle": "^0.0.8", + "@agoric/install-ses": "^0.2.0", + "@agoric/produce-promise": "^0.1.3", + "@agoric/swingset-vat": "^0.6.0", + "anylogger": "^1.0.4", + "esm": "^3.2.5" + }, + "keywords": [], + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://gist.github.com/f8e0b5d838079a994784d599c282cce7.git" + }, + "author": "Agoric", + "license": "Apache-2.0", + "bugs": { + "url": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7" + }, + "homepage": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7" +} diff --git a/packages/xs-vat-worker/src-native/fdchan.c b/packages/xs-vat-worker/src-native/fdchan.c new file mode 100644 index 00000000000..6ee2dd1bc55 --- /dev/null +++ b/packages/xs-vat-worker/src-native/fdchan.c @@ -0,0 +1,74 @@ +/** + * ref Netstrings 19970201 by Bernstein + * https://cr.yp.to/proto/netstrings.txt + * + * Moddable in C + * https://github.com/Moddable-OpenSource/moddable/blob/public/documentation/xs/XS%20in%20C.md + * + * stdio + * https://pubs.opengroup.org/onlinepubs/009695399/basedefs/stdio.h.html + */ + +#include +#include + +#include "xsAll.h" +#include "xs.h" + +void xs_Reader(xsMachine *the) { + int argc = xsToInteger(xsArgc); + if (argc < 1) { + mxTypeError("expected fd"); + } + int fd = xsToInteger(xsArg(0)); + FILE *inStream = fdopen(fd, "rb"); + if (!inStream) { + mxUnknownError("fdopen failed"); + } + xsSetHostData(xsThis, (void *)((uintptr_t)inStream)); + + // modInstrumentationAdjust(Files, +1); +} + +void xs_Writer(xsMachine *the) { + int argc = xsToInteger(xsArgc); + if (argc < 1) { + mxTypeError("expected fd"); + } + int fd = xsToInteger(xsArg(0)); + FILE *outStream = fdopen(fd, "wb"); + if (!outStream) { + mxUnknownError("fdopen failed"); + } + xsSetHostData(xsThis, (void *)((uintptr_t)outStream)); + + // modInstrumentationAdjust(Files, +1); +} + +void xs_read_netstring(xsMachine *the) { + size_t len; + char* buf = NULL; + FILE *inStream = xsGetHostData(xsThis); + assert(inStream); + + if (fscanf(inStream, "%9lu", &len) < 1) { goto BARF; } /* >999999999 bytes is bad */ + // fprintf(stderr, "xs_stdin_read_netstring len %lu\n", len); + if (fgetc(inStream) != ':') { goto BARF; } + buf = malloc(len + 1); /* malloc(0) is not portable */ + if (!buf) { goto BARF; } + if (fread(buf, 1, len, inStream) < len) { goto BARF; } + if (fgetc(inStream) != ',') { goto BARF; } + + xsResult = xsStringBuffer(buf, len); + free(buf); + // fprintf(stderr, "xs_stdin_read_nestring return\n"); + return; + +BARF: + free(buf); + xsUnknownError("getline failed"); +} + +void xs_fdchan_destructor() { + +} diff --git a/packages/xs-vat-worker/src-native/fdchan.js b/packages/xs-vat-worker/src-native/fdchan.js new file mode 100644 index 00000000000..f7d42f380d6 --- /dev/null +++ b/packages/xs-vat-worker/src-native/fdchan.js @@ -0,0 +1,9 @@ +export class Reader @ "xs_fdchan_destructor" { + constructor(fd) @ "xs_Reader" + read_netstring() @ "xs_read_netstring"; +} + +export class Writer @ "xs_fdchan_destructor" { + constructor(fd) @ "xs_Writer" + write(...items) @ "xs_file_write"; +} diff --git a/packages/xs-vat-worker/src/console.js b/packages/xs-vat-worker/src/console.js new file mode 100644 index 00000000000..ac8ba1bf457 --- /dev/null +++ b/packages/xs-vat-worker/src/console.js @@ -0,0 +1,25 @@ +/** console for xs platform */ +/* global trace, globalThis */ +const harden = x => Object.freeze(x, true); + +const text = it => (typeof it === 'object' ? JSON.stringify(it) : `${it}`); +const combine = (...things) => `${things.map(text).join(' ')}\n`; + +export function makeConsole(write_) { + const write = write_ || trace; // note ocap exception for tracing / logging + return harden({ + log(...things) { + write(combine(...things)); + }, + // node.js docs say this is just an alias for error + warn(...things) { + write(combine('WARNING: ', ...things)); + }, + // node docs say this goes to stderr + error(...things) { + write(combine('ERROR: ', ...things)); + }, + }); +} + +globalThis.console = makeConsole(); diff --git a/packages/xs-vat-worker/src/index.js b/packages/xs-vat-worker/src/index.js new file mode 100755 index 00000000000..1cde573ea1d --- /dev/null +++ b/packages/xs-vat-worker/src/index.js @@ -0,0 +1,111 @@ +// vatWorker driver for node.js +// contrast with main.js for xs +import '@agoric/install-ses'; + +import { main as vatWorker } from './vatWorker'; + +const INFD = 3; +const OUTFD = 4; + +function makePipe(io, sleep) { + function write(data) { + let done = 0; + for (;;) { + try { + done += io.writeSync(OUTFD, data.slice(done)); + if (done >= data.length) { + return done; + } + } catch (writeFailed) { + if (writeFailed.code === 'EAGAIN') { + sleep(0.1); + // try again + } else { + throw writeFailed; + } + } + } + } + + return harden({ + writeMessage(msg) { + write(`${msg.length}:`); + write(msg); + write(','); + }, + readMessage(EOF) { + let buf = Buffer.from('999999999:', 'utf-8'); + let len = null; + let colonPos = null; + let offset = 0; + + for (;;) { + // console.error('readMessage', { length: buf.length, len, colonPos, offset }); + try { + offset += io.readSync(INFD, buf, { + offset, + length: buf.length - offset, + }); + } catch (err) { + if (err.code === 'EAGAIN') { + sleep(0.1); + // eslint-disable-next-line no-continue + continue; + } else if (err.code === 'EOF') { + throw EOF; + } else { + throw err; + } + } + if (len === null) { + colonPos = buf.indexOf(':'); + if (colonPos > 0) { + const digits = buf.slice(0, colonPos).toString('utf-8'); + len = parseInt(digits, 10); + const rest = Buffer.alloc(len + 1); + // console.error('parsed len. copy', { digits, len, targetStart: 0, sourceStart: colonPos + 1, sourceEnd: offset }); + buf.copy(rest, 0, colonPos + 1, offset); + buf = rest; + offset -= colonPos + 1; + } + } else if (offset === len + 1) { + const delim = buf.slice(-1).toString('utf-8'); + if (delim !== ',') { + throw new Error( + `bad netstring: length ${len} expected , found [${delim}]`, + ); + } + const result = buf.slice(0, -1).toString('utf-8'); + // console.error({ colon: colonPos, len, result: result.slice(0, 20) }); + return result; + } + } + }, + }); +} + +async function main({ setImmediate, fs, spawnSync }) { + const sleep = secs => spawnSync('sleep', [secs]); + const pipe = makePipe(fs, sleep); + return vatWorker({ + readMessage: pipe.readMessage, + writeMessage: pipe.writeMessage, + setImmediate, + }); +} + +main({ + setImmediate, + // eslint-disable-next-line global-require + spawnSync: require('child_process').spawnSync, + fs: { + // eslint-disable-next-line global-require + readSync: require('fs').readSync, + // eslint-disable-next-line global-require + writeSync: require('fs').writeSync, + }, +}) + .catch(err => { + console.error(err); + }) + .then(() => process.exit(0)); diff --git a/packages/xs-vat-worker/src/main.js b/packages/xs-vat-worker/src/main.js new file mode 100644 index 00000000000..eb03ae228d3 --- /dev/null +++ b/packages/xs-vat-worker/src/main.js @@ -0,0 +1,26 @@ +// add harden; align xs's Compartment with Agoric's +// ISSUE: use a different module name? +import '@agoric/install-ses'; + +import './console'; // sets globalThis.console. ew. +import './timer-ticks'; // globalThis.setTimeout. ew. + +import { main as vatWorker } from './vatWorker'; +import { Reader, Writer } from './fdchan'; + +const INFD = 3; +const OUTFD = 4; + +export default async function main() { + const inStream = new Reader(INFD); + const outStream = new Writer(OUTFD); + + return vatWorker({ + setImmediate, + readMessage: () => inStream.read_netstring(), + writeMessage: message => { + // ISSUE: should be byte length + outStream.write(`${message.length}:`, message, ','); + }, + }); +} diff --git a/packages/xs-vat-worker/src/netstring.js b/packages/xs-vat-worker/src/netstring.js new file mode 100644 index 00000000000..3bd8edc1bdc --- /dev/null +++ b/packages/xs-vat-worker/src/netstring.js @@ -0,0 +1,51 @@ +export async function readNetstring(input) { + let prefix = Buffer.from([]); + let colonPos = -1; + + const nextChunk = () => + new Promise((resolve, _reject) => { + const rx = data => { + input.pause(); + input.removeListener('data', rx); + resolve(data); + }; + input.on('data', rx); + input.resume(); + }); + + while (colonPos < 0) { + // eslint-disable-next-line no-await-in-loop + const more = await nextChunk(); + prefix = Buffer.concat([prefix, more]); + colonPos = prefix.indexOf(':'); + } + let len; + const digits = prefix.slice(0, colonPos).toString('utf-8'); + try { + len = parseInt(digits, 10); + } catch (badLen) { + throw new Error(`bad netstring length ${digits}`); + } + // console.error('readNetstring parsed len', { digits, len }); + let data = prefix.slice(colonPos + 1); + while (data.length <= len) { + // console.log('netstring: looking for payload', data.length, len); + // eslint-disable-next-line no-await-in-loop + const more = await nextChunk(input); + data = Buffer.concat([data, more]); + } + if (data.slice(len).toString('utf-8') !== ',') { + throw new Error( + `bad netstring: expected , got ${data.slice(-1)} [${data.slice(-20)}]`, + ); + } + return data.slice(0, len).toString('utf-8'); +} + +export async function writeNetstring(out, payload) { + // ISSUE: non-ASCII length + // console.log('kernelSimulator send size', content.length); + await out.write(`${payload.length}:`); + await out.write(payload); + await out.write(','); +} diff --git a/packages/xs-vat-worker/src/timer-ticks.js b/packages/xs-vat-worker/src/timer-ticks.js new file mode 100644 index 00000000000..39c96d0580e --- /dev/null +++ b/packages/xs-vat-worker/src/timer-ticks.js @@ -0,0 +1,17 @@ +// ref moddable/examples/base/timers/main.js +/* global globalThis */ + +// eslint-disable-next-line import/no-unresolved +import Timer from 'timer'; // moddable timer + +globalThis.setImmediate = callback => { + Timer.set(callback); +}; + +globalThis.setTimeout = (callback, delay) => { + Timer.set(callback, delay); +}; + +globalThis.setInterval = (callback, delay) => { + Timer.repeat(callback, delay); +}; diff --git a/packages/xs-vat-worker/src/vatWorker.js b/packages/xs-vat-worker/src/vatWorker.js new file mode 100644 index 00000000000..a7e741379eb --- /dev/null +++ b/packages/xs-vat-worker/src/vatWorker.js @@ -0,0 +1,159 @@ +import { importBundle } from '@agoric/import-bundle'; +import { HandledPromise } from '@agoric/eventual-send'; +// TODO? import anylogger from 'anylogger'; +import { makeLiveSlots } from '@agoric/swingset-vat/src/kernel/liveSlots'; + +const EOF = new Error('EOF'); + +// from SwingSet/src/controller.js +function makeConsole(_tag) { + const log = console; // TODO? anylogger(tag); + const cons = {}; + for (const level of ['debug', 'log', 'info', 'warn', 'error']) { + cons[level] = log[level]; + } + return harden(cons); +} + +function makeVatEndowments(consoleTag) { + return harden({ + console: makeConsole(`SwingSet:${consoleTag}`), + HandledPromise, + // TODO: re2 is a RegExp work-a-like that disables backtracking expressions for + // safer memory consumption + RegExp, + }); +} + +// see also: detecting an empty vat promise queue (end of "crank") +// https://github.com/Agoric/agoric-sdk/issues/45 +function endOfCrank(setImmediate) { + return new Promise((resolve, _reject) => { + setImmediate(() => { + // console.log('hello from setImmediate callback. The promise queue is presumably empty.'); + resolve(); + }); + }); +} + +function makeWorker(io, setImmediate) { + let vatNS = null; + let dispatch; + let state; + + const format = msg => JSON.stringify(msg); + const sync = (method, args) => { + io.writeMessage(format({ msgtype: 'syscall', method, args })); + return JSON.parse(io.readMessage()); + }; + const syscall = harden({ + subscribe(...args) { + return sync('subscribe', args); + }, + send(...args) { + return sync('send', args); + }, + fulfillToData(...args) { + return sync('fulfillToData', args); + }, + fulfillToPresence(...args) { + return sync('fulfillToPresence', args); + }, + reject(...args) { + return sync('reject', args); + }, + }); + + async function loadBundle(name, bundle) { + if (vatNS !== null) { + throw new Error('bundle already loaded'); + } + + vatNS = await importBundle(bundle, { + filePrefix: name, + endowments: makeVatEndowments(name), + }); + // TODO: be sure console.log isn't mixed with protocol stream + // console.log('loaded bundle with methods', Object.keys(vatNS)); + + state = {}; // ?? + dispatch = makeLiveSlots(syscall, state, vatNS.buildRootObject); + } + + function turnCrank(dispatchType, args) { + return new Promise((resolve, reject) => { + try { + dispatch[dispatchType](...args); + } catch (error) { + // console.log({ dispatchError: error }); + reject(error); + return; + } + endOfCrank(setImmediate).then(resolve); + }); + } + + const name = 'WORKER'; // TODO? + async function handle(message) { + switch (message.msgtype) { + case 'load-bundle': + try { + await loadBundle(name, message.bundle); + } catch (error) { + // console.log('load-bundle failed:', error); + io.writeMessage( + format({ msgtype: 'load-bundle-nak', error: error.message }), + ); + break; + } + io.writeMessage(format({ msgtype: 'load-bundle-ack' })); + break; + case 'dispatch': + try { + await turnCrank(message.type, message.args); + } catch (error) { + io.writeMessage( + format({ + msgtype: 'dispatch-nak', + error: error instanceof Error ? error.message : error, + }), + ); + break; + } + io.writeMessage(format({ msgtype: 'dispatch-ack' })); + break; + case 'finish': + io.writeMessage(format({ msgtype: 'finish-ack' })); + break; + default: + console.warn('unexpected msgtype', message.msgtype); + } + return message.msgtype; + } + + return harden({ handle }); +} + +export async function main({ readMessage, writeMessage, setImmediate }) { + const worker = makeWorker({ readMessage, writeMessage }, setImmediate); + + for (;;) { + let message; + try { + // eslint-disable-next-line no-await-in-loop + message = JSON.parse(readMessage(EOF)); + } catch (noMessage) { + if (noMessage === EOF) { + return; + } + console.warn('problem getting message:', noMessage); + // eslint-disable-next-line no-continue + continue; + } + // eslint-disable-next-line no-await-in-loop + const msgtype = await worker.handle(message); + if (msgtype === 'finish') { + break; + } + } +} diff --git a/packages/xs-vat-worker/test/Makefile b/packages/xs-vat-worker/test/Makefile new file mode 100644 index 00000000000..36ca5b3b3f8 --- /dev/null +++ b/packages/xs-vat-worker/test/Makefile @@ -0,0 +1,29 @@ +NODE_ESM=node -r esm + +transcript.txt: swingset-kernel-state + $(NODE_ESM) ../swingset-runner/bin/kerneldump --filedb ./swingset-kernel-state | grep v1.t |sort >$@ + +swingset-kernel-state: bootstrap.js vat-target.js + $(NODE_ESM) ../swingset-runner/bin/runner --filedb run + +ZOE1=../zoe/test/swingsetTests/zoe + + +zoe: transcript-zoe.txt ./node-vat-worker index.js vatWorker.js kernelSimulator.js + VAT1=$(ZOE1)/vat-zoe.js TRANSCRIPT=transcript-zoe.txt node -r esm kernelSimulator.js + +zoe-node: transcript-zoe.txt ./node-vat-worker index.js vatWorker.js kernelSimulator.js + VAT1=$(ZOE1)/vat-zoe.js TRANSCRIPT=transcript-zoe.txt WORKERBIN=./node-vat-worker node -r esm kernelSimulator.js + + +transcript-zoe.txt: $(ZOE1)/swingset-kernel-state + $(NODE_ESM) ../swingset-runner/bin/kerneldump --filedb $(ZOE1)/swingset-kernel-state | grep ^v5.t | grep -v nextID >,zoe-all + python sort_transcript.py <,zoe-all >$@ + + +$(ZOE1)/swingset-kernel-state: ,zoe-patched + (cd $(ZOE1) && $(NODE_ESM) ../../../../swingset-runner/bin/runner --filedb run) + +,zoe-patched: $(ZOE1)/bootstrap.js zoe-test.patch + cd ../.. && patch -p1 0, + () => 0, + ); +} + +export function buildRootObject() { + const callbackObj = harden({ + callback(arg1, arg2) { + console.log(`callback`, arg1, arg2); + return ['data', callbackObj]; // four, resolves pF + }, + }); + + const precD = producePromise(); + const precE = producePromise(); + + return harden({ + bootstrap(_argv, vats) { + const pA = E(vats.target).zero(callbackObj, precD.promise, precE.promise); + E(vats.target).one(); + precD.resolve(callbackObj); // two + precE.reject(Error('four')); // three + pA.then(([pB, pC]) => { + ignore(pB); + ignore(pC); + }); + }, + }); +} diff --git a/packages/xs-vat-worker/test/kernelSimulator.js b/packages/xs-vat-worker/test/kernelSimulator.js new file mode 100644 index 00000000000..4bb07c6c05b --- /dev/null +++ b/packages/xs-vat-worker/test/kernelSimulator.js @@ -0,0 +1,134 @@ +// Usage: node -r esm kernelSimulator.js +// context: https://github.com/Agoric/agoric-sdk/issues/1299 +import '@agoric/install-ses'; + +import { readNetstring, writeNetstring } from '../src/netstring'; + +const INFD = 3; +const OUTFD = 4; + +function options(env) { + return { + vat1: env.VAT1 || 'vat-target.js', + transcript: env.TRANSCRIPT || 'transcript.txt', + workerBin: env.WORKERBIN || './build/bin/lin/release/xs-vat-worker', + }; +} + +function makeWorker(child) { + const format = obj => JSON.stringify(obj); + const send = obj => writeNetstring(child.stdio[INFD], format(obj)); + + child.stdio[OUTFD].pause(); + + const expect = async msgtype => { + const txt = await readNetstring(child.stdio[OUTFD]); + let msg; + try { + msg = JSON.parse(txt); + } catch (badJSON) { + console.error('bad JSON ', txt.length, ' chars: [', txt, ']', badJSON); + throw badJSON; + } + if (msg.msgtype !== msgtype) { + throw new Error(`expected ${msgtype}; found: ${msg.msgtype}; error: ${msg.error} [${JSON.stringify(msg)}]`); + } + return msg; + }; + + return harden({ + async loadVat(bundle) { + await send({ msgtype: 'load-bundle', bundle }); + return expect('load-bundle-ack'); + }, + async dispatch({ d, syscalls, _crankNumber }) { + await send({ msgtype: 'dispatch', type: d[0], args: d.slice(1) }); + for (const syscall of syscalls) { + // eslint-disable-next-line no-await-in-loop + const request = await expect('syscall'); + console.log('syscall request', request); + // eslint-disable-next-line no-await-in-loop + await send({ msgtype: 'syscall-ack', response: syscall.response }); + } + return expect('dispatch-ack'); + }, + async finish() { + await send({ msgtype: 'finish' }); + await expect('finish-ack'); + }, + }); +} + +async function runTranscript(w1, bundle, transcript) { + await w1.loadVat(bundle); + console.log('loadVat done.'); + + const events = transcript.split('\n'); + + while (events.length > 0) { + const event = events.shift(); + if (!event) { + // eslint-disable-next-line no-continue + continue; + } + const found = event.match(/^(?[^ ]+) :: (?.*)/); + if (!found) { + console.log('unexpected transcript format', { line: event }); + // eslint-disable-next-line no-continue + continue; + } + const { id, payload } = found.groups; + + const obj = JSON.parse(payload); + if (typeof obj !== 'object') { + console.log('not a dispatch event', id, obj); + // eslint-disable-next-line no-continue + continue; + } + console.log('dispatching:', id); + // eslint-disable-next-line no-await-in-loop + await w1.dispatch(obj); + console.log('dispatch done:', id); + } + + await w1.finish(); + console.log('END OF TRANSCRIPT.'); +} + +// eslint-disable-next-line no-shadow +export async function main(argv, { env, io, bundleSource, spawn }) { + const { vat1, transcript, workerBin } = options(env); + + const bundle = await bundleSource(vat1); + + console.log('spawning', { workerBin }); + const child = await spawn(workerBin, [], { + stdio: ['inherit', 'inherit', 'inherit', 'pipe', 'pipe'], + }); + child.on('exit', (code, signal) => { + if (code !== 0) { + console.error('unexpected exit:', { code, signal }); + } + }); + const w1 = makeWorker(child); + + const text = await io.readFile(transcript, 'utf-8'); + await runTranscript(w1, bundle, text); +} + +if (require.main === module) { + main(process.argv, { + env: process.env, + // eslint-disable-next-line global-require + io: { + // eslint-disable-next-line global-require + readFile: require('fs').promises.readFile, + }, + // eslint-disable-next-line global-require + bundleSource: require('@agoric/bundle-source').default, + // eslint-disable-next-line global-require + spawn: require('child_process').spawn, + }).catch(err => { + console.error(err); + }); +} diff --git a/packages/xs-vat-worker/test/sort_transcript.py b/packages/xs-vat-worker/test/sort_transcript.py new file mode 100644 index 00000000000..c6c478a0c5c --- /dev/null +++ b/packages/xs-vat-worker/test/sort_transcript.py @@ -0,0 +1,5 @@ +import sys +lines = sys.stdin.readlines() +lines.sort(key=lambda line: int(line.split(" ", 1)[0].split(".")[2])) +for line in lines: + sys.stdout.write(line) diff --git a/packages/xs-vat-worker/test/test-vat1.js b/packages/xs-vat-worker/test/test-vat1.js new file mode 100644 index 00000000000..16701075e0e --- /dev/null +++ b/packages/xs-vat-worker/test/test-vat1.js @@ -0,0 +1,28 @@ +import { test } from 'tape-promise/tape'; + +import { main } from './kernelSimulator'; + +const resolve = p => require.resolve(p); + +test('replay simple transcript with node vatWorker', async t => { + process.env.WORKERBIN = resolve('../bin/node-vat-worker'); + process.env.VAT1 = resolve('./vat-target.js'); + process.env.TRANSCRIPT = resolve('./transcript.txt'); + + process.chdir(resolve('..')); // so node-vat-worker can find stuff in src/ + await main(process.argv, { + env: process.env, + // eslint-disable-next-line global-require + io: { + // eslint-disable-next-line global-require + readFile: require('fs').promises.readFile, + }, + // eslint-disable-next-line global-require + bundleSource: require('@agoric/bundle-source').default, + // eslint-disable-next-line global-require + spawn: require('child_process').spawn, + }); + + t.ok('did not crash'); + t.end(); +}); diff --git a/packages/xs-vat-worker/test/transcript.txt b/packages/xs-vat-worker/test/transcript.txt new file mode 100644 index 00000000000..4f6443500d8 --- /dev/null +++ b/packages/xs-vat-worker/test/transcript.txt @@ -0,0 +1,6 @@ +v1.t.0 :: {"d":["deliver","o+0","zero",{"body":"[{\"@qclass\":\"slot\",\"index\":0},{\"@qclass\":\"slot\",\"index\":1},{\"@qclass\":\"slot\",\"index\":2}]","slots":["o-50","p-60","p-61"]},"p-62"],"syscalls":[{"d":["subscribe","p-60"],"response":null},{"d":["subscribe","p-61"],"response":null},{"d":["send","o-50","callback",{"body":"[11,12]","slots":[]},"p+5"],"response":null},{"d":["subscribe","p+5"],"response":null},{"d":["fulfillToData","p-62",{"body":"[{\"@qclass\":\"slot\",\"index\":0},{\"@qclass\":\"slot\",\"index\":1}]","slots":["p+6","p+7"]}],"response":null}],"crankNumber":2} +v1.t.1 :: {"d":["deliver","o+0","one",{"body":"[]","slots":[]},"p-63"],"syscalls":[{"d":["fulfillToPresence","p+6","o-50"],"response":null},{"d":["reject","p+7",{"body":"{\"@qclass\":\"error\",\"name\":\"Error\",\"message\":\"oops\"}","slots":[]}],"response":null},{"d":["fulfillToData","p-63",{"body":"1","slots":[]}],"response":null}],"crankNumber":3} +v1.t.2 :: {"d":["notifyFulfillToPresence","p-60","o-50"],"syscalls":[],"crankNumber":4} +v1.t.3 :: {"d":["notifyReject","p-61",{"body":"{\"@qclass\":\"error\",\"name\":\"Error\",\"message\":\"four\"}","slots":[]}],"syscalls":[],"crankNumber":5} +v1.t.4 :: {"d":["notifyFulfillToData","p+5",{"body":"[\"data\",{\"@qclass\":\"slot\",\"index\":0}]","slots":["o-50"]}],"syscalls":[],"crankNumber":9} +v1.t.nextID :: 5 diff --git a/packages/xs-vat-worker/test/vat-target.js b/packages/xs-vat-worker/test/vat-target.js new file mode 100644 index 00000000000..a8ceee3b723 --- /dev/null +++ b/packages/xs-vat-worker/test/vat-target.js @@ -0,0 +1,55 @@ +import { E } from '@agoric/eventual-send'; +import { producePromise } from '@agoric/produce-promise'; + +function ignore(p) { + p.then( + () => 0, + () => 0, + ); +} + +// We arrange for this vat, 'vat-target', to receive a specific set of +// inbound events ('dispatch'), which will provoke a set of outbound events +// ('syscall'), that cover the full range of the dispatch/syscall interface + +export function buildRootObject() { + const precB = producePromise(); + const precC = producePromise(); + let callbackObj; + + // zero: dispatch.deliver(target, method="one", result=pA, args=[callbackObj, pD, pE]) + // syscall.subscribe(pD) + // syscall.subscribe(pE) + // syscall.send(callbackObj, method="callback", result=rp2, args=[11, 12]); + // syscall.subscribe(rp2) + // syscall.fulfillToData(pA, [pB, pC]); + function zero(obj, pD, pE) { + callbackObj = obj; + E(callbackObj).callback(11, 12); // syscall.send + ignore(pD); + ignore(pE); + return [precB.promise, precC.promise]; // syscall.fulfillToData + } + + // one: dispatch.deliver(target, method="two", result=rp3, args=[]) + // syscall.fulfillToPresence(pB, callbackObj) + // syscall.reject(pC, Error('oops')) + // syscall.fulfillToData(rp3, 1) + function one() { + precB.resolve(callbackObj); // syscall.fulfillToPresence + precC.reject(Error('oops')); // syscall.reject + return 1; + } + + // two: dispatch.notifyFulfillToPresence(pD, callbackObj) + // three: dispatch.notifyReject(pE, Error('four')) + + // four: dispatch.notifyFulfillToData(pF, ['data', callbackObj]) + + const target = harden({ + zero, + one, + }); + + return target; +} diff --git a/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js b/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js new file mode 100644 index 00000000000..a218e5a39ef --- /dev/null +++ b/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js @@ -0,0 +1,17 @@ +/* global globalThis, Compartment */ + +function harden(x) { + return Object.freeze(x, true); +} + +harden(harden); + +globalThis.harden = harden; + +function tweakCompartmentAPI(C) { + return function Compartment(endowments, cmap, _options) { + return new C({ harden, ...endowments }, { '*': cmap }); + }; +} + +globalThis.Compartment = tweakCompartmentAPI(Compartment); From 903027a30299e9d9b03246bb0476bc4b94fddcf9 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 3 Aug 2020 18:54:28 -0500 Subject: [PATCH 02/20] feat: use npm style imports with XS build tools via compartmap - tools/findmods.js uses npm tools to walk source tree and build compartmap with endo-style inter-package import info plus intra-package info - for now, we check in the generated compartmap.json - compartmap.json also includes a "modules" key, which is used by reference from the XS manifest.json - start-xs sets up SES compartments with harden etc. - note harden SES issue 104 - src/endo-load.js does endo-style compartment-per-package loading solve xs compartmap constraints: - remove .js from "modules" section of xs manifest - remove .js when building compartment map from endo manifest Note as of Jun 5 62c7f1e xs's Compartment constructor needed tweaking, but no longer as of Aug 12 4842c4. --- packages/xs-vat-worker/bin/node-vat-worker | 2 +- packages/xs-vat-worker/compartmap.json | 152 +++++++++++++++ packages/xs-vat-worker/manifest.json | 79 ++++---- packages/xs-vat-worker/package.json | 2 + packages/xs-vat-worker/src/console.js | 5 +- packages/xs-vat-worker/src/endo-load.js | 54 ++++++ packages/xs-vat-worker/src/harden.js | 7 + packages/xs-vat-worker/src/main.js | 26 --- .../src/{index.js => start-node.js} | 0 packages/xs-vat-worker/src/timer-ticks.js | 17 -- packages/xs-vat-worker/start-xs.js | 42 +++++ .../xs-vat-worker/test/kernelSimulator.js | 6 +- packages/xs-vat-worker/tools/findmods.js | 175 ++++++++++++++++++ .../@agoric/install-ses/install-ses.js | 17 -- 14 files changed, 474 insertions(+), 110 deletions(-) create mode 100644 packages/xs-vat-worker/compartmap.json create mode 100644 packages/xs-vat-worker/src/endo-load.js create mode 100644 packages/xs-vat-worker/src/harden.js delete mode 100644 packages/xs-vat-worker/src/main.js rename packages/xs-vat-worker/src/{index.js => start-node.js} (100%) delete mode 100644 packages/xs-vat-worker/src/timer-ticks.js create mode 100644 packages/xs-vat-worker/start-xs.js create mode 100644 packages/xs-vat-worker/tools/findmods.js delete mode 100644 packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js diff --git a/packages/xs-vat-worker/bin/node-vat-worker b/packages/xs-vat-worker/bin/node-vat-worker index 9461d5dbf47..4d16a05043a 100755 --- a/packages/xs-vat-worker/bin/node-vat-worker +++ b/packages/xs-vat-worker/bin/node-vat-worker @@ -1,2 +1,2 @@ #!/bin/sh -node -r esm ./src/index.js +node -r esm ./src/start-node.js diff --git a/packages/xs-vat-worker/compartmap.json b/packages/xs-vat-worker/compartmap.json new file mode 100644 index 00000000000..08336b1d9ba --- /dev/null +++ b/packages/xs-vat-worker/compartmap.json @@ -0,0 +1,152 @@ +{ + "main": "packages/xs-vat-worker/", + "compartments": { + "packages/xs-vat-worker/": { + "label": "@agoric/xs-vat-worker@0.1.0", + "location": "packages/xs-vat-worker/", + "contents": [ + "./src/vatWorker.js" + ], + "modules": { + "@agoric/import-bundle": { + "compartment": "node_modules/@agoric/import-bundle/", + "module": "./src/index.js" + }, + "@agoric/eventual-send": { + "compartment": "node_modules/@agoric/eventual-send/", + "module": "./src/index.js" + }, + "@agoric/marshal": { + "compartment": "node_modules/@agoric/marshal/", + "module": "./marshal.js" + }, + "@agoric/swingset-vat/src/kernel/liveSlots": { + "compartment": "node_modules/@agoric/swingset-vat/", + "module": "./src/kernel/liveSlots.js" + } + } + }, + "node_modules/@agoric/import-bundle/": { + "label": "@agoric/import-bundle@0.0.8", + "location": "node_modules/@agoric/import-bundle/", + "contents": [ + "./src/index.js", + "./src/compartment-wrapper.js" + ], + "modules": {} + }, + "node_modules/@agoric/eventual-send/": { + "label": "@agoric/eventual-send@0.9.3", + "location": "node_modules/@agoric/eventual-send/", + "contents": [ + "./src/index.js", + "./src/E.js" + ], + "modules": {} + }, + "node_modules/@agoric/marshal/": { + "label": "@agoric/marshal@0.2.3", + "location": "node_modules/@agoric/marshal/", + "contents": [ + "./marshal.js" + ], + "modules": { + "@agoric/nat": { + "compartment": "node_modules/@agoric/nat/", + "module": "./dist/nat.esm.js" + }, + "@agoric/promise-kit": { + "compartment": "node_modules/@agoric/promise-kit/", + "module": "./src/promiseKit.js" + } + } + }, + "node_modules/@agoric/swingset-vat/": { + "label": "@agoric/swingset-vat@0.6.0", + "location": "node_modules/@agoric/swingset-vat/", + "contents": [ + "./src/kernel/liveSlots.js", + "./src/parseVatSlots.js", + "./src/capdata.js" + ], + "modules": { + "@agoric/eventual-send": { + "compartment": "node_modules/@agoric/eventual-send/", + "module": "./src/index.js" + }, + "@agoric/marshal": { + "compartment": "node_modules/@agoric/marshal/", + "module": "./marshal.js" + }, + "@agoric/assert": { + "compartment": "node_modules/@agoric/assert/", + "module": "./src/assert.js" + }, + "@agoric/promise-kit": { + "compartment": "node_modules/@agoric/promise-kit/", + "module": "./src/promiseKit.js" + }, + "@agoric/nat": { + "compartment": "node_modules/@agoric/nat/", + "module": "./dist/nat.esm.js" + } + } + }, + "node_modules/@agoric/nat/": { + "label": "@agoric/nat@2.0.1", + "location": "node_modules/@agoric/nat/", + "contents": [ + "./dist/nat.esm.js" + ], + "modules": {} + }, + "node_modules/@agoric/promise-kit/": { + "label": "@agoric/promise-kit@0.1.3", + "location": "node_modules/@agoric/promise-kit/", + "contents": [ + "./src/promiseKit.js" + ], + "modules": { + "@agoric/eventual-send": { + "compartment": "node_modules/@agoric/eventual-send/", + "module": "./src/index.js" + } + } + }, + "node_modules/@agoric/assert/": { + "label": "@agoric/assert@0.0.8", + "location": "node_modules/@agoric/assert/", + "contents": [ + "./src/assert.js", + "./src/types.js" + ], + "modules": {} + } + }, + "modules": { + "node_modules/0_MKDIR": "$(ROOT)/node_modules/0_MKDIR", + "node_modules/@agoric/0_MKDIR": "$(ROOT)/node_modules/@agoric/0_MKDIR", + "node_modules/@agoric/assert/0_MKDIR": "$(ROOT)/node_modules/@agoric/assert/0_MKDIR", + "node_modules/@agoric/assert/src/assert": "$(ROOT)/node_modules/@agoric/assert/src/assert", + "node_modules/@agoric/assert/src/types": "$(ROOT)/node_modules/@agoric/assert/src/types", + "node_modules/@agoric/eventual-send/0_MKDIR": "$(ROOT)/node_modules/@agoric/eventual-send/0_MKDIR", + "node_modules/@agoric/eventual-send/src/E": "$(ROOT)/node_modules/@agoric/eventual-send/src/E", + "node_modules/@agoric/eventual-send/src/index": "$(ROOT)/node_modules/@agoric/eventual-send/src/index", + "node_modules/@agoric/import-bundle/0_MKDIR": "$(ROOT)/node_modules/@agoric/import-bundle/0_MKDIR", + "node_modules/@agoric/import-bundle/src/compartment-wrapper": "$(ROOT)/node_modules/@agoric/import-bundle/src/compartment-wrapper", + "node_modules/@agoric/import-bundle/src/index": "$(ROOT)/node_modules/@agoric/import-bundle/src/index", + "node_modules/@agoric/marshal/marshal": "$(ROOT)/node_modules/@agoric/marshal/marshal", + "node_modules/@agoric/nat/0_MKDIR": "$(ROOT)/node_modules/@agoric/nat/0_MKDIR", + "node_modules/@agoric/nat/dist/nat.esm": "$(ROOT)/node_modules/@agoric/nat/dist/nat.esm", + "node_modules/@agoric/promise-kit/0_MKDIR": "$(ROOT)/node_modules/@agoric/promise-kit/0_MKDIR", + "node_modules/@agoric/promise-kit/src/promiseKit": "$(ROOT)/node_modules/@agoric/promise-kit/src/promiseKit", + "node_modules/@agoric/swingset-vat/0_MKDIR": "$(ROOT)/node_modules/@agoric/swingset-vat/0_MKDIR", + "node_modules/@agoric/swingset-vat/src/0_MKDIR": "$(ROOT)/node_modules/@agoric/swingset-vat/src/0_MKDIR", + "node_modules/@agoric/swingset-vat/src/capdata": "$(ROOT)/node_modules/@agoric/swingset-vat/src/capdata", + "node_modules/@agoric/swingset-vat/src/kernel/liveSlots": "$(ROOT)/node_modules/@agoric/swingset-vat/src/kernel/liveSlots", + "node_modules/@agoric/swingset-vat/src/parseVatSlots": "$(ROOT)/node_modules/@agoric/swingset-vat/src/parseVatSlots", + "packages/0_MKDIR": "$(ROOT)/packages/0_MKDIR", + "packages/xs-vat-worker/0_MKDIR": "$(ROOT)/packages/xs-vat-worker/0_MKDIR", + "packages/xs-vat-worker/src/vatWorker": "$(ROOT)/packages/xs-vat-worker/src/vatWorker" + } +} \ No newline at end of file diff --git a/packages/xs-vat-worker/manifest.json b/packages/xs-vat-worker/manifest.json index 411a331daf3..1254b0276c3 100644 --- a/packages/xs-vat-worker/manifest.json +++ b/packages/xs-vat-worker/manifest.json @@ -1,48 +1,39 @@ { - "include": [ - "$(MODDABLE)/examples/manifest_base.json" + "build": { + "BUILD": "$(MODDABLE)/build", + "MODULES": "$(MODDABLE)/modules" + }, + "creation": { + "keys": { + "available": 4096 + }, + "stack": 4096, + "parser": { + "buffer": 32768 + } + }, + "strip": [], + "modules": { + "@moddable/files": [ + "$(MODULES)/base/instrumentation/*", + "$(MODULES)/files/file/*", + "$(MODULES)/files/file/lin/*" ], - "$note": [ - "horrible @agoric/E KLUDGE below", - "liveSlots is a bit KLUDGy too" + "@moddable/Resource": [ "$(MODULES)/files/resource/Resource" ], + "@moddable/timer": [ + "$(MODULES)/base/timer/timer", + "$(MODULES)/base/timer/lin/*" ], - "creation": { - "keys": { - "available": 4096 - }, - "stack": 4096, - "parser": { - "buffer": 32768 - } - }, - "strip": [], - "modules": { - "files": [ "$(MODULES)/files/file/*", "$(MODULES)/files/file/lin/*" ], - "fdchan": [ "./fdchan" ], - "timer": [ - "$(MODULES)/base/timer/timer", - "$(MODULES)/base/timer/lin/*", - ], - "@agoric/nat": "xs_modules/@agoric/nat/nat.esm", - "@agoric/install-ses": "xs_modules/@agoric/install-ses/install-ses", - "@agoric/import-bundle": "xs_modules/@agoric/import-bundle/index", - "@agoric/compartment-wrapper": "xs_modules/@agoric/import-bundle/compartment-wrapper", - "@agoric/eventual-send": [ - "xs_modules/@agoric/eventual-send/index" - ], - "@agoric/E": "xs_modules/@agoric/eventual-send/E", - "@agoric/marshal": "xs_modules/@agoric/marshal/marshal", - "@agoric/produce-promise": "xs_modules/@agoric/produce-promise/producePromise", - "@agoric/assert": "xs_modules/@agoric/assert/assert", - "@agoric/types": "xs_modules/@agoric/assert/types", - "swingset/capdata": "./swingset/capdata", - "swingset/parseVatSlots": "./swingset/parseVatSlots", - "swingset/kernel/liveSlots": "./swingset/kernel/liveSlots", - "*": [ - "./main", - "./console", - "./timer-ticks", - "./vatWorker" - ] - } + + "main": "./start-xs", + "src/console": "./src/console", + "src/harden": "./src/harden", + "src-native/fdchan": [ "./src-native/fdchan" ], + "src/endo-load": "./src/endo-load", + "src/vatWorker": "./src/vatWorker" + }, + "data": { + "compartmap": "./compartmap" + }, + "include": [ "./compartmap.json" ] } diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 66624812561..42765924843 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -18,12 +18,14 @@ "lint-check-jessie": "eslint -c '.eslintrc-jessie.js' '**/*.{js,jsx}'" }, "devDependencies": { + "detective-es6": "^2.2.0", "eslint": "^6.1.0", "eslint-config-airbnb-base": "^14.0.0", "eslint-config-jessie": "0.0.3", "eslint-config-prettier": "^6.0.0", "eslint-plugin-import": "^2.18.2", "eslint-plugin-prettier": "^3.1.0", + "filing-cabinet": "^2.5.1", "prettier": "^1.18.2", "tap-spec": "^5.0.0", "tape": "^4.11.0", diff --git a/packages/xs-vat-worker/src/console.js b/packages/xs-vat-worker/src/console.js index ac8ba1bf457..da6d9b29211 100644 --- a/packages/xs-vat-worker/src/console.js +++ b/packages/xs-vat-worker/src/console.js @@ -1,12 +1,11 @@ /** console for xs platform */ -/* global trace, globalThis */ const harden = x => Object.freeze(x, true); const text = it => (typeof it === 'object' ? JSON.stringify(it) : `${it}`); const combine = (...things) => `${things.map(text).join(' ')}\n`; export function makeConsole(write_) { - const write = write_ || trace; // note ocap exception for tracing / logging + const write = write_; return harden({ log(...things) { write(combine(...things)); @@ -21,5 +20,3 @@ export function makeConsole(write_) { }, }); } - -globalThis.console = makeConsole(); diff --git a/packages/xs-vat-worker/src/endo-load.js b/packages/xs-vat-worker/src/endo-load.js new file mode 100644 index 00000000000..119f4776fba --- /dev/null +++ b/packages/xs-vat-worker/src/endo-load.js @@ -0,0 +1,54 @@ +/* global Compartment */ + +function debug(..._args) { + // console.log(...args); +} + +function SESCompartment(endowments, map, options) { + debug('SESCompartment', { endowments, map, options }); + const sesGlobals = { harden, console, Compartment: SESCompartment }; + return new Compartment({ ...sesGlobals, ...endowments }, map, options); +} + +export function loadMain(compartmap) { + // ISSUE: doesn't seem to work: const { entries, fromEntries, keys } = Object; + // debug('entries, ...', { entries: typeof entries, fromEntries: typeof fromEntries, keys: typeof keys }); + const entries = o => Object.entries(o); + const fromEntries = pvs => Object.fromEntries(pvs); + const keys = o => Object.keys(o); + + const memoize = f => { + const cache = {}; + return k => cache[k] || (cache[k] = f(k)); + }; + const unjs = spec => spec.replace(/\.js$/, ''); + const unrel = spec => spec.replace(/^\.\//, '/'); + const join = (base, ref) => `${base}${unjs(ref.slice(2))}`; + const pkgCompartment = memoize(loc => { + const intraPkg = ref => { + debug('intraPkg', { loc, ref }); + return [unjs(unrel(ref).slice(1)), join(loc, ref)]; + }; + function interPkg([specifier, { compartment, module }]) { + const pc = pkgCompartment(compartment); + const fullSpecifier = unjs(module.slice(2)); + // const fullSpecifier = join(compartment, module); + debug('interPkg', { + loc, + specifier, + compartment, + module, + fullSpecifier, + }); + return [specifier, pc.importNow(fullSpecifier)]; + } + const { contents, modules } = compartmap.compartments[loc]; + const cmap = fromEntries([ + ...contents.map(intraPkg), + ...entries(modules).map(interPkg), + ]); + debug({ loc, contents, modules: keys(modules), map: cmap }); + return new SESCompartment({}, cmap); + }); + return pkgCompartment(compartmap.main); +} diff --git a/packages/xs-vat-worker/src/harden.js b/packages/xs-vat-worker/src/harden.js new file mode 100644 index 00000000000..8a74c0ef5c6 --- /dev/null +++ b/packages/xs-vat-worker/src/harden.js @@ -0,0 +1,7 @@ +// ISSUE: harden compat? https://github.com/Agoric/SES-shim/issues/104 +export function harden(x) { + return Object.freeze(x); +} + +harden(harden); +globalThis.harden = harden; diff --git a/packages/xs-vat-worker/src/main.js b/packages/xs-vat-worker/src/main.js deleted file mode 100644 index eb03ae228d3..00000000000 --- a/packages/xs-vat-worker/src/main.js +++ /dev/null @@ -1,26 +0,0 @@ -// add harden; align xs's Compartment with Agoric's -// ISSUE: use a different module name? -import '@agoric/install-ses'; - -import './console'; // sets globalThis.console. ew. -import './timer-ticks'; // globalThis.setTimeout. ew. - -import { main as vatWorker } from './vatWorker'; -import { Reader, Writer } from './fdchan'; - -const INFD = 3; -const OUTFD = 4; - -export default async function main() { - const inStream = new Reader(INFD); - const outStream = new Writer(OUTFD); - - return vatWorker({ - setImmediate, - readMessage: () => inStream.read_netstring(), - writeMessage: message => { - // ISSUE: should be byte length - outStream.write(`${message.length}:`, message, ','); - }, - }); -} diff --git a/packages/xs-vat-worker/src/index.js b/packages/xs-vat-worker/src/start-node.js similarity index 100% rename from packages/xs-vat-worker/src/index.js rename to packages/xs-vat-worker/src/start-node.js diff --git a/packages/xs-vat-worker/src/timer-ticks.js b/packages/xs-vat-worker/src/timer-ticks.js deleted file mode 100644 index 39c96d0580e..00000000000 --- a/packages/xs-vat-worker/src/timer-ticks.js +++ /dev/null @@ -1,17 +0,0 @@ -// ref moddable/examples/base/timers/main.js -/* global globalThis */ - -// eslint-disable-next-line import/no-unresolved -import Timer from 'timer'; // moddable timer - -globalThis.setImmediate = callback => { - Timer.set(callback); -}; - -globalThis.setTimeout = (callback, delay) => { - Timer.set(callback, delay); -}; - -globalThis.setInterval = (callback, delay) => { - Timer.repeat(callback, delay); -}; diff --git a/packages/xs-vat-worker/start-xs.js b/packages/xs-vat-worker/start-xs.js new file mode 100644 index 00000000000..4d80d226364 --- /dev/null +++ b/packages/xs-vat-worker/start-xs.js @@ -0,0 +1,42 @@ +/* global trace */ + +// eslint-disable-next-line import/no-unresolved +import Timer from '@moddable/timer'; +// eslint-disable-next-line import/no-unresolved +import Resource from '@moddable/Resource'; + +import { harden } from './src/harden'; +import { makeConsole } from './src/console'; +import { loadMain } from './src/endo-load'; + +// eslint-disable-next-line import/named +import { Reader, Writer } from './src-native/fdchan'; + +const INFD = 3; +const OUTFD = 4; + +globalThis.console = harden(makeConsole(trace)); + +const compartmapROM = new Resource('compartmap.json'); +const compartmap0 = JSON.parse(String.fromArrayBuffer(compartmapROM.slice(0))); + +export default async function main() { + const inStream = new Reader(INFD); + const outStream = new Writer(OUTFD); + + const c1 = loadMain(compartmap0); + console.log('about to import vatWorker'); + const vw = await c1.import('src/vatWorker'); + + console.log('about to run vatWorker.main...'); + return vw.main({ + setImmediate: callback => { + Timer.set(callback); + }, + readMessage: () => inStream.read_netstring(), + writeMessage: message => { + // ISSUE: should be byte length + outStream.write(`${message.length}:`, message, ','); + }, + }); +} diff --git a/packages/xs-vat-worker/test/kernelSimulator.js b/packages/xs-vat-worker/test/kernelSimulator.js index 4bb07c6c05b..faaa31a762a 100644 --- a/packages/xs-vat-worker/test/kernelSimulator.js +++ b/packages/xs-vat-worker/test/kernelSimulator.js @@ -31,7 +31,11 @@ function makeWorker(child) { throw badJSON; } if (msg.msgtype !== msgtype) { - throw new Error(`expected ${msgtype}; found: ${msg.msgtype}; error: ${msg.error} [${JSON.stringify(msg)}]`); + throw new Error( + `expected ${msgtype}; found: ${msg.msgtype}; error: ${ + msg.error + } [${JSON.stringify(msg)}]`, + ); } return msg; }; diff --git a/packages/xs-vat-worker/tools/findmods.js b/packages/xs-vat-worker/tools/findmods.js new file mode 100644 index 00000000000..484666b32d3 --- /dev/null +++ b/packages/xs-vat-worker/tools/findmods.js @@ -0,0 +1,175 @@ +/* eslint-disable no-redeclare */ +/* eslint-disable no-continue */ +/* eslint-disable no-await-in-loop */ +import { URL } from 'url'; + +// eslint-disable-next-line import/no-extraneous-dependencies +import detective from 'detective-es6'; + +const USAGE = `findmods WORKSPACE FILE EXCLUDE`; + +const resolve = (ref, base) => new URL(ref, base).toString(); + +function containingPackage( + loc, + { readFile, fileURL }, + sentinel = 'package.json', +) { + async function recur(here, step) { + const parent = resolve(step, here); + + if (parent === here && step === '..') { + throw new Error(`no containing package: ${loc}`); + } + + const descriptorPath = fileURL.toPath(resolve(sentinel, parent)); + return readFile(descriptorPath, 'utf8') + .then(txt => { + const { name, version = '' } = JSON.parse(txt); + return { location: parent, name, version, label: `${name}@${version}` }; + }) + .catch(err => { + if (['ENOTDIR', 'ENOENT'].includes(err.code)) { + return recur(parent, '..'); + } + throw err; + }); + } + + return recur(loc, './'); +} + +function relativeTo(target, base) { + if (!base.endsWith('/')) { + base += '/'; + } + + if (target.startsWith(base)) { + return target.slice(base.length); + } + return target; +} + +async function walk(workspace, start, exclude, { fsp, cabinet, fileURL }) { + const pkgOf = u => containingPackage(u, { readFile: fsp.readFile, fileURL }); + const u2p = u => fileURL.toPath(u); + const p2u = p => fileURL.fromPath(p).toString(); + const inW = u => relativeTo(u, workspace); + const unjs = p => p.replace(/\.js$/, ''); + + const todo = [start]; + const seen = new Set(); + const compartments = {}; + + while (todo.length > 0) { + const importer = todo.shift(); + if (seen.has(importer)) { + continue; + } + seen.add(importer); + + const srcPkg = await pkgOf(importer); + const pkgName = inW(srcPkg.location); + const compartment = + compartments[pkgName] || + (compartments[pkgName] = { + label: srcPkg.label, + location: pkgName, + contents: [], + modules: {}, + }); + const contents = compartment.contents; + const modules = compartment.modules; + contents.push(`./${relativeTo(importer, srcPkg.location)}`); + + const src = await fsp.readFile(u2p(importer), 'utf8'); + + const dependencies = detective(src); + // console.log({ importer, srcLength: src.length, dependencies }); + // console.log({ src: src.slice(0, 30) }); + for (const specifier of dependencies) { + if (exclude && specifier.match(exclude)) { + // console.log('excluded:', specifier, ' by:', exclude); + modules[specifier] = { exclude }; + continue; + } + + const target = cabinet({ + partial: specifier, + directory: u2p(workspace), + filename: u2p(importer), + nodeModulesConfig: { entry: 'module' }, + }); + // console.log({ specifier, target }); + if (target.length > 0) { + const imported = p2u(target); + todo.push(imported); + + const destPkg = await pkgOf(imported); + const ref = relativeTo(imported, destPkg.location); + if (destPkg.location !== srcPkg.location) { + modules[specifier] = { + compartment: inW(destPkg.location), + module: `./${ref}`, + }; + } + } + } + } + + const ancestors = spec => + spec + .split('/') + .slice(0, -2) + .reduce( + ([p, ps], seg) => [`${p}/${seg}`, [...ps, `${p}/${seg}`.slice(1)]], + ['', []], + )[1]; + const modules = Object.fromEntries( + Array.from(seen) + .map(m => inW(m)) + .map(m => [m, ...ancestors(m).map(d => `${d}/0_MKDIR`)]) + .flat() + .map(m => [unjs(m), `$(ROOT)/${unjs(m)}`]) + .sort(), + ); + const mainPkg = await pkgOf(start); + return { + main: inW(mainPkg.location), + compartments, + modules, + }; +} + +async function main(argv, { stdout, fsp, fileURL, cabinet }) { + const [workspace, start, exclude] = argv.slice(2); + if (!workspace || !start) { + throw USAGE; + } + + const p2u = p => fileURL.fromPath(p).toString(); + const info = await walk(p2u(workspace), p2u(start), exclude, { + fsp, + cabinet, + fileURL, + }); + + stdout.write(JSON.stringify(info, null, 2)); +} + +/* global require, module, process */ +if (require.main === module) { + main(process.argv, { + stdout: process.stdout, + // eslint-disable-next-line global-require + fsp: require('fs').promises, + fileURL: { + // eslint-disable-next-line global-require + fromPath: require('url').pathToFileURL, + // eslint-disable-next-line global-require + toPath: require('url').fileURLToPath, + }, + // eslint-disable-next-line + cabinet: require('filing-cabinet'), + }).catch(err => console.error('main:', err)); +} diff --git a/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js b/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js deleted file mode 100644 index a218e5a39ef..00000000000 --- a/packages/xs-vat-worker/xs_modules/@agoric/install-ses/install-ses.js +++ /dev/null @@ -1,17 +0,0 @@ -/* global globalThis, Compartment */ - -function harden(x) { - return Object.freeze(x, true); -} - -harden(harden); - -globalThis.harden = harden; - -function tweakCompartmentAPI(C) { - return function Compartment(endowments, cmap, _options) { - return new C({ harden, ...endowments }, { '*': cmap }); - }; -} - -globalThis.Compartment = tweakCompartmentAPI(Compartment); From f961bfa6c4f8e26d1d0779b043da7e9f3416c839 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 10 Aug 2020 20:47:57 -0500 Subject: [PATCH 03/20] test: avoid SIGSEGV; promise-kit bundle is 28K, not 200K catching up with agoric-sdk's use of promise-kit in place of produce-promise eliminated a large swath of dependent code. SIGSEGV came from some interaction of Compartments and @agoric/make-hardener ? --- packages/xs-vat-worker/package.json | 2 +- packages/xs-vat-worker/test/bootstrap.js | 2 +- packages/xs-vat-worker/test/vat-target.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 42765924843..2a6fca29517 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -37,7 +37,7 @@ "@agoric/eventual-send": "^0.9.3", "@agoric/import-bundle": "^0.0.8", "@agoric/install-ses": "^0.2.0", - "@agoric/produce-promise": "^0.1.3", + "@agoric/promise-kit": "^0.1.3", "@agoric/swingset-vat": "^0.6.0", "anylogger": "^1.0.4", "esm": "^3.2.5" diff --git a/packages/xs-vat-worker/test/bootstrap.js b/packages/xs-vat-worker/test/bootstrap.js index 0b64ca2b269..5fa65d26237 100644 --- a/packages/xs-vat-worker/test/bootstrap.js +++ b/packages/xs-vat-worker/test/bootstrap.js @@ -1,5 +1,5 @@ import { E } from '@agoric/eventual-send'; -import { producePromise } from '@agoric/produce-promise'; +import { producePromise } from '@agoric/promise-kit'; function ignore(p) { p.then( diff --git a/packages/xs-vat-worker/test/vat-target.js b/packages/xs-vat-worker/test/vat-target.js index a8ceee3b723..5e63ccfbaaf 100644 --- a/packages/xs-vat-worker/test/vat-target.js +++ b/packages/xs-vat-worker/test/vat-target.js @@ -1,5 +1,5 @@ import { E } from '@agoric/eventual-send'; -import { producePromise } from '@agoric/produce-promise'; +import { makePromiseKit as producePromise } from '@agoric/promise-kit'; function ignore(p) { p.then( From b08b0edfdeb789f56819d59cc871138f2903ad28 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 12 Aug 2020 22:02:19 -0500 Subject: [PATCH 04/20] build(xs-vat-worker): build moddable SDK, xs vat worker - add moddable SDK git submodule - makefile stanza to document some git submodule usage - avoid much of x11 for moddable by using only libgio, not all of libgtk - ignore xs build dir - makefile stanza to document compartmap rebuilding --- .gitmodules | 4 ++++ packages/xs-vat-worker/.gitignore | 2 ++ packages/xs-vat-worker/moddable | 1 + packages/xs-vat-worker/package.json | 5 +--- packages/xs-vat-worker/xs-lin.mk | 36 +++++++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 .gitmodules create mode 100644 packages/xs-vat-worker/.gitignore create mode 160000 packages/xs-vat-worker/moddable create mode 100644 packages/xs-vat-worker/xs-lin.mk diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..d91b53755d5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "packages/xs-vat-worker/moddable"] + path = packages/xs-vat-worker/moddable + url = https://github.com/agoric-labs/moddable.git + branch = ag-linux-cli diff --git a/packages/xs-vat-worker/.gitignore b/packages/xs-vat-worker/.gitignore new file mode 100644 index 00000000000..3fe6843e951 --- /dev/null +++ b/packages/xs-vat-worker/.gitignore @@ -0,0 +1,2 @@ +.envrc +build/ diff --git a/packages/xs-vat-worker/moddable b/packages/xs-vat-worker/moddable new file mode 160000 index 00000000000..4dcdcffd92d --- /dev/null +++ b/packages/xs-vat-worker/moddable @@ -0,0 +1 @@ +Subproject commit 4dcdcffd92de733d40f91ad34c0c539ec98d331c diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 2a6fca29517..1360ab991f2 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -6,10 +6,7 @@ "module": "dist/vat-worker.esm.js", "browser": "dist/vat-worker.umd.js", "scripts": { - "xs-build": "mkdir -p build; mcconfig -o build -p x-cli-lin -m -d", - "xs-run": "yarn xs-build && ./build/bin/lin/debug/make-vat-transcript", - "xs-build-release": "mkdir -p build; mcconfig -o build -p x-cli-lin -m", - "xs-run-release": "yarn xs-build-release && ./build/bin/lin/release/make-vat-transcript", + "build:xs-lin": "make -f xs-lin.mk", "test": "tape -r esm 'test/**/test-*.js'", "build": "exit 0", "lint-fix": "eslint --fix '**/*.{js,jsx}'", diff --git a/packages/xs-vat-worker/xs-lin.mk b/packages/xs-vat-worker/xs-lin.mk new file mode 100644 index 00000000000..efb139354b2 --- /dev/null +++ b/packages/xs-vat-worker/xs-lin.mk @@ -0,0 +1,36 @@ +MODDABLE=$(PWD)/moddable +TOOLS=$(MODDABLE)/build/bin/lin/release/ + +build/bin/lin/debug/xs-vat-worker: build $(TOOLS)/mcconfig moddable/xs/platforms/lin_xs_cli.c compartmap.json manifest.json + ROOT=$(ROOT) PATH=$(TOOLS):$$PATH MODDABLE=$(MODDABLE) mcconfig -o build -p x-cli-lin -m -d + +compartmap.json: src/vatWorker.js package.json + node -r esm tools/findmods.js $(ROOT) src/vatWorker.js >$@ + +build: + mkdir -p build + +moddable/xs/platforms/lin_xs_cli.c: moddable/xs/platforms/lin_xs.h + touch $@ + +./moddable/README.md: + git submodule init + git submodule update + +moddable/xs/platforms/lin_xs.h: /usr/include/glib-2.0/gio/gio.h + +/usr/include/glib-2.0/gio/gio.h: + sudo apt-get -y update && sudo apt-get -y install libgio2.0-dev + +$(TOOLS)/mcconfig: + cd moddable && \ + export MODDABLE=$$PWD && echo MODDABLE=$$MODDABLE && \ + cd build/makefiles/lin && \ + make headless + +clean: + rm -rf build + +realclean: + rm -rf build + cd moddable/build/makefiles/lin && make clean From d67e47177f6354edfe2784a02a6adb04e8aebad0 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 12 Aug 2020 22:50:05 -0500 Subject: [PATCH 05/20] test(xs-vat-worker): locate xs worker binary (WIP) export, test --- packages/xs-vat-worker/package.json | 10 +++++----- packages/xs-vat-worker/src/locate-undefined.js | 1 + packages/xs-vat-worker/test/test-locate.js | 8 ++++++++ packages/xs-vat-worker/xs-lin.mk | 9 +++++++-- 4 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 packages/xs-vat-worker/src/locate-undefined.js create mode 100644 packages/xs-vat-worker/test/test-locate.js diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 1360ab991f2..45ce24dea4f 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -1,11 +1,11 @@ { "name": "@agoric/xs-vat-worker", - "version": "0.0.1", - "description": "???", - "main": "dist/vat-worker.cjs.js", - "module": "dist/vat-worker.esm.js", - "browser": "dist/vat-worker.umd.js", + "version": "0.1.0", + "description": "swingset-vat worker for XS js runtime", + "main": "src/locate.js", + "module": "src/locate.js", "scripts": { + "postinstall": "cp src/locate-undefined.js src/locate.js", "build:xs-lin": "make -f xs-lin.mk", "test": "tape -r esm 'test/**/test-*.js'", "build": "exit 0", diff --git a/packages/xs-vat-worker/src/locate-undefined.js b/packages/xs-vat-worker/src/locate-undefined.js new file mode 100644 index 00000000000..1ba903d1b52 --- /dev/null +++ b/packages/xs-vat-worker/src/locate-undefined.js @@ -0,0 +1 @@ +export const xsWorkerBin = undefined; diff --git a/packages/xs-vat-worker/test/test-locate.js b/packages/xs-vat-worker/test/test-locate.js new file mode 100644 index 00000000000..82e7182b06b --- /dev/null +++ b/packages/xs-vat-worker/test/test-locate.js @@ -0,0 +1,8 @@ +import { test } from 'tape-promise/tape'; + +import { xsWorkerBin } from '../src/locate'; + +test('locateWorkerBin', t => { + t.ok(!xsWorkerBin || xsWorkerBin.endsWith('xs-vat-worker')); + t.end(); +}); diff --git a/packages/xs-vat-worker/xs-lin.mk b/packages/xs-vat-worker/xs-lin.mk index efb139354b2..6bc4f185896 100644 --- a/packages/xs-vat-worker/xs-lin.mk +++ b/packages/xs-vat-worker/xs-lin.mk @@ -1,5 +1,11 @@ MODDABLE=$(PWD)/moddable TOOLS=$(MODDABLE)/build/bin/lin/release/ +NODE_MODULES=$(PWD)/node_modules +ROOT=$(PWD)/../.. + +$(NODE_MODULES)/.bin/xs-vat-worker: build/bin/lin/debug/xs-vat-worker + cp $< $@ + echo "export const xsWorkerBin =\n '$@';" >src/locate.js build/bin/lin/debug/xs-vat-worker: build $(TOOLS)/mcconfig moddable/xs/platforms/lin_xs_cli.c compartmap.json manifest.json ROOT=$(ROOT) PATH=$(TOOLS):$$PATH MODDABLE=$(MODDABLE) mcconfig -o build -p x-cli-lin -m -d @@ -11,7 +17,6 @@ build: mkdir -p build moddable/xs/platforms/lin_xs_cli.c: moddable/xs/platforms/lin_xs.h - touch $@ ./moddable/README.md: git submodule init @@ -29,7 +34,7 @@ $(TOOLS)/mcconfig: make headless clean: - rm -rf build + rm -rf build $(NODE_MODULES)/.bin/xs-vat-worker realclean: rm -rf build From 5313fdf2a83ae7493f65fa17fc873fe7c391ca65 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 12 Aug 2020 22:51:34 -0500 Subject: [PATCH 06/20] style(xs-vat-worker): ignore build, moddable, bundles in lint --- packages/xs-vat-worker/.eslintignore | 5 +++++ packages/xs-vat-worker/.eslintrc.js | 1 + 2 files changed, 6 insertions(+) diff --git a/packages/xs-vat-worker/.eslintignore b/packages/xs-vat-worker/.eslintignore index ea65306b6ba..5c0aef66585 100644 --- a/packages/xs-vat-worker/.eslintignore +++ b/packages/xs-vat-worker/.eslintignore @@ -2,3 +2,8 @@ /scripts/ /xs_modules/ /swingset/ +/moddable/ +build/ +bundle-*.js +test/bug1/ +src-native/ diff --git a/packages/xs-vat-worker/.eslintrc.js b/packages/xs-vat-worker/.eslintrc.js index 3f42e1d90c7..23d02400696 100644 --- a/packages/xs-vat-worker/.eslintrc.js +++ b/packages/xs-vat-worker/.eslintrc.js @@ -7,6 +7,7 @@ module.exports = { }, globals: { "harden": "readonly", + "globalThis": "writeable", }, rules: { 'implicit-arrow-linebreak': 'off', From 2db022d966a416c9b765c18ed543dd5adb31cc6d Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 15 Aug 2020 13:53:23 -0500 Subject: [PATCH 07/20] feat(swingset-vat): add xs-worker managerType If any vats are configured to use it, but xs-vat-worker does not have an executable to offer, swingset will throw an error. (IOU a test?) - Swingset package depends on xs-vat-worker - ISSUE: cycle. factor out liveslots? - add startXsWorker kernel endowment, which is undefined unless binary is built - parameterize execPath, args in startSubprocessWorker() (mantain previous defaults) - xs-vat-worker: adopt array style protocol from SwingSet (WIP) - based on subprocessSupervisor.js copied SwingSet/src/kernel/vatManager/subprocessSupervisor.js wholesale; applied ocap discipline to module-level mutable state; adapted IO interface. Compare: Initially it would hang after the first 2 syscalls of the first deliver: - [ 'subscribe', 'p-60' ] - [ 'subscribe', 'p-61' ] Then I fixed handle() to return a promise in all cases to be sure to finish delivery before next blocking read. --- packages/SwingSet/package.json | 1 + packages/SwingSet/src/controller.js | 6 + packages/SwingSet/src/kernel/kernel.js | 2 + .../SwingSet/src/kernel/vatManager/factory.js | 13 + .../SwingSet/src/spawnSubprocessWorker.js | 6 +- packages/SwingSet/test/workers/test-worker.js | 18 ++ packages/xs-vat-worker/package.json | 1 + packages/xs-vat-worker/src/vatWorker.js | 231 ++++++++++-------- 8 files changed, 179 insertions(+), 99 deletions(-) diff --git a/packages/SwingSet/package.json b/packages/SwingSet/package.json index 5640f0cbb63..aef9293b1f0 100644 --- a/packages/SwingSet/package.json +++ b/packages/SwingSet/package.json @@ -39,6 +39,7 @@ "@agoric/tame-metering": "^1.2.3", "@agoric/transform-eventual-send": "^1.3.1", "@agoric/transform-metering": "^1.3.0", + "@agoric/xs-vat-worker": "^0.1.0", "@babel/core": "^7.5.0", "@babel/generator": "^7.6.4", "anylogger": "^0.21.0", diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 0e3c955c115..1691abcea2c 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -16,6 +16,7 @@ import { importBundle } from '@agoric/import-bundle'; import { initSwingStore } from '@agoric/swing-store-simple'; import { makeMeteringTransformer } from '@agoric/transform-metering'; import { makeTransform } from '@agoric/transform-eventual-send'; +import { xsWorkerBin } from '@agoric/xs-vat-worker'; import { startSubprocessWorker } from './spawnSubprocessWorker'; import { assertKnownOptions } from './assertOptions'; @@ -321,6 +322,10 @@ export async function buildVatController( // console.log(`--slog ${JSON.stringify(obj)}`); } + const startXsWorker = xsWorkerBin + ? () => startSubprocessWorker(xsWorkerBin, []) + : undefined; + const kernelEndowments = { waitUntilQuiescent, hostStorage, @@ -333,6 +338,7 @@ export async function buildVatController( makeNodeWorker, startSubprocessWorker, writeSlogObject, + startXsWorker, }; const kernelOptions = { verbose }; diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 025f7c38e30..aefb2426879 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -52,6 +52,7 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) { makeNodeWorker, startSubprocessWorker, writeSlogObject, + startXsWorker, } = kernelEndowments; const { verbose } = kernelOptions; const logStartup = verbose ? console.debug : () => 0; @@ -613,6 +614,7 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) { waitUntilQuiescent, makeNodeWorker, startSubprocessWorker, + startXsWorker, }); function buildVatSyscallHandler(vatID, translators) { diff --git a/packages/SwingSet/src/kernel/vatManager/factory.js b/packages/SwingSet/src/kernel/vatManager/factory.js index f3f6706ed0d..6b85e83126a 100644 --- a/packages/SwingSet/src/kernel/vatManager/factory.js +++ b/packages/SwingSet/src/kernel/vatManager/factory.js @@ -14,6 +14,7 @@ export function makeVatManagerFactory({ waitUntilQuiescent, makeNodeWorker, startSubprocessWorker, + startXsWorker, }) { const localFactory = makeLocalVatManagerFactory({ allVatPowers, @@ -34,6 +35,11 @@ export function makeVatManagerFactory({ kernelKeeper, }); + const xsWorkerFactory = makeNodeSubprocessFactory({ + startSubprocessWorker: startXsWorker, + kernelKeeper, + }); + function validateManagerOptions(managerOptions) { assertKnownOptions(managerOptions, [ 'enablePipelining', @@ -93,6 +99,13 @@ export function makeVatManagerFactory({ ); } + if (managerType === 'xs-worker') { + if (!xsWorkerFactory) { + throw new Error('manager type xs-worker not available'); + } + return xsWorkerFactory.createFromBundle(vatID, bundle, managerOptions); + } + throw Error( `unknown type ${managerType}, not local/nodeWorker/node-subprocess`, ); diff --git a/packages/SwingSet/src/spawnSubprocessWorker.js b/packages/SwingSet/src/spawnSubprocessWorker.js index 057612825e8..ec7dfe0534b 100644 --- a/packages/SwingSet/src/spawnSubprocessWorker.js +++ b/packages/SwingSet/src/spawnSubprocessWorker.js @@ -20,8 +20,10 @@ const supercode = require.resolve( // always be Node. const stdio = harden(['inherit', 'inherit', 'inherit', 'pipe', 'pipe']); -export function startSubprocessWorker() { - const proc = spawn(process.execPath, ['-r', 'esm', supercode], { stdio }); +export function startSubprocessWorker(execPath, args) { + execPath = execPath || process.execPath; + args = args || ['-r', 'esm', supercode]; + const proc = spawn(execPath, args, { stdio }); const toChild = Netstring.writeStream(); toChild.pipe(proc.stdio[3]); diff --git a/packages/SwingSet/test/workers/test-worker.js b/packages/SwingSet/test/workers/test-worker.js index c4cd0d889bc..4d877490b72 100644 --- a/packages/SwingSet/test/workers/test-worker.js +++ b/packages/SwingSet/test/workers/test-worker.js @@ -1,7 +1,25 @@ import '@agoric/install-ses'; import test from 'ava'; +import { xsWorkerBin } from '@agoric/xs-vat-worker/src/locate'; import { loadBasedir, buildVatController } from '../../src/index'; +test('xs vat manager', async t => { + if (!xsWorkerBin) { + console.warn('XS vat worker not built; skipping'); + t.falsy.skip(false); + return; + } + + const config = await loadBasedir(__dirname); + config.vats.target.creationOptions = { managerType: 'xs-worker' }; + const c = await buildVatController(config, []); + + await c.run(); + t.is(c.bootstrapResult.status(), 'fulfilled'); + + await c.shutdown(); +}); + test('nodeWorker vat manager', async t => { const config = await loadBasedir(__dirname); config.vats.target.creationOptions = { managerType: 'nodeWorker' }; diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 45ce24dea4f..736d505a961 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -34,6 +34,7 @@ "@agoric/eventual-send": "^0.9.3", "@agoric/import-bundle": "^0.0.8", "@agoric/install-ses": "^0.2.0", + "@agoric/marshal": "^0.2.3", "@agoric/promise-kit": "^0.1.3", "@agoric/swingset-vat": "^0.6.0", "anylogger": "^1.0.4", diff --git a/packages/xs-vat-worker/src/vatWorker.js b/packages/xs-vat-worker/src/vatWorker.js index a7e741379eb..2a30509bc08 100644 --- a/packages/xs-vat-worker/src/vatWorker.js +++ b/packages/xs-vat-worker/src/vatWorker.js @@ -1,11 +1,19 @@ import { importBundle } from '@agoric/import-bundle'; import { HandledPromise } from '@agoric/eventual-send'; +import { Remotable, getInterfaceOf } from '@agoric/marshal'; // TODO? import anylogger from 'anylogger'; import { makeLiveSlots } from '@agoric/swingset-vat/src/kernel/liveSlots'; -const EOF = new Error('EOF'); +function workerLog(first, ...args) { + console.log(`---worker: ${first}`, ...args); +} + +function assert(ok, whynot) { + if (!ok) { + throw new Error(whynot); + } +} -// from SwingSet/src/controller.js function makeConsole(_tag) { const log = console; // TODO? anylogger(tag); const cons = {}; @@ -15,19 +23,9 @@ function makeConsole(_tag) { return harden(cons); } -function makeVatEndowments(consoleTag) { - return harden({ - console: makeConsole(`SwingSet:${consoleTag}`), - HandledPromise, - // TODO: re2 is a RegExp work-a-like that disables backtracking expressions for - // safer memory consumption - RegExp, - }); -} - // see also: detecting an empty vat promise queue (end of "crank") // https://github.com/Agoric/agoric-sdk/issues/45 -function endOfCrank(setImmediate) { +function waitUntilQuiescent(setImmediate) { return new Promise((resolve, _reject) => { setImmediate(() => { // console.log('hello from setImmediate callback. The promise queue is presumably empty.'); @@ -36,106 +34,145 @@ function endOfCrank(setImmediate) { }); } +function runAndWait(f, errmsg, setImmediate) { + Promise.resolve() + .then(f) + .then(undefined, err => workerLog(`doProcess: ${errmsg}:`, err)); + return waitUntilQuiescent(setImmediate); +} + function makeWorker(io, setImmediate) { - let vatNS = null; let dispatch; - let state; - - const format = msg => JSON.stringify(msg); - const sync = (method, args) => { - io.writeMessage(format({ msgtype: 'syscall', method, args })); - return JSON.parse(io.readMessage()); - }; - const syscall = harden({ - subscribe(...args) { - return sync('subscribe', args); - }, - send(...args) { - return sync('send', args); - }, - fulfillToData(...args) { - return sync('fulfillToData', args); - }, - fulfillToPresence(...args) { - return sync('fulfillToPresence', args); - }, - reject(...args) { - return sync('reject', args); - }, - }); - - async function loadBundle(name, bundle) { - if (vatNS !== null) { - throw new Error('bundle already loaded'); - } - vatNS = await importBundle(bundle, { - filePrefix: name, - endowments: makeVatEndowments(name), - }); - // TODO: be sure console.log isn't mixed with protocol stream - // console.log('loaded bundle with methods', Object.keys(vatNS)); + async function doProcess(dispatchRecord, errmsg) { + const dispatchOp = dispatchRecord[0]; + const dispatchArgs = dispatchRecord.slice(1); + workerLog(`runAndWait`); + await runAndWait( + () => dispatch[dispatchOp](...dispatchArgs), + errmsg, + setImmediate, + ); + workerLog(`doProcess done`); + } - state = {}; // ?? - dispatch = makeLiveSlots(syscall, state, vatNS.buildRootObject); + function doNotify(vpid, vp) { + const errmsg = `vat.promise[${vpid}] ${vp.state} failed`; + switch (vp.state) { + case 'fulfilledToPresence': + return doProcess(['notifyFulfillToPresence', vpid, vp.slot], errmsg); + case 'redirected': + throw new Error('not implemented yet'); + case 'fulfilledToData': + return doProcess(['notifyFulfillToData', vpid, vp.data], errmsg); + case 'rejected': + return doProcess(['notifyReject', vpid, vp.data], errmsg); + default: + throw Error(`unknown promise state '${vp.state}'`); + } } - function turnCrank(dispatchType, args) { - return new Promise((resolve, reject) => { - try { - dispatch[dispatchType](...args); - } catch (error) { - // console.log({ dispatchError: error }); - reject(error); - return; - } - endOfCrank(setImmediate).then(resolve); - }); + function sendUplink(msg) { + assert(msg instanceof Array, `msg must be an Array`); + io.writeMessage(JSON.stringify(msg)); } - const name = 'WORKER'; // TODO? - async function handle(message) { - switch (message.msgtype) { - case 'load-bundle': - try { - await loadBundle(name, message.bundle); - } catch (error) { - // console.log('load-bundle failed:', error); - io.writeMessage( - format({ msgtype: 'load-bundle-nak', error: error.message }), - ); - break; - } - io.writeMessage(format({ msgtype: 'load-bundle-ack' })); - break; - case 'dispatch': - try { - await turnCrank(message.type, message.args); - } catch (error) { - io.writeMessage( - format({ - msgtype: 'dispatch-nak', - error: error instanceof Error ? error.message : error, - }), - ); - break; + // fromParent.on('data', data => { + // workerLog('data from parent', data); + // toParent.write('child ack'); + // }); + + let syscallLog; + const handle = harden(async ([type, ...margs]) => { + workerLog(`received`, type); + if (type === 'start') { + // TODO: parent should send ['start', vatID] + workerLog(`got start`); + sendUplink(['gotStart']); + } else if (type === 'setBundle') { + const [bundle, vatParameters] = margs; + const endowments = { + console: makeConsole(`SwingSet:vatWorker`), + HandledPromise, + }; + // ISSUE: this draft code is contorted because it started + // as code that didn't return anything but now it + // has to return a promise to be resolved before + // reading the next input. + return importBundle(bundle, { endowments }).then(vatNS => { + workerLog(`got vatNS:`, Object.keys(vatNS).join(',')); + sendUplink(['gotBundle']); + + function doSyscall(vatSyscallObject) { + sendUplink(['syscall', ...vatSyscallObject]); } - io.writeMessage(format({ msgtype: 'dispatch-ack' })); - break; - case 'finish': - io.writeMessage(format({ msgtype: 'finish-ack' })); - break; - default: - console.warn('unexpected msgtype', message.msgtype); + const syscall = harden({ + send: (...args) => doSyscall(['send', ...args]), + callNow: (..._args) => { + throw Error(`nodeWorker cannot syscall.callNow`); + }, + subscribe: (...args) => doSyscall(['subscribe', ...args]), + fulfillToData: (...args) => doSyscall(['fulfillToData', ...args]), + fulfillToPresence: (...args) => + doSyscall(['fulfillToPresence', ...args]), + reject: (...args) => doSyscall(['reject', ...args]), + }); + + const state = null; + const vatID = 'demo-vatID'; + // todo: maybe add transformTildot, makeGetMeter/transformMetering to + // vatPowers, but only if options tell us they're wanted. Maybe + // transformTildot should be async and outsourced to the kernel + // process/thread. + const vatPowers = { Remotable, getInterfaceOf }; + dispatch = makeLiveSlots( + syscall, + state, + vatNS.buildRootObject, + vatID, + vatPowers, + vatParameters, + ); + workerLog(`got dispatch:`, Object.keys(dispatch).join(',')); + sendUplink(['dispatchReady']); + return type; + }); + } else if (type === 'deliver') { + if (!dispatch) { + workerLog(`error: deliver before dispatchReady`); + return undefined; + } + const [dtype, ...dargs] = margs; + if (dtype === 'message') { + const [targetSlot, msg] = dargs; + const errmsg = `vat[${targetSlot}].${msg.method} dispatch failed`; + await doProcess( + ['deliver', targetSlot, msg.method, msg.args, msg.result], + errmsg, + ).then(() => { + sendUplink(['deliverDone']); + }); + } else if (dtype === 'notify') { + await doNotify(...dargs).then(() => + sendUplink(['deliverDone', syscallLog]), + ); + } else { + throw Error(`bad delivery type ${dtype}`); + } + } else { + workerLog(`unrecognized downlink message ${type}`); } - return message.msgtype; - } + return type; + }); return harden({ handle }); } export async function main({ readMessage, writeMessage, setImmediate }) { + workerLog(`supervisor started`); + const worker = makeWorker({ readMessage, writeMessage }, setImmediate); + const EOF = new Error('EOF'); for (;;) { let message; From 1b4d963ffb1f94911088e87d1a9b659c29090248 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Sat, 22 Aug 2020 17:33:37 -0500 Subject: [PATCH 08/20] chore(xs-vat-worker): prune obsolete test materials --- packages/xs-vat-worker/test/Makefile | 29 ---- packages/xs-vat-worker/test/bootstrap.js | 34 ----- .../xs-vat-worker/test/kernelSimulator.js | 138 ------------------ .../xs-vat-worker/test/sort_transcript.py | 5 - packages/xs-vat-worker/test/test-vat1.js | 28 ---- packages/xs-vat-worker/test/transcript.txt | 6 - packages/xs-vat-worker/test/vat-target.js | 55 ------- 7 files changed, 295 deletions(-) delete mode 100644 packages/xs-vat-worker/test/Makefile delete mode 100644 packages/xs-vat-worker/test/bootstrap.js delete mode 100644 packages/xs-vat-worker/test/kernelSimulator.js delete mode 100644 packages/xs-vat-worker/test/sort_transcript.py delete mode 100644 packages/xs-vat-worker/test/test-vat1.js delete mode 100644 packages/xs-vat-worker/test/transcript.txt delete mode 100644 packages/xs-vat-worker/test/vat-target.js diff --git a/packages/xs-vat-worker/test/Makefile b/packages/xs-vat-worker/test/Makefile deleted file mode 100644 index 36ca5b3b3f8..00000000000 --- a/packages/xs-vat-worker/test/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -NODE_ESM=node -r esm - -transcript.txt: swingset-kernel-state - $(NODE_ESM) ../swingset-runner/bin/kerneldump --filedb ./swingset-kernel-state | grep v1.t |sort >$@ - -swingset-kernel-state: bootstrap.js vat-target.js - $(NODE_ESM) ../swingset-runner/bin/runner --filedb run - -ZOE1=../zoe/test/swingsetTests/zoe - - -zoe: transcript-zoe.txt ./node-vat-worker index.js vatWorker.js kernelSimulator.js - VAT1=$(ZOE1)/vat-zoe.js TRANSCRIPT=transcript-zoe.txt node -r esm kernelSimulator.js - -zoe-node: transcript-zoe.txt ./node-vat-worker index.js vatWorker.js kernelSimulator.js - VAT1=$(ZOE1)/vat-zoe.js TRANSCRIPT=transcript-zoe.txt WORKERBIN=./node-vat-worker node -r esm kernelSimulator.js - - -transcript-zoe.txt: $(ZOE1)/swingset-kernel-state - $(NODE_ESM) ../swingset-runner/bin/kerneldump --filedb $(ZOE1)/swingset-kernel-state | grep ^v5.t | grep -v nextID >,zoe-all - python sort_transcript.py <,zoe-all >$@ - - -$(ZOE1)/swingset-kernel-state: ,zoe-patched - (cd $(ZOE1) && $(NODE_ESM) ../../../../swingset-runner/bin/runner --filedb run) - -,zoe-patched: $(ZOE1)/bootstrap.js zoe-test.patch - cd ../.. && patch -p1 0, - () => 0, - ); -} - -export function buildRootObject() { - const callbackObj = harden({ - callback(arg1, arg2) { - console.log(`callback`, arg1, arg2); - return ['data', callbackObj]; // four, resolves pF - }, - }); - - const precD = producePromise(); - const precE = producePromise(); - - return harden({ - bootstrap(_argv, vats) { - const pA = E(vats.target).zero(callbackObj, precD.promise, precE.promise); - E(vats.target).one(); - precD.resolve(callbackObj); // two - precE.reject(Error('four')); // three - pA.then(([pB, pC]) => { - ignore(pB); - ignore(pC); - }); - }, - }); -} diff --git a/packages/xs-vat-worker/test/kernelSimulator.js b/packages/xs-vat-worker/test/kernelSimulator.js deleted file mode 100644 index faaa31a762a..00000000000 --- a/packages/xs-vat-worker/test/kernelSimulator.js +++ /dev/null @@ -1,138 +0,0 @@ -// Usage: node -r esm kernelSimulator.js -// context: https://github.com/Agoric/agoric-sdk/issues/1299 -import '@agoric/install-ses'; - -import { readNetstring, writeNetstring } from '../src/netstring'; - -const INFD = 3; -const OUTFD = 4; - -function options(env) { - return { - vat1: env.VAT1 || 'vat-target.js', - transcript: env.TRANSCRIPT || 'transcript.txt', - workerBin: env.WORKERBIN || './build/bin/lin/release/xs-vat-worker', - }; -} - -function makeWorker(child) { - const format = obj => JSON.stringify(obj); - const send = obj => writeNetstring(child.stdio[INFD], format(obj)); - - child.stdio[OUTFD].pause(); - - const expect = async msgtype => { - const txt = await readNetstring(child.stdio[OUTFD]); - let msg; - try { - msg = JSON.parse(txt); - } catch (badJSON) { - console.error('bad JSON ', txt.length, ' chars: [', txt, ']', badJSON); - throw badJSON; - } - if (msg.msgtype !== msgtype) { - throw new Error( - `expected ${msgtype}; found: ${msg.msgtype}; error: ${ - msg.error - } [${JSON.stringify(msg)}]`, - ); - } - return msg; - }; - - return harden({ - async loadVat(bundle) { - await send({ msgtype: 'load-bundle', bundle }); - return expect('load-bundle-ack'); - }, - async dispatch({ d, syscalls, _crankNumber }) { - await send({ msgtype: 'dispatch', type: d[0], args: d.slice(1) }); - for (const syscall of syscalls) { - // eslint-disable-next-line no-await-in-loop - const request = await expect('syscall'); - console.log('syscall request', request); - // eslint-disable-next-line no-await-in-loop - await send({ msgtype: 'syscall-ack', response: syscall.response }); - } - return expect('dispatch-ack'); - }, - async finish() { - await send({ msgtype: 'finish' }); - await expect('finish-ack'); - }, - }); -} - -async function runTranscript(w1, bundle, transcript) { - await w1.loadVat(bundle); - console.log('loadVat done.'); - - const events = transcript.split('\n'); - - while (events.length > 0) { - const event = events.shift(); - if (!event) { - // eslint-disable-next-line no-continue - continue; - } - const found = event.match(/^(?[^ ]+) :: (?.*)/); - if (!found) { - console.log('unexpected transcript format', { line: event }); - // eslint-disable-next-line no-continue - continue; - } - const { id, payload } = found.groups; - - const obj = JSON.parse(payload); - if (typeof obj !== 'object') { - console.log('not a dispatch event', id, obj); - // eslint-disable-next-line no-continue - continue; - } - console.log('dispatching:', id); - // eslint-disable-next-line no-await-in-loop - await w1.dispatch(obj); - console.log('dispatch done:', id); - } - - await w1.finish(); - console.log('END OF TRANSCRIPT.'); -} - -// eslint-disable-next-line no-shadow -export async function main(argv, { env, io, bundleSource, spawn }) { - const { vat1, transcript, workerBin } = options(env); - - const bundle = await bundleSource(vat1); - - console.log('spawning', { workerBin }); - const child = await spawn(workerBin, [], { - stdio: ['inherit', 'inherit', 'inherit', 'pipe', 'pipe'], - }); - child.on('exit', (code, signal) => { - if (code !== 0) { - console.error('unexpected exit:', { code, signal }); - } - }); - const w1 = makeWorker(child); - - const text = await io.readFile(transcript, 'utf-8'); - await runTranscript(w1, bundle, text); -} - -if (require.main === module) { - main(process.argv, { - env: process.env, - // eslint-disable-next-line global-require - io: { - // eslint-disable-next-line global-require - readFile: require('fs').promises.readFile, - }, - // eslint-disable-next-line global-require - bundleSource: require('@agoric/bundle-source').default, - // eslint-disable-next-line global-require - spawn: require('child_process').spawn, - }).catch(err => { - console.error(err); - }); -} diff --git a/packages/xs-vat-worker/test/sort_transcript.py b/packages/xs-vat-worker/test/sort_transcript.py deleted file mode 100644 index c6c478a0c5c..00000000000 --- a/packages/xs-vat-worker/test/sort_transcript.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys -lines = sys.stdin.readlines() -lines.sort(key=lambda line: int(line.split(" ", 1)[0].split(".")[2])) -for line in lines: - sys.stdout.write(line) diff --git a/packages/xs-vat-worker/test/test-vat1.js b/packages/xs-vat-worker/test/test-vat1.js deleted file mode 100644 index 16701075e0e..00000000000 --- a/packages/xs-vat-worker/test/test-vat1.js +++ /dev/null @@ -1,28 +0,0 @@ -import { test } from 'tape-promise/tape'; - -import { main } from './kernelSimulator'; - -const resolve = p => require.resolve(p); - -test('replay simple transcript with node vatWorker', async t => { - process.env.WORKERBIN = resolve('../bin/node-vat-worker'); - process.env.VAT1 = resolve('./vat-target.js'); - process.env.TRANSCRIPT = resolve('./transcript.txt'); - - process.chdir(resolve('..')); // so node-vat-worker can find stuff in src/ - await main(process.argv, { - env: process.env, - // eslint-disable-next-line global-require - io: { - // eslint-disable-next-line global-require - readFile: require('fs').promises.readFile, - }, - // eslint-disable-next-line global-require - bundleSource: require('@agoric/bundle-source').default, - // eslint-disable-next-line global-require - spawn: require('child_process').spawn, - }); - - t.ok('did not crash'); - t.end(); -}); diff --git a/packages/xs-vat-worker/test/transcript.txt b/packages/xs-vat-worker/test/transcript.txt deleted file mode 100644 index 4f6443500d8..00000000000 --- a/packages/xs-vat-worker/test/transcript.txt +++ /dev/null @@ -1,6 +0,0 @@ -v1.t.0 :: {"d":["deliver","o+0","zero",{"body":"[{\"@qclass\":\"slot\",\"index\":0},{\"@qclass\":\"slot\",\"index\":1},{\"@qclass\":\"slot\",\"index\":2}]","slots":["o-50","p-60","p-61"]},"p-62"],"syscalls":[{"d":["subscribe","p-60"],"response":null},{"d":["subscribe","p-61"],"response":null},{"d":["send","o-50","callback",{"body":"[11,12]","slots":[]},"p+5"],"response":null},{"d":["subscribe","p+5"],"response":null},{"d":["fulfillToData","p-62",{"body":"[{\"@qclass\":\"slot\",\"index\":0},{\"@qclass\":\"slot\",\"index\":1}]","slots":["p+6","p+7"]}],"response":null}],"crankNumber":2} -v1.t.1 :: {"d":["deliver","o+0","one",{"body":"[]","slots":[]},"p-63"],"syscalls":[{"d":["fulfillToPresence","p+6","o-50"],"response":null},{"d":["reject","p+7",{"body":"{\"@qclass\":\"error\",\"name\":\"Error\",\"message\":\"oops\"}","slots":[]}],"response":null},{"d":["fulfillToData","p-63",{"body":"1","slots":[]}],"response":null}],"crankNumber":3} -v1.t.2 :: {"d":["notifyFulfillToPresence","p-60","o-50"],"syscalls":[],"crankNumber":4} -v1.t.3 :: {"d":["notifyReject","p-61",{"body":"{\"@qclass\":\"error\",\"name\":\"Error\",\"message\":\"four\"}","slots":[]}],"syscalls":[],"crankNumber":5} -v1.t.4 :: {"d":["notifyFulfillToData","p+5",{"body":"[\"data\",{\"@qclass\":\"slot\",\"index\":0}]","slots":["o-50"]}],"syscalls":[],"crankNumber":9} -v1.t.nextID :: 5 diff --git a/packages/xs-vat-worker/test/vat-target.js b/packages/xs-vat-worker/test/vat-target.js deleted file mode 100644 index 5e63ccfbaaf..00000000000 --- a/packages/xs-vat-worker/test/vat-target.js +++ /dev/null @@ -1,55 +0,0 @@ -import { E } from '@agoric/eventual-send'; -import { makePromiseKit as producePromise } from '@agoric/promise-kit'; - -function ignore(p) { - p.then( - () => 0, - () => 0, - ); -} - -// We arrange for this vat, 'vat-target', to receive a specific set of -// inbound events ('dispatch'), which will provoke a set of outbound events -// ('syscall'), that cover the full range of the dispatch/syscall interface - -export function buildRootObject() { - const precB = producePromise(); - const precC = producePromise(); - let callbackObj; - - // zero: dispatch.deliver(target, method="one", result=pA, args=[callbackObj, pD, pE]) - // syscall.subscribe(pD) - // syscall.subscribe(pE) - // syscall.send(callbackObj, method="callback", result=rp2, args=[11, 12]); - // syscall.subscribe(rp2) - // syscall.fulfillToData(pA, [pB, pC]); - function zero(obj, pD, pE) { - callbackObj = obj; - E(callbackObj).callback(11, 12); // syscall.send - ignore(pD); - ignore(pE); - return [precB.promise, precC.promise]; // syscall.fulfillToData - } - - // one: dispatch.deliver(target, method="two", result=rp3, args=[]) - // syscall.fulfillToPresence(pB, callbackObj) - // syscall.reject(pC, Error('oops')) - // syscall.fulfillToData(rp3, 1) - function one() { - precB.resolve(callbackObj); // syscall.fulfillToPresence - precC.reject(Error('oops')); // syscall.reject - return 1; - } - - // two: dispatch.notifyFulfillToPresence(pD, callbackObj) - // three: dispatch.notifyReject(pE, Error('four')) - - // four: dispatch.notifyFulfillToData(pF, ['data', callbackObj]) - - const target = harden({ - zero, - one, - }); - - return target; -} From 29406ad7f14f06ed5e089049e73bca1fe8a53be4 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Mon, 24 Aug 2020 20:17:48 -0500 Subject: [PATCH 09/20] chore(xs-vat-worker): HandledPromise transition --- packages/xs-vat-worker/compartmap.json | 27 +------------------------ packages/xs-vat-worker/manifest.json | 1 + packages/xs-vat-worker/src/endo-load.js | 19 ++++++++++------- packages/xs-vat-worker/src/vatWorker.js | 2 +- packages/xs-vat-worker/start-xs.js | 5 ++++- 5 files changed, 19 insertions(+), 35 deletions(-) diff --git a/packages/xs-vat-worker/compartmap.json b/packages/xs-vat-worker/compartmap.json index 08336b1d9ba..f668b2f6892 100644 --- a/packages/xs-vat-worker/compartmap.json +++ b/packages/xs-vat-worker/compartmap.json @@ -12,10 +12,6 @@ "compartment": "node_modules/@agoric/import-bundle/", "module": "./src/index.js" }, - "@agoric/eventual-send": { - "compartment": "node_modules/@agoric/eventual-send/", - "module": "./src/index.js" - }, "@agoric/marshal": { "compartment": "node_modules/@agoric/marshal/", "module": "./marshal.js" @@ -35,15 +31,6 @@ ], "modules": {} }, - "node_modules/@agoric/eventual-send/": { - "label": "@agoric/eventual-send@0.9.3", - "location": "node_modules/@agoric/eventual-send/", - "contents": [ - "./src/index.js", - "./src/E.js" - ], - "modules": {} - }, "node_modules/@agoric/marshal/": { "label": "@agoric/marshal@0.2.3", "location": "node_modules/@agoric/marshal/", @@ -70,10 +57,6 @@ "./src/capdata.js" ], "modules": { - "@agoric/eventual-send": { - "compartment": "node_modules/@agoric/eventual-send/", - "module": "./src/index.js" - }, "@agoric/marshal": { "compartment": "node_modules/@agoric/marshal/", "module": "./marshal.js" @@ -106,12 +89,7 @@ "contents": [ "./src/promiseKit.js" ], - "modules": { - "@agoric/eventual-send": { - "compartment": "node_modules/@agoric/eventual-send/", - "module": "./src/index.js" - } - } + "modules": {} }, "node_modules/@agoric/assert/": { "label": "@agoric/assert@0.0.8", @@ -129,9 +107,6 @@ "node_modules/@agoric/assert/0_MKDIR": "$(ROOT)/node_modules/@agoric/assert/0_MKDIR", "node_modules/@agoric/assert/src/assert": "$(ROOT)/node_modules/@agoric/assert/src/assert", "node_modules/@agoric/assert/src/types": "$(ROOT)/node_modules/@agoric/assert/src/types", - "node_modules/@agoric/eventual-send/0_MKDIR": "$(ROOT)/node_modules/@agoric/eventual-send/0_MKDIR", - "node_modules/@agoric/eventual-send/src/E": "$(ROOT)/node_modules/@agoric/eventual-send/src/E", - "node_modules/@agoric/eventual-send/src/index": "$(ROOT)/node_modules/@agoric/eventual-send/src/index", "node_modules/@agoric/import-bundle/0_MKDIR": "$(ROOT)/node_modules/@agoric/import-bundle/0_MKDIR", "node_modules/@agoric/import-bundle/src/compartment-wrapper": "$(ROOT)/node_modules/@agoric/import-bundle/src/compartment-wrapper", "node_modules/@agoric/import-bundle/src/index": "$(ROOT)/node_modules/@agoric/import-bundle/src/index", diff --git a/packages/xs-vat-worker/manifest.json b/packages/xs-vat-worker/manifest.json index 1254b0276c3..5dd60e444a8 100644 --- a/packages/xs-vat-worker/manifest.json +++ b/packages/xs-vat-worker/manifest.json @@ -24,6 +24,7 @@ "$(MODULES)/base/timer/timer", "$(MODULES)/base/timer/lin/*" ], + "@agoric/eventual-send": "../eventual-send/src/index", "main": "./start-xs", "src/console": "./src/console", diff --git a/packages/xs-vat-worker/src/endo-load.js b/packages/xs-vat-worker/src/endo-load.js index 119f4776fba..45465cef768 100644 --- a/packages/xs-vat-worker/src/endo-load.js +++ b/packages/xs-vat-worker/src/endo-load.js @@ -4,19 +4,24 @@ function debug(..._args) { // console.log(...args); } -function SESCompartment(endowments, map, options) { - debug('SESCompartment', { endowments, map, options }); - const sesGlobals = { harden, console, Compartment: SESCompartment }; - return new Compartment({ ...sesGlobals, ...endowments }, map, options); -} - -export function loadMain(compartmap) { +export function loadMain(compartmap, HandledPromise) { // ISSUE: doesn't seem to work: const { entries, fromEntries, keys } = Object; // debug('entries, ...', { entries: typeof entries, fromEntries: typeof fromEntries, keys: typeof keys }); const entries = o => Object.entries(o); const fromEntries = pvs => Object.fromEntries(pvs); const keys = o => Object.keys(o); + function SESCompartment(endowments, map, options) { + debug('SESCompartment', { endowments, map, options }); + const sesGlobals = { + harden, + console, + HandledPromise, + Compartment: SESCompartment, + }; + return new Compartment({ ...sesGlobals, ...endowments }, map, options); + } + const memoize = f => { const cache = {}; return k => cache[k] || (cache[k] = f(k)); diff --git a/packages/xs-vat-worker/src/vatWorker.js b/packages/xs-vat-worker/src/vatWorker.js index 2a30509bc08..cfdfc58a441 100644 --- a/packages/xs-vat-worker/src/vatWorker.js +++ b/packages/xs-vat-worker/src/vatWorker.js @@ -1,5 +1,5 @@ +/* global HandledPromise */ import { importBundle } from '@agoric/import-bundle'; -import { HandledPromise } from '@agoric/eventual-send'; import { Remotable, getInterfaceOf } from '@agoric/marshal'; // TODO? import anylogger from 'anylogger'; import { makeLiveSlots } from '@agoric/swingset-vat/src/kernel/liveSlots'; diff --git a/packages/xs-vat-worker/start-xs.js b/packages/xs-vat-worker/start-xs.js index 4d80d226364..e4841bfb586 100644 --- a/packages/xs-vat-worker/start-xs.js +++ b/packages/xs-vat-worker/start-xs.js @@ -4,6 +4,7 @@ import Timer from '@moddable/timer'; // eslint-disable-next-line import/no-unresolved import Resource from '@moddable/Resource'; +import { makeHandledPromise } from '@agoric/eventual-send'; import { harden } from './src/harden'; import { makeConsole } from './src/console'; @@ -24,7 +25,9 @@ export default async function main() { const inStream = new Reader(INFD); const outStream = new Writer(OUTFD); - const c1 = loadMain(compartmap0); + const HandledPromise = makeHandledPromise(); + + const c1 = loadMain(compartmap0, HandledPromise); console.log('about to import vatWorker'); const vw = await c1.import('src/vatWorker'); From aecaeb143668825183c5aa1b9a5c76d954b51501 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 14:24:02 -0500 Subject: [PATCH 10/20] feat(xs-vat-worker): locateWorkerBin finds built executable --- packages/SwingSet/src/controller.js | 5 +++-- packages/SwingSet/test/workers/test-worker.js | 9 ++++++--- packages/xs-vat-worker/package.json | 1 - packages/xs-vat-worker/src/locate-undefined.js | 1 - packages/xs-vat-worker/src/locate.js | 15 +++++++++++++++ packages/xs-vat-worker/test/test-locate.js | 8 ++++++-- packages/xs-vat-worker/xs-lin.mk | 1 - 7 files changed, 30 insertions(+), 10 deletions(-) delete mode 100644 packages/xs-vat-worker/src/locate-undefined.js create mode 100644 packages/xs-vat-worker/src/locate.js diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index 1691abcea2c..cc67db0eb3f 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -16,7 +16,7 @@ import { importBundle } from '@agoric/import-bundle'; import { initSwingStore } from '@agoric/swing-store-simple'; import { makeMeteringTransformer } from '@agoric/transform-metering'; import { makeTransform } from '@agoric/transform-eventual-send'; -import { xsWorkerBin } from '@agoric/xs-vat-worker'; +import { locateWorkerBin } from '@agoric/xs-vat-worker'; import { startSubprocessWorker } from './spawnSubprocessWorker'; import { assertKnownOptions } from './assertOptions'; @@ -322,7 +322,8 @@ export async function buildVatController( // console.log(`--slog ${JSON.stringify(obj)}`); } - const startXsWorker = xsWorkerBin + const xsWorkerBin = locateWorkerBin({ resolve: path.resolve }); + const startXsWorker = fs.existsSync(xsWorkerBin) ? () => startSubprocessWorker(xsWorkerBin, []) : undefined; diff --git a/packages/SwingSet/test/workers/test-worker.js b/packages/SwingSet/test/workers/test-worker.js index 4d877490b72..84a82ecf30b 100644 --- a/packages/SwingSet/test/workers/test-worker.js +++ b/packages/SwingSet/test/workers/test-worker.js @@ -1,11 +1,14 @@ import '@agoric/install-ses'; import test from 'ava'; -import { xsWorkerBin } from '@agoric/xs-vat-worker/src/locate'; +import { resolve } from 'path'; +import { existsSync } from 'fs'; +import { locateWorkerBin } from '@agoric/xs-vat-worker'; import { loadBasedir, buildVatController } from '../../src/index'; test('xs vat manager', async t => { - if (!xsWorkerBin) { - console.warn('XS vat worker not built; skipping'); + const bin = locateWorkerBin({ resolve }); + if (!existsSync(bin)) { + console.warn(`XS vat worker ${bin} not built; skipping`); t.falsy.skip(false); return; } diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 736d505a961..d7f543b20ce 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -5,7 +5,6 @@ "main": "src/locate.js", "module": "src/locate.js", "scripts": { - "postinstall": "cp src/locate-undefined.js src/locate.js", "build:xs-lin": "make -f xs-lin.mk", "test": "tape -r esm 'test/**/test-*.js'", "build": "exit 0", diff --git a/packages/xs-vat-worker/src/locate-undefined.js b/packages/xs-vat-worker/src/locate-undefined.js deleted file mode 100644 index 1ba903d1b52..00000000000 --- a/packages/xs-vat-worker/src/locate-undefined.js +++ /dev/null @@ -1 +0,0 @@ -export const xsWorkerBin = undefined; diff --git a/packages/xs-vat-worker/src/locate.js b/packages/xs-vat-worker/src/locate.js new file mode 100644 index 00000000000..4f946231069 --- /dev/null +++ b/packages/xs-vat-worker/src/locate.js @@ -0,0 +1,15 @@ +/** + * Locate the XS vat worker executable. + * + * Note: executable is not built by default. + * @see the `build:xs-lin` script in package.json + * + * @param {{ resolve: (...string) => string }} filesystem path access + * @returns { string } full path where linux debug executable is built; + * not guaranteed to exist. + */ +export function locateWorkerBin({ resolve }) { + const goal = 'debug'; // ISSUE: support, test release too? + const os = 'lin'; // ISSUE: support, test mac too? + return resolve(__dirname, '../build/bin', os, goal, 'xs-vat-worker'); +} diff --git a/packages/xs-vat-worker/test/test-locate.js b/packages/xs-vat-worker/test/test-locate.js index 82e7182b06b..a1769c80e37 100644 --- a/packages/xs-vat-worker/test/test-locate.js +++ b/packages/xs-vat-worker/test/test-locate.js @@ -1,8 +1,12 @@ +import { resolve } from 'path'; + import { test } from 'tape-promise/tape'; -import { xsWorkerBin } from '../src/locate'; +import { locateWorkerBin } from '../src/locate'; test('locateWorkerBin', t => { - t.ok(!xsWorkerBin || xsWorkerBin.endsWith('xs-vat-worker')); + const bin = locateWorkerBin({ resolve }); + t.ok(bin.startsWith('/')); + t.ok(bin.endsWith('/xs-vat-worker')); t.end(); }); diff --git a/packages/xs-vat-worker/xs-lin.mk b/packages/xs-vat-worker/xs-lin.mk index 6bc4f185896..9db23af4279 100644 --- a/packages/xs-vat-worker/xs-lin.mk +++ b/packages/xs-vat-worker/xs-lin.mk @@ -5,7 +5,6 @@ ROOT=$(PWD)/../.. $(NODE_MODULES)/.bin/xs-vat-worker: build/bin/lin/debug/xs-vat-worker cp $< $@ - echo "export const xsWorkerBin =\n '$@';" >src/locate.js build/bin/lin/debug/xs-vat-worker: build $(TOOLS)/mcconfig moddable/xs/platforms/lin_xs_cli.c compartmap.json manifest.json ROOT=$(ROOT) PATH=$(TOOLS):$$PATH MODDABLE=$(MODDABLE) mcconfig -o build -p x-cli-lin -m -d From da824e9cecbb15383eee97a44980bc89f97dfb15 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:08:57 -0500 Subject: [PATCH 11/20] test: expose xsWorkerBin skip status via ava --- packages/SwingSet/test/workers/test-worker.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/SwingSet/test/workers/test-worker.js b/packages/SwingSet/test/workers/test-worker.js index 84a82ecf30b..d64eba155ac 100644 --- a/packages/SwingSet/test/workers/test-worker.js +++ b/packages/SwingSet/test/workers/test-worker.js @@ -5,14 +5,10 @@ import { existsSync } from 'fs'; import { locateWorkerBin } from '@agoric/xs-vat-worker'; import { loadBasedir, buildVatController } from '../../src/index'; -test('xs vat manager', async t => { - const bin = locateWorkerBin({ resolve }); - if (!existsSync(bin)) { - console.warn(`XS vat worker ${bin} not built; skipping`); - t.falsy.skip(false); - return; - } +const xsWorkerBin = locateWorkerBin({ resolve }); +const maybeTestXS = existsSync(xsWorkerBin) ? test : test.skip; +maybeTestXS('xs vat manager', async t => { const config = await loadBasedir(__dirname); config.vats.target.creationOptions = { managerType: 'xs-worker' }; const c = await buildVatController(config, []); From dcf0590038e6f8e58f6127152c2068a8486dc647 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:13:32 -0500 Subject: [PATCH 12/20] test(xs-vat-worker): migrate to ava from tape --- packages/xs-vat-worker/package.json | 10 +++++++--- packages/xs-vat-worker/test/test-locate.js | 7 +++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index d7f543b20ce..addc0f0c2cd 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -6,13 +6,18 @@ "module": "src/locate.js", "scripts": { "build:xs-lin": "make -f xs-lin.mk", - "test": "tape -r esm 'test/**/test-*.js'", + "test": "ava", "build": "exit 0", "lint-fix": "eslint --fix '**/*.{js,jsx}'", "lint-check": "eslint '**/*.{js,jsx}'", "lint-fix-jessie": "eslint -c '.eslintrc-jessie.js' --fix '**/*.{js,jsx}'", "lint-check-jessie": "eslint -c '.eslintrc-jessie.js' '**/*.{js,jsx}'" }, + "ava": { + "files": ["test/**/test-*.js"], + "require": ["esm"], + "timeout": "2m" + }, "devDependencies": { "detective-es6": "^2.2.0", "eslint": "^6.1.0", @@ -24,8 +29,7 @@ "filing-cabinet": "^2.5.1", "prettier": "^1.18.2", "tap-spec": "^5.0.0", - "tape": "^4.11.0", - "tape-promise": "^4.0.0" + "ava": "^3.11.1" }, "dependencies": { "@agoric/assert": "^0.0.8", diff --git a/packages/xs-vat-worker/test/test-locate.js b/packages/xs-vat-worker/test/test-locate.js index a1769c80e37..3b1b3ffd554 100644 --- a/packages/xs-vat-worker/test/test-locate.js +++ b/packages/xs-vat-worker/test/test-locate.js @@ -1,12 +1,11 @@ import { resolve } from 'path'; -import { test } from 'tape-promise/tape'; +import test from 'ava'; import { locateWorkerBin } from '../src/locate'; test('locateWorkerBin', t => { const bin = locateWorkerBin({ resolve }); - t.ok(bin.startsWith('/')); - t.ok(bin.endsWith('/xs-vat-worker')); - t.end(); + t.truthy(bin.startsWith('/')); + t.truthy(bin.endsWith('/xs-vat-worker')); }); From cc8c3be490b683764f5876c60c400382316d2392 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:15:33 -0500 Subject: [PATCH 13/20] build(xs-vat-worker): never mind installing in node_modules/.bin --- packages/xs-vat-worker/xs-lin.mk | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/xs-vat-worker/xs-lin.mk b/packages/xs-vat-worker/xs-lin.mk index 9db23af4279..78450905693 100644 --- a/packages/xs-vat-worker/xs-lin.mk +++ b/packages/xs-vat-worker/xs-lin.mk @@ -3,9 +3,6 @@ TOOLS=$(MODDABLE)/build/bin/lin/release/ NODE_MODULES=$(PWD)/node_modules ROOT=$(PWD)/../.. -$(NODE_MODULES)/.bin/xs-vat-worker: build/bin/lin/debug/xs-vat-worker - cp $< $@ - build/bin/lin/debug/xs-vat-worker: build $(TOOLS)/mcconfig moddable/xs/platforms/lin_xs_cli.c compartmap.json manifest.json ROOT=$(ROOT) PATH=$(TOOLS):$$PATH MODDABLE=$(MODDABLE) mcconfig -o build -p x-cli-lin -m -d From 0c8c6021fe2277d5755e0a3dac1db45a90120e91 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:18:20 -0500 Subject: [PATCH 14/20] build(xs-vat-worker): separate install-gio target ack: warner --- packages/xs-vat-worker/xs-lin.mk | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/xs-vat-worker/xs-lin.mk b/packages/xs-vat-worker/xs-lin.mk index 78450905693..dc169ef21a7 100644 --- a/packages/xs-vat-worker/xs-lin.mk +++ b/packages/xs-vat-worker/xs-lin.mk @@ -20,7 +20,13 @@ moddable/xs/platforms/lin_xs_cli.c: moddable/xs/platforms/lin_xs.h moddable/xs/platforms/lin_xs.h: /usr/include/glib-2.0/gio/gio.h +.PHONY: install-gio /usr/include/glib-2.0/gio/gio.h: + @echo "GIO not installed, need root to run apt-get install libgio2.0-dev + @echo "feel free to stop now and run `make install-gio` directly, then rebuild" + $(MAKE) install-gio + +install-gio: sudo apt-get -y update && sudo apt-get -y install libgio2.0-dev $(TOOLS)/mcconfig: From 1fb5d238f423e20a54cf276860a84b79be35ea79 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:28:14 -0500 Subject: [PATCH 15/20] chore: prune more obsolete test materials --- packages/xs-vat-worker/bin/node-vat-worker | 2 - packages/xs-vat-worker/src/netstring.js | 51 ---------- packages/xs-vat-worker/src/start-node.js | 111 --------------------- 3 files changed, 164 deletions(-) delete mode 100755 packages/xs-vat-worker/bin/node-vat-worker delete mode 100644 packages/xs-vat-worker/src/netstring.js delete mode 100755 packages/xs-vat-worker/src/start-node.js diff --git a/packages/xs-vat-worker/bin/node-vat-worker b/packages/xs-vat-worker/bin/node-vat-worker deleted file mode 100755 index 4d16a05043a..00000000000 --- a/packages/xs-vat-worker/bin/node-vat-worker +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -node -r esm ./src/start-node.js diff --git a/packages/xs-vat-worker/src/netstring.js b/packages/xs-vat-worker/src/netstring.js deleted file mode 100644 index 3bd8edc1bdc..00000000000 --- a/packages/xs-vat-worker/src/netstring.js +++ /dev/null @@ -1,51 +0,0 @@ -export async function readNetstring(input) { - let prefix = Buffer.from([]); - let colonPos = -1; - - const nextChunk = () => - new Promise((resolve, _reject) => { - const rx = data => { - input.pause(); - input.removeListener('data', rx); - resolve(data); - }; - input.on('data', rx); - input.resume(); - }); - - while (colonPos < 0) { - // eslint-disable-next-line no-await-in-loop - const more = await nextChunk(); - prefix = Buffer.concat([prefix, more]); - colonPos = prefix.indexOf(':'); - } - let len; - const digits = prefix.slice(0, colonPos).toString('utf-8'); - try { - len = parseInt(digits, 10); - } catch (badLen) { - throw new Error(`bad netstring length ${digits}`); - } - // console.error('readNetstring parsed len', { digits, len }); - let data = prefix.slice(colonPos + 1); - while (data.length <= len) { - // console.log('netstring: looking for payload', data.length, len); - // eslint-disable-next-line no-await-in-loop - const more = await nextChunk(input); - data = Buffer.concat([data, more]); - } - if (data.slice(len).toString('utf-8') !== ',') { - throw new Error( - `bad netstring: expected , got ${data.slice(-1)} [${data.slice(-20)}]`, - ); - } - return data.slice(0, len).toString('utf-8'); -} - -export async function writeNetstring(out, payload) { - // ISSUE: non-ASCII length - // console.log('kernelSimulator send size', content.length); - await out.write(`${payload.length}:`); - await out.write(payload); - await out.write(','); -} diff --git a/packages/xs-vat-worker/src/start-node.js b/packages/xs-vat-worker/src/start-node.js deleted file mode 100755 index 1cde573ea1d..00000000000 --- a/packages/xs-vat-worker/src/start-node.js +++ /dev/null @@ -1,111 +0,0 @@ -// vatWorker driver for node.js -// contrast with main.js for xs -import '@agoric/install-ses'; - -import { main as vatWorker } from './vatWorker'; - -const INFD = 3; -const OUTFD = 4; - -function makePipe(io, sleep) { - function write(data) { - let done = 0; - for (;;) { - try { - done += io.writeSync(OUTFD, data.slice(done)); - if (done >= data.length) { - return done; - } - } catch (writeFailed) { - if (writeFailed.code === 'EAGAIN') { - sleep(0.1); - // try again - } else { - throw writeFailed; - } - } - } - } - - return harden({ - writeMessage(msg) { - write(`${msg.length}:`); - write(msg); - write(','); - }, - readMessage(EOF) { - let buf = Buffer.from('999999999:', 'utf-8'); - let len = null; - let colonPos = null; - let offset = 0; - - for (;;) { - // console.error('readMessage', { length: buf.length, len, colonPos, offset }); - try { - offset += io.readSync(INFD, buf, { - offset, - length: buf.length - offset, - }); - } catch (err) { - if (err.code === 'EAGAIN') { - sleep(0.1); - // eslint-disable-next-line no-continue - continue; - } else if (err.code === 'EOF') { - throw EOF; - } else { - throw err; - } - } - if (len === null) { - colonPos = buf.indexOf(':'); - if (colonPos > 0) { - const digits = buf.slice(0, colonPos).toString('utf-8'); - len = parseInt(digits, 10); - const rest = Buffer.alloc(len + 1); - // console.error('parsed len. copy', { digits, len, targetStart: 0, sourceStart: colonPos + 1, sourceEnd: offset }); - buf.copy(rest, 0, colonPos + 1, offset); - buf = rest; - offset -= colonPos + 1; - } - } else if (offset === len + 1) { - const delim = buf.slice(-1).toString('utf-8'); - if (delim !== ',') { - throw new Error( - `bad netstring: length ${len} expected , found [${delim}]`, - ); - } - const result = buf.slice(0, -1).toString('utf-8'); - // console.error({ colon: colonPos, len, result: result.slice(0, 20) }); - return result; - } - } - }, - }); -} - -async function main({ setImmediate, fs, spawnSync }) { - const sleep = secs => spawnSync('sleep', [secs]); - const pipe = makePipe(fs, sleep); - return vatWorker({ - readMessage: pipe.readMessage, - writeMessage: pipe.writeMessage, - setImmediate, - }); -} - -main({ - setImmediate, - // eslint-disable-next-line global-require - spawnSync: require('child_process').spawnSync, - fs: { - // eslint-disable-next-line global-require - readSync: require('fs').readSync, - // eslint-disable-next-line global-require - writeSync: require('fs').writeSync, - }, -}) - .catch(err => { - console.error(err); - }) - .then(() => process.exit(0)); From 8a704cfa8932c2abf297ed359b6f6f1c9fd09198 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:30:58 -0500 Subject: [PATCH 16/20] build(xs-vat-worker): include trailing newline in compartmap --- packages/xs-vat-worker/compartmap.json | 2 +- packages/xs-vat-worker/tools/findmods.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/xs-vat-worker/compartmap.json b/packages/xs-vat-worker/compartmap.json index f668b2f6892..cf92f156707 100644 --- a/packages/xs-vat-worker/compartmap.json +++ b/packages/xs-vat-worker/compartmap.json @@ -124,4 +124,4 @@ "packages/xs-vat-worker/0_MKDIR": "$(ROOT)/packages/xs-vat-worker/0_MKDIR", "packages/xs-vat-worker/src/vatWorker": "$(ROOT)/packages/xs-vat-worker/src/vatWorker" } -} \ No newline at end of file +} diff --git a/packages/xs-vat-worker/tools/findmods.js b/packages/xs-vat-worker/tools/findmods.js index 484666b32d3..b394b5bd236 100644 --- a/packages/xs-vat-worker/tools/findmods.js +++ b/packages/xs-vat-worker/tools/findmods.js @@ -155,6 +155,7 @@ async function main(argv, { stdout, fsp, fileURL, cabinet }) { }); stdout.write(JSON.stringify(info, null, 2)); + stdout.write('\n'); } /* global require, module, process */ From 6bc5a98de05602ddab09e3a922e16b1154a3f3ad Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 15:34:41 -0500 Subject: [PATCH 17/20] chore(xs-vat-worker): git repo = agoric-sdk --- packages/xs-vat-worker/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index addc0f0c2cd..6cd03807cf5 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -49,12 +49,12 @@ ], "repository": { "type": "git", - "url": "https://gist.github.com/f8e0b5d838079a994784d599c282cce7.git" + "url": "git+https://github.com/Agoric/agoric-sdk.git" }, "author": "Agoric", "license": "Apache-2.0", "bugs": { - "url": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7" + "url": "https://github.com/Agoric/agoric-sdk/issues" }, - "homepage": "https://gist.github.com/dckc/f8e0b5d838079a994784d599c282cce7" + "homepage": "https://github.com/Agoric/agoric-sdk#readme" } From 684d07a521482fa7461464f20c3ff2edee3cfab7 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 16:04:12 -0500 Subject: [PATCH 18/20] refactor: spawn(ANY_PATH) in the kernel goes against POLA startSubprocessWorker was a capability provided to the kernel; adding an execPath argument gave the kernel the ability to launch arbitrary executables, which is well beyond the least authority it needs. Instead, provide startSubprocessWorkerNode and startSubprocessWorkerXS powers, each of which can launch a specific executable. ack: warner --- packages/SwingSet/src/controller.js | 9 +++++---- packages/SwingSet/src/kernel/kernel.js | 8 ++++---- packages/SwingSet/src/kernel/vatManager/factory.js | 10 +++++----- packages/SwingSet/src/spawnSubprocessWorker.js | 6 +++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/packages/SwingSet/src/controller.js b/packages/SwingSet/src/controller.js index cc67db0eb3f..2c24c722146 100644 --- a/packages/SwingSet/src/controller.js +++ b/packages/SwingSet/src/controller.js @@ -322,9 +322,10 @@ export async function buildVatController( // console.log(`--slog ${JSON.stringify(obj)}`); } + const startSubprocessWorkerNode = () => startSubprocessWorker(); const xsWorkerBin = locateWorkerBin({ resolve: path.resolve }); - const startXsWorker = fs.existsSync(xsWorkerBin) - ? () => startSubprocessWorker(xsWorkerBin, []) + const startSubprocessWorkerXS = fs.existsSync(xsWorkerBin) + ? () => startSubprocessWorker({ execPath: xsWorkerBin, args: [] }) : undefined; const kernelEndowments = { @@ -337,9 +338,9 @@ export async function buildVatController( transformMetering, transformTildot, makeNodeWorker, - startSubprocessWorker, + startSubprocessWorkerNode, + startSubprocessWorkerXS, writeSlogObject, - startXsWorker, }; const kernelOptions = { verbose }; diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index aefb2426879..3fe6689f0b1 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -50,9 +50,9 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) { transformMetering, transformTildot, makeNodeWorker, - startSubprocessWorker, + startSubprocessWorkerNode, + startSubprocessWorkerXS, writeSlogObject, - startXsWorker, } = kernelEndowments; const { verbose } = kernelOptions; const logStartup = verbose ? console.debug : () => 0; @@ -613,8 +613,8 @@ export default function buildKernel(kernelEndowments, kernelOptions = {}) { transformMetering, waitUntilQuiescent, makeNodeWorker, - startSubprocessWorker, - startXsWorker, + startSubprocessWorkerNode, + startSubprocessWorkerXS, }); function buildVatSyscallHandler(vatID, translators) { diff --git a/packages/SwingSet/src/kernel/vatManager/factory.js b/packages/SwingSet/src/kernel/vatManager/factory.js index 6b85e83126a..6ce792c2518 100644 --- a/packages/SwingSet/src/kernel/vatManager/factory.js +++ b/packages/SwingSet/src/kernel/vatManager/factory.js @@ -13,8 +13,8 @@ export function makeVatManagerFactory({ transformMetering, waitUntilQuiescent, makeNodeWorker, - startSubprocessWorker, - startXsWorker, + startSubprocessWorkerNode, + startSubprocessWorkerXS, }) { const localFactory = makeLocalVatManagerFactory({ allVatPowers, @@ -31,12 +31,12 @@ export function makeVatManagerFactory({ }); const nodeSubprocessFactory = makeNodeSubprocessFactory({ - startSubprocessWorker, + startSubprocessWorker: startSubprocessWorkerNode, kernelKeeper, }); const xsWorkerFactory = makeNodeSubprocessFactory({ - startSubprocessWorker: startXsWorker, + startSubprocessWorker: startSubprocessWorkerXS, kernelKeeper, }); @@ -100,7 +100,7 @@ export function makeVatManagerFactory({ } if (managerType === 'xs-worker') { - if (!xsWorkerFactory) { + if (!startSubprocessWorkerXS) { throw new Error('manager type xs-worker not available'); } return xsWorkerFactory.createFromBundle(vatID, bundle, managerOptions); diff --git a/packages/SwingSet/src/spawnSubprocessWorker.js b/packages/SwingSet/src/spawnSubprocessWorker.js index ec7dfe0534b..9bc3e0ae0ed 100644 --- a/packages/SwingSet/src/spawnSubprocessWorker.js +++ b/packages/SwingSet/src/spawnSubprocessWorker.js @@ -20,9 +20,9 @@ const supercode = require.resolve( // always be Node. const stdio = harden(['inherit', 'inherit', 'inherit', 'pipe', 'pipe']); -export function startSubprocessWorker(execPath, args) { - execPath = execPath || process.execPath; - args = args || ['-r', 'esm', supercode]; +export function startSubprocessWorker(options) { + const execPath = options.execPath || process.execPath; + const args = options.args || ['-r', 'esm', supercode]; const proc = spawn(execPath, args, { stdio }); const toChild = Netstring.writeStream(); From 2cec8f609b8f96eb52a6a47e7b5dd54bad9610fd Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 16:50:26 -0500 Subject: [PATCH 19/20] build(xs-vat-worker): sync anylogger, devDependencies - prune spurious tap-spec dep --- packages/xs-vat-worker/package.json | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/xs-vat-worker/package.json b/packages/xs-vat-worker/package.json index 6cd03807cf5..3b269c4f410 100644 --- a/packages/xs-vat-worker/package.json +++ b/packages/xs-vat-worker/package.json @@ -20,15 +20,16 @@ }, "devDependencies": { "detective-es6": "^2.2.0", - "eslint": "^6.1.0", + "eslint": "^6.8.0", + "eslint-config-airbnb": "^18.0.1", "eslint-config-airbnb-base": "^14.0.0", - "eslint-config-jessie": "0.0.3", - "eslint-config-prettier": "^6.0.0", - "eslint-plugin-import": "^2.18.2", - "eslint-plugin-prettier": "^3.1.0", + "eslint-config-jessie": "^0.0.4", + "eslint-config-prettier": "^6.9.0", + "eslint-plugin-import": "^2.20.0", + "eslint-plugin-jsx-a11y": "^6.2.3", + "eslint-plugin-prettier": "^3.1.2", "filing-cabinet": "^2.5.1", "prettier": "^1.18.2", - "tap-spec": "^5.0.0", "ava": "^3.11.1" }, "dependencies": { @@ -40,7 +41,7 @@ "@agoric/marshal": "^0.2.3", "@agoric/promise-kit": "^0.1.3", "@agoric/swingset-vat": "^0.6.0", - "anylogger": "^1.0.4", + "anylogger": "^0.21.0", "esm": "^3.2.5" }, "keywords": [], From 73b9cc04d238f830e1f6d000c4c87b1b3cb94a2e Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 25 Aug 2020 17:43:42 -0500 Subject: [PATCH 20/20] build(xs-vat-worker): yarn.lock --- yarn.lock | 197 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 0ff7993b7bc..099c01bb830 100644 --- a/yarn.lock +++ b/yarn.lock @@ -362,6 +362,11 @@ esutils "^2.0.2" js-tokens "^4.0.0" +"@babel/parser@^7.0.0": + version "7.11.4" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.11.4.tgz#6fa1a118b8b0d80d0267b719213dc947e88cc0ca" + integrity sha512-MggwidiH+E9j5Sh8pbrX5sJvMcsqS5o+7iB42M9/k0CD63MjYbdP4nhSh7uB5wnv2/RVzTZFTxzF/kIa5mrCqA== + "@babel/parser@^7.10.3": version "7.10.3" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.10.3.tgz#7e71d892b0d6e7d04a1af4c3c79d72c1f10f5315" @@ -1825,6 +1830,11 @@ anymatch@~3.1.1: normalize-path "^3.0.0" picomatch "^2.0.4" +app-module-path@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/app-module-path/-/app-module-path-2.2.0.tgz#641aa55dfb7d6a6f0a8141c4b9c0aa50b6c24dd5" + integrity sha1-ZBqlXft9am8KgUHEucCqULbCTdU= + append-transform@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/append-transform/-/append-transform-1.0.0.tgz#046a52ae582a228bd72f58acfbe2967c678759ab" @@ -1994,6 +2004,11 @@ assign-symbols@^1.0.0: resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= +ast-module-types@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/ast-module-types/-/ast-module-types-2.6.0.tgz#f9f367fd273bbe01e52f2c51b5f46b65801d5d7f" + integrity sha512-zXSoVaMrf2R+r+ISid5/9a8SXm1LLdkhHzh6pSRhj9jklzruOOl1hva1YmFT33wAstg/f9ZndJAlq1BSrFLSGA== + ast-types-flow@0.0.7, ast-types-flow@^0.0.7: version "0.0.7" resolved "https://registry.yarnpkg.com/ast-types-flow/-/ast-types-flow-0.0.7.tgz#f70b735c6bca1a5c9c22d982c3e39e7feba3bdad" @@ -2827,7 +2842,7 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@2, commander@^2.11.0, commander@^2.20.0, commander@~2.20.3: +commander@2, commander@^2.11.0, commander@^2.13.0, commander@^2.16.0, commander@^2.20.0, commander@^2.8.1, commander@~2.20.3: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== @@ -3534,6 +3549,13 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= +decomment@^0.9.2: + version "0.9.3" + resolved "https://registry.yarnpkg.com/decomment/-/decomment-0.9.3.tgz#b913f32e5fe1113848f516caa5c7afefa9544d38" + integrity sha512-5skH5BfUL3n09RDmMVaHS1QGCiZRnl2nArUwmsE9JRY93Ueh3tihYl5wIrDdAuXnoFhxVis/DmRWREO2c6DG3w== + dependencies: + esprima "4.0.1" + decompress-response@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" @@ -3690,6 +3712,13 @@ detect-libc@^1.0.2, detect-libc@^1.0.3: resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= +detective-es6@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/detective-es6/-/detective-es6-2.2.0.tgz#8f2baba3f8cd90a5cfd748f5ac436f0158ed2585" + integrity sha512-fSpNY0SLER7/sVgQZ1NxJPwmc9uCTzNgdkQDhAaj8NPYwr7Qji9QBcmbNvtMCnuuOGMuKn3O7jv0An+/WRWJZQ== + dependencies: + node-source-walk "^4.0.0" + detective@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/detective/-/detective-5.2.0.tgz#feb2a77e85b904ecdea459ad897cc90a99bd2a7b" @@ -3882,6 +3911,15 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: dependencies: once "^1.4.0" +enhanced-resolve@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz#3b806f3bfafc1ec7de69551ef93cca46c1704126" + integrity sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ== + dependencies: + graceful-fs "^4.1.2" + memory-fs "^0.5.0" + tapable "^1.0.0" + entities@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.0.0.tgz#68d6084cab1b079767540d80e56a39b423e4abf4" @@ -3907,6 +3945,13 @@ err-code@^1.0.0: resolved "https://registry.yarnpkg.com/err-code/-/err-code-1.1.2.tgz#06e0116d3028f6aef4806849eb0ea6a748ae6960" integrity sha1-BuARbTAo9q70gGhJ6w6mp0iuaWA= +errno@^0.1.3: + version "0.1.7" + resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.7.tgz#4684d71779ad39af177e3f007996f7c67c852618" + integrity sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg== + dependencies: + prr "~1.0.1" + error-ex@^1.2.0, error-ex@^1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -4252,7 +4297,7 @@ espree@^6.1.2: acorn-jsx "^5.1.0" eslint-visitor-keys "^1.1.0" -esprima@^4.0.0, esprima@~4.0.0: +esprima@4.0.1, esprima@^4.0.0, esprima@~4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -4525,11 +4570,35 @@ file-entry-cache@^5.0.1: dependencies: flat-cache "^2.0.1" +file-exists-dazinatorfork@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/file-exists-dazinatorfork/-/file-exists-dazinatorfork-1.0.2.tgz#cd8d0d85f63e39dc81eceb0b687c44a2cca95c47" + integrity sha512-r70c72ln2YHzQINNfxDp02hAhbGkt1HffZ+Du8oetWDLjDtFja/Lm10lUaSh9e+wD+7VDvPee0b0C9SAy8pWZg== + file-uri-to-path@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== +filing-cabinet@^2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/filing-cabinet/-/filing-cabinet-2.5.1.tgz#f920976d46310710595ed995f033a301570ef6ab" + integrity sha512-GWOdObzou2L0HrJUk8MpJa01q0ZOwuTwTssM2+P+ABJWEGlVWd6ueEatANFdin94/3rdkVSdqpH14VqCNqp3RA== + dependencies: + app-module-path "^2.2.0" + commander "^2.13.0" + debug "^4.1.1" + decomment "^0.9.2" + enhanced-resolve "^4.1.0" + is-relative-path "^1.0.2" + module-definition "^3.0.0" + module-lookup-amd "^6.1.0" + resolve "^1.11.1" + resolve-dependency-path "^2.0.0" + sass-lookup "^3.0.0" + stylus-lookup "^3.0.1" + typescript "^3.0.3" + fill-range@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" @@ -4607,6 +4676,13 @@ find-yarn-workspace-root@^1.2.1: fs-extra "^4.0.3" micromatch "^3.1.4" +find@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/find/-/find-0.3.0.tgz#4082e8fc8d8320f1a382b5e4f521b9bc50775cb8" + integrity sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw== + dependencies: + traverse-chain "~0.1.0" + findit@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/findit/-/findit-2.0.0.tgz#6509f0126af4c178551cfa99394e032e13a4d56e" @@ -4846,6 +4922,11 @@ get-caller-file@^2.0.1: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-own-enumerable-property-symbols@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz#b5fde77f22cbe35f390b4e089922c50bce6ef664" + integrity sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g== + get-pkg-repo@^1.0.0: version "1.4.0" resolved "https://registry.yarnpkg.com/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz#c73b489c06d80cc5536c2c853f9e05232056972d" @@ -5777,7 +5858,7 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-obj@^1.0.0: +is-obj@^1.0.0, is-obj@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-1.0.1.tgz#3e4729ac1f5fde025cd7d83a896dab9f4f67db0f" integrity sha1-PkcprB9f3gJc19g6iW2rn09n2w8= @@ -5852,6 +5933,16 @@ is-regex@^1.0.5: dependencies: has "^1.0.3" +is-regexp@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-regexp/-/is-regexp-1.0.0.tgz#fd2d883545c46bac5a633e7b9a09e87fa2cb5069" + integrity sha1-/S2INUXEa6xaYz57mgnof6LLUGk= + +is-relative-path@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-relative-path/-/is-relative-path-1.0.2.tgz#091b46a0d67c1ed0fe85f1f8cfdde006bb251d46" + integrity sha1-CRtGoNZ8HtD+hfH4z93gBrslHUY= + is-resolvable@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.1.0.tgz#fb18f87ce1feb925169c9a407c19318a3206ed88" @@ -6634,6 +6725,14 @@ mem@^6.1.0: map-age-cleaner "^0.1.3" mimic-fn "^3.0.0" +memory-fs@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.5.0.tgz#324c01288b88652966d161db77838720845a8e3c" + integrity sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA== + dependencies: + errno "^0.1.3" + readable-stream "^2.0.1" + meow@^3.3.0: version "3.7.0" resolved "https://registry.yarnpkg.com/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb" @@ -6932,6 +7031,26 @@ modify-values@^1.0.0: resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== +module-definition@^3.0.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/module-definition/-/module-definition-3.3.0.tgz#aae06d68c99c5f93841e59b8a4469b974956d4d4" + integrity sha512-HTplA9xwDzH67XJFC1YvZMUElWJD28DV0dUq7lhTs+JKJamUOWA/CcYWSlhW5amJO66uWtY7XdltT+LfX0wIVg== + dependencies: + ast-module-types "^2.6.0" + node-source-walk "^4.0.0" + +module-lookup-amd@^6.1.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/module-lookup-amd/-/module-lookup-amd-6.2.0.tgz#70600008b3f26630fde9ef9ae6165ac69de6ecbb" + integrity sha512-uxHCj5Pw9psZiC1znjU2qPsubt6haCSsN9m7xmIdoTciEgfxUkE1vhtDvjHPuOXEZrVJhjKgkmkP+w73rRuelQ== + dependencies: + commander "^2.8.1" + debug "^4.1.0" + file-exists-dazinatorfork "^1.0.2" + find "^0.3.0" + requirejs "^2.3.5" + requirejs-config-file "^3.1.1" + morgan@^1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/morgan/-/morgan-1.9.1.tgz#0a8d16734a1d9afbc824b99df87e738e58e2da59" @@ -7182,6 +7301,13 @@ node-releases@^1.1.58: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.60.tgz#6948bdfce8286f0b5d0e5a88e8384e954dfe7084" integrity sha512-gsO4vjEdQaTusZAEebUWp2a5d7dF5DYoIpDG7WySnk7BuZDW+GPpHXoXXuYawRBr/9t5q54tirPz79kFIWg4dA== +node-source-walk@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/node-source-walk/-/node-source-walk-4.2.0.tgz#c2efe731ea8ba9c03c562aa0a9d984e54f27bc2c" + integrity sha512-hPs/QMe6zS94f5+jG3kk9E7TNm4P2SulrKiLWMzKszBfNZvL/V6wseHlTd7IvfW0NZWqPtK3+9yYNr+3USGteA== + dependencies: + "@babel/parser" "^7.0.0" + noop-logger@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/noop-logger/-/noop-logger-0.1.1.tgz#94a2b1633c4f1317553007d8966fd0e841b6a4c2" @@ -8542,6 +8668,11 @@ proxy-addr@~2.0.5: forwarded "~0.1.2" ipaddr.js "1.9.0" +prr@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" + integrity sha1-0/wRS6BplaRexok/SEzrHXj19HY= + pseudomap@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" @@ -8808,7 +8939,7 @@ read@1, read@~1.0.1: dependencies: mute-stream "~0.0.4" -"readable-stream@1 || 2", readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.1, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -9009,6 +9140,20 @@ requireindex@^1.2.0: resolved "https://registry.yarnpkg.com/requireindex/-/requireindex-1.2.0.tgz#3463cdb22ee151902635aa6c9535d4de9c2ef1ef" integrity sha512-L9jEkOi3ASd9PYit2cwRfyppc9NoABujTP8/5gFcbERmo5jUoAKovIC3fsF17pkTnGsrByysqX+Kxd2OTNI1ww== +requirejs-config-file@^3.1.1: + version "3.1.2" + resolved "https://registry.yarnpkg.com/requirejs-config-file/-/requirejs-config-file-3.1.2.tgz#de8c0b3eebdf243511c994a8a24b006f8b825997" + integrity sha512-sdLWywcDuNz7EIOhenSbRfT4YF84nItDv90coN2htbokjmU2QeyQuSBZILQUKNksepl8UPVU+hgYySFaDxbJPQ== + dependencies: + esprima "^4.0.0" + make-dir "^2.1.0" + stringify-object "^3.2.1" + +requirejs@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/requirejs/-/requirejs-2.3.6.tgz#e5093d9601c2829251258c0b9445d4d19fa9e7c9" + integrity sha512-ipEzlWQe6RK3jkzikgCupiTbTvm4S0/CAU5GlgptkN5SO6F3u0UD0K18wy6ErDqiCyP4J4YYe1HuAShvsxePLg== + resolve-cwd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-2.0.0.tgz#00a9f7387556e27038eae232caa372a6a59b665a" @@ -9023,6 +9168,11 @@ resolve-cwd@^3.0.0: dependencies: resolve-from "^5.0.0" +resolve-dependency-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/resolve-dependency-path/-/resolve-dependency-path-2.0.0.tgz#11700e340717b865d216c66cabeb4a2a3c696736" + integrity sha512-DIgu+0Dv+6v2XwRaNWnumKu7GPufBBOr5I1gRPJHkvghrfCGOooJODFvgFimX/KRxk9j0whD2MnKHzM1jYvk9w== + resolve-from@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-3.0.0.tgz#b22c7af7d9d6881bc8b6e653335eebcb0a188748" @@ -9332,6 +9482,13 @@ safe-regex@^1.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sass-lookup@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/sass-lookup/-/sass-lookup-3.0.0.tgz#3b395fa40569738ce857bc258e04df2617c48cac" + integrity sha512-TTsus8CfFRn1N44bvdEai1no6PqdmDiQUiqW5DlpmtT+tYnIt1tXtDIph5KA1efC+LmioJXSnCtUVpcK9gaKIg== + dependencies: + commander "^2.16.0" + sax@^1.2.4, sax@~1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9930,6 +10087,15 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" +stringify-object@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629" + integrity sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw== + dependencies: + get-own-enumerable-property-symbols "^3.0.0" + is-obj "^1.0.1" + is-regexp "^1.0.0" + strip-ansi@^3.0.0, strip-ansi@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" @@ -10027,6 +10193,14 @@ stylehacks@^4.0.0: postcss "^7.0.0" postcss-selector-parser "^3.0.0" +stylus-lookup@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/stylus-lookup/-/stylus-lookup-3.0.2.tgz#c9eca3ff799691020f30b382260a67355fefdddd" + integrity sha512-oEQGHSjg/AMaWlKe7gqsnYzan8DLcGIHe0dUaFkucZZ14z4zjENRlQMCHT4FNsiWnJf17YN9OvrCfCoi7VvOyg== + dependencies: + commander "^2.8.1" + debug "^4.1.0" + supercop.js@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/supercop.js/-/supercop.js-2.0.1.tgz#1fcfe9fc5ff6e42aef4e3683636c8cb891594b18" @@ -10253,6 +10427,11 @@ tap@^14.10.5: yaml "^1.7.2" yapool "^1.0.0" +tapable@^1.0.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/tapable/-/tapable-1.1.3.tgz#a1fccc06b58db61fd7a45da2da44f5f3a3e67ba2" + integrity sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA== + tape-promise@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/tape-promise/-/tape-promise-4.0.0.tgz#c1f3553959b2e9d64b1546e7276b8a017c616897" @@ -10526,6 +10705,11 @@ tr46@^1.0.1: dependencies: punycode "^2.1.0" +traverse-chain@~0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/traverse-chain/-/traverse-chain-0.1.0.tgz#61dbc2d53b69ff6091a12a168fd7d433107e40f1" + integrity sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE= + treport@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/treport/-/treport-1.0.2.tgz#5f99e68198982984415434a2a84df2af2dd7171d" @@ -10640,6 +10824,11 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= +typescript@^3.0.3: + version "3.9.7" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.7.tgz#98d600a5ebdc38f40cb277522f12dc800e9e25fa" + integrity sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw== + typescript@^3.7.2: version "3.8.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.8.3.tgz#409eb8544ea0335711205869ec458ab109ee1061"