From 12e1f48483776f5808cdee40b78f9a7f1575b61b Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Wed, 27 Mar 2024 17:14:45 -0700 Subject: [PATCH 1/5] feat(xsnap): Build sensitivity to presence of build toolchain --- packages/xsnap/package.json | 9 ++- packages/xsnap/scripts/test-package.sh | 21 ++++++ packages/xsnap/src/build.js | 98 ++++++++++++++++++++++++-- packages/xsnap/src/xsnap.js | 2 +- 4 files changed, 124 insertions(+), 6 deletions(-) create mode 100755 packages/xsnap/scripts/test-package.sh diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 40a282ce3dc..190a1a7af87 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -50,7 +50,14 @@ "LICENSE*", "api.js", "build.env", - "src" + "moddable/modules/data", + "moddable/xs/includes", + "moddable/xs/makefiles", + "moddable/xs/platforms/*.h", + "moddable/xs/sources", + "src", + "xsnap-native/xsnap/makefiles", + "xsnap-native/xsnap/sources" ], "publishConfig": { "access": "public" diff --git a/packages/xsnap/scripts/test-package.sh b/packages/xsnap/scripts/test-package.sh new file mode 100755 index 00000000000..c6ef5c8e800 --- /dev/null +++ b/packages/xsnap/scripts/test-package.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Verifies that files in package.json covers everything xsnap needs to compile +# from sources out of an npm package. +set -xueo pipefail + +TEMP=$(mktemp -d) +# function cleanup() { +# rm -rf "$TEMP" +# } +# trap cleanup EXIT + +yarn pack -f "$TEMP/package.tar" +( + cd "$TEMP" + tar xvf package.tar + cd package + yarn + time yarn build + time yarn build + time yarn build +) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index b2b95361581..28cd44312eb 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -149,12 +149,9 @@ const makeSubmodule = (path, repoUrl, { git }) => { * rmdirSync: typeof import('fs').rmdirSync, * readFile: typeof import('fs').promises.readFile, * }, - * os: { - * type: typeof import('os').type, - * } * }} io */ -async function main(args, { env, stdout, spawn, fs, os }) { +const updateSubmodules = async (args, { env, stdout, spawn, fs }) => { const git = makeCLI('git', { spawn }); // When changing/adding entries here, make sure to search the whole project @@ -214,7 +211,22 @@ async function main(args, { env, stdout, spawn, fs, os }) { await submodule.init(); } } +}; +/** + * @param {{ + * spawn: typeof import('child_process').spawn, + * fs: { + * existsSync: typeof import('fs').existsSync, + * rmdirSync: typeof import('fs').rmdirSync, + * readFile: typeof import('fs').promises.readFile, + * }, + * os: { + * type: typeof import('os').type, + * } + * }} io + */ +const makeXsnap = async ({ spawn, fs, os }) => { const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); @@ -239,6 +251,84 @@ async function main(args, { env, stdout, spawn, fs, os }) { }, ); } +}; + +/** + * @param {string[]} args + * @param {{ + * env: Record, + * stdout: typeof process.stdout, + * spawn: typeof import('child_process').spawn, + * fs: { + * existsSync: typeof import('fs').existsSync, + * rmdirSync: typeof import('fs').rmdirSync, + * readFile: typeof import('fs').promises.readFile, + * }, + * os: { + * type: typeof import('os').type, + * } + * }} io + */ +async function main(args, { env, stdout, spawn, fs, os }) { + // I solemnly swear I will do no synchronous work followed by a variable + // number turns of the event loop. + await null; + + const osType = os.type(); + const platform = { + Linux: 'lin', + Darwin: 'mac', + // Windows_NT: 'win', // One can dream. + }[osType]; + if (platform === undefined) { + throw Error(`xsnap does not support platform ${osType}`); + } + + // If this is a working copy of xsnap in a checkout of agoric-sdk, we need to + // either clone or update submodules. + // Otherwise, we are running from an extracted npm tarball and we should not + // attempt to update Git submodules and should make the binary from the + // published source. + // + // These steps will avoid rebuilding native xsnap in the common case for end + // users. + // + // || | X || git + // || X | X || make + // || ---- | ---- || ---- + // | bin | src | .git || pack | work || + // | --- | --- | ---- || ---- | ---- || + // | | | || | X || + // | | | X || | || + // | | X | || X | || + // | | X | X || | X || + // | X | | || | || + // | X | | X || | || + // | X | X | || X | || + // | X | X | X || | X || + // + // We build both release and debug, so checking for one should suffice. + // XXX This will need to account for the .exe extension if we recover support + // for Windows. + const hasBin = fs.existsSync( + asset(`../xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker`), + ); + let hasSource = fs.existsSync(asset('moddable/xs/includes/xs.h')); + const hasGit = fs.existsSync(asset('moddable/.git')); + const isWorkingCopy = hasGit || (!hasSource && !hasBin); + + if (isWorkingCopy || args.includes('--show-env')) { + await updateSubmodules(args, { env, stdout, spawn, fs }); + hasSource = true; + } + + if (hasSource) { + await makeXsnap({ spawn, fs, os }); + } else if (!hasBin) { + throw new Error( + 'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?', + ); + } } const run = () => diff --git a/packages/xsnap/src/xsnap.js b/packages/xsnap/src/xsnap.js index 5067197da43..2c002aa23e3 100644 --- a/packages/xsnap/src/xsnap.js +++ b/packages/xsnap/src/xsnap.js @@ -98,7 +98,7 @@ export async function xsnap(options) { const platform = { Linux: 'lin', Darwin: 'mac', - Windows_NT: 'win', + // Windows_NT: 'win', // One can dream. }[os]; if (platform === undefined) { From 3b64b2a0d76e9a38bca2a152a461936899e753c6 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Mon, 1 Apr 2024 13:06:27 -0700 Subject: [PATCH 2/5] fixup more surgical --- packages/xsnap/package.json | 2 +- packages/xsnap/src/build.js | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 190a1a7af87..668961f5256 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -13,7 +13,7 @@ "scripts": { "repl": "node src/xsrepl.js", "build:bin": "if test -d ./test; then node src/build.js; else yarn build:from-env; fi", - "build:env": "test -d ./test && node src/build.js --show-env > build.env", + "build:env": "node src/build.js --show-env > build.env", "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env", "build": "yarn build:bin && yarn build:env", "postinstall": "yarn build:from-env", diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 28cd44312eb..c5ff53f06a1 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -139,7 +139,7 @@ const makeSubmodule = (path, repoUrl, { git }) => { }; /** - * @param {string[]} args + * @param {boolean} showEnv * @param {{ * env: Record, * stdout: typeof process.stdout, @@ -151,7 +151,7 @@ const makeSubmodule = (path, repoUrl, { git }) => { * }, * }} io */ -const updateSubmodules = async (args, { env, stdout, spawn, fs }) => { +const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => { const git = makeCLI('git', { spawn }); // When changing/adding entries here, make sure to search the whole project @@ -173,7 +173,7 @@ const updateSubmodules = async (args, { env, stdout, spawn, fs }) => { ]; await null; - if (args.includes('--show-env')) { + if (showEnv) { for (const submodule of submodules) { const { path, envPrefix, commitHash } = submodule; if (!commitHash) { @@ -316,9 +316,13 @@ async function main(args, { env, stdout, spawn, fs, os }) { let hasSource = fs.existsSync(asset('moddable/xs/includes/xs.h')); const hasGit = fs.existsSync(asset('moddable/.git')); const isWorkingCopy = hasGit || (!hasSource && !hasBin); + const showEnv = args.includes('--show-env'); - if (isWorkingCopy || args.includes('--show-env')) { - await updateSubmodules(args, { env, stdout, spawn, fs }); + if (isWorkingCopy || showEnv) { + if (showEnv && !isWorkingCopy) { + throw new Error('XSnap requires a working copy and git to --show-env'); + } + await updateSubmodules(showEnv, { env, stdout, spawn, fs }); hasSource = true; } From fc95375149dbc4edcaaa5c60d2e792d88b33028f Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Mon, 1 Apr 2024 13:56:04 -0700 Subject: [PATCH 3/5] fixup fix --- packages/xsnap/src/build.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index c5ff53f06a1..1f227a66fa9 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -313,8 +313,8 @@ async function main(args, { env, stdout, spawn, fs, os }) { const hasBin = fs.existsSync( asset(`../xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker`), ); - let hasSource = fs.existsSync(asset('moddable/xs/includes/xs.h')); - const hasGit = fs.existsSync(asset('moddable/.git')); + let hasSource = fs.existsSync(asset('../moddable/xs/includes/xs.h')); + const hasGit = fs.existsSync(asset('../moddable/.git')); const isWorkingCopy = hasGit || (!hasSource && !hasBin); const showEnv = args.includes('--show-env'); From aebb5c614e1f1f7a1cd23ee36e2876fb85fdbe14 Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Mon, 1 Apr 2024 14:23:48 -0700 Subject: [PATCH 4/5] fixup fixes --- packages/xsnap/scripts/test-package.sh | 8 ++++---- packages/xsnap/src/build.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/xsnap/scripts/test-package.sh b/packages/xsnap/scripts/test-package.sh index c6ef5c8e800..5b90bb4f78b 100755 --- a/packages/xsnap/scripts/test-package.sh +++ b/packages/xsnap/scripts/test-package.sh @@ -14,8 +14,8 @@ yarn pack -f "$TEMP/package.tar" cd "$TEMP" tar xvf package.tar cd package - yarn - time yarn build - time yarn build - time yarn build + time yarn + time yarn + time yarn + time yarn ) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 1f227a66fa9..f1891bbbad2 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -326,7 +326,7 @@ async function main(args, { env, stdout, spawn, fs, os }) { hasSource = true; } - if (hasSource) { + if (hasSource && !showEnv) { await makeXsnap({ spawn, fs, os }); } else if (!hasBin) { throw new Error( From efbaf26894e8ff0166f96d5e2408f34a2b8a318a Mon Sep 17 00:00:00 2001 From: Kris Kowal Date: Mon, 1 Apr 2024 14:57:36 -0700 Subject: [PATCH 5/5] show env suppress make --- packages/xsnap/src/build.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index f1891bbbad2..3bd1d4c0454 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -326,12 +326,14 @@ async function main(args, { env, stdout, spawn, fs, os }) { hasSource = true; } - if (hasSource && !showEnv) { - await makeXsnap({ spawn, fs, os }); - } else if (!hasBin) { - throw new Error( - 'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?', - ); + if (!showEnv) { + if (hasSource) { + await makeXsnap({ spawn, fs, os }); + } else if (!hasBin) { + throw new Error( + 'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?', + ); + } } }