From 1617fee9f0a3e22228faa17f7363acff46b7d102 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sat, 6 Jan 2024 09:53:22 -0600 Subject: [PATCH 1/4] build(agd): rely on checksums, not timestamps --- .github/actions/restore-node/action.yml | 59 ++++++++++++++++--- bin/agd | 78 ++++++++++++++----------- scripts/smoketest-binaries.sh | 2 +- 3 files changed, 97 insertions(+), 42 deletions(-) diff --git a/.github/actions/restore-node/action.yml b/.github/actions/restore-node/action.yml index bbc6d93b0b4..6d7bd9c6116 100644 --- a/.github/actions/restore-node/action.yml +++ b/.github/actions/restore-node/action.yml @@ -72,13 +72,17 @@ runs: - name: Move Endo checkout outside the working directory id: endo-sha run: |- - set -e + set -ex if test -e ./replacement-endo; then mv ./replacement-endo ~/endo - echo "sha=$(cd ~/endo && git rev-parse HEAD)" >> $GITHUB_OUTPUT + sha=$(cd ~/endo && git rev-parse HEAD) else - echo "sha=NOPE" >> $GITHUB_OUTPUT + sha=NOPE fi + echo "sha=$sha" >> $GITHUB_OUTPUT + cd "${{ inputs.path }}" + echo "$sha" > endo-sha.txt + git add endo-sha.txt shell: bash - name: merge endo integration branch id: endo-integration-merge @@ -99,7 +103,9 @@ runs: with: node-version: ${{ inputs.node-version }} cache: yarn - cache-dependency-path: ${{ inputs.path }}/yarn.lock + cache-dependency-path: | + ${{ inputs.path }}/yarn.lock + ${{ inputs.path }}/endo-sha.txt - uses: kenchan0130/actions-system-info@master id: system-info - name: restore built files @@ -113,7 +119,7 @@ runs: - name: yarn install working-directory: ${{ inputs.path }} run: |- - set -e + set -ex if test "${{ inputs.xsnap-random-init }}" != 0 && test -d /etc/apt; then # Need libbsd-dev, as it's referenced in xsnapPlatform.h sudo apt-get update @@ -124,8 +130,45 @@ runs: scripts/get-packed-versions.sh ~/endo | scripts/resolve-versions.sh fi yarn install - mkdir -p node_modules/.cache/agoric - date > node_modules/.cache/agoric/yarn-installed + + if ! cmp -s <(git cat-file blob HEAD:package.json) package.json; then + # In the event that the package.json has been modified by Endo + # replacements, we need to have a yarn-installed.sum that matches + # the unmodified package.json. As long as we don't explicitly `yarn + # install` anywhere other than in this action and in the bin/agd + # script, we should be able to reuse even Endo-overridden built caches + # successfully. + git checkout HEAD -- package.json + fi + + function diffsha1() { + stamp=$1 + shift + find ${1+"$@"} -exec sha1sum {} \; | sort +1 > "$stamp.new" || true + if test ! -s "$stamp.new"; then + echo "No new dependencies found for $stamp" 1>&2 + return 0 + fi + diff -u "$stamp" "$stamp.new" || return 1 + return 0 + } + + # Find the current list of package.jsons. + files=( package.json ) + while IFS= read -r line; do + files+=( "$line" ) + done < <(yarn -s workspaces info | + sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }') + + STAMPS=node_modules/.cache/agoric + + mkdir -p "$STAMPS" + sum="$STAMPS/yarn-installed.sum" + diffsha1 "$sum" "${files[@]}" || { + mv "$sum.new" "$sum" + } + rm -f "$sum.new" + if test -e ~/endo; then # Stage the redirected `yarn install` consequences. git add package.json yarn.lock @@ -136,7 +179,7 @@ runs: - name: yarn build working-directory: ${{ inputs.path }} run: |- - set -e + set -ex yarn build mkdir -p node_modules/.cache/agoric date > node_modules/.cache/agoric/yarn-built diff --git a/bin/agd b/bin/agd index fb41d863896..467de3ec353 100755 --- a/bin/agd +++ b/bin/agd @@ -19,6 +19,18 @@ function fatal() { exit 1 } +function diffsha1() { + stamp=$1 + shift + find ${1+"$@"} -exec sha1sum {} \; | sort +1 > "$stamp.new" || true + if test ! -s "$stamp.new"; then + echo "No new dependencies found for $stamp" 1>&2 + return 0 + fi + diff -u "$stamp" "$stamp.new" || return 1 + return 0 +} + GVM_URL=${GVM_URL-https://github.com/devnw/gvm/releases/download/latest/gvm} NVM_GIT_REPO=${NVM_GIT_REPO-https://github.com/nvm-sh/nvm.git} @@ -31,7 +43,8 @@ thisdir=$(cd "$(dirname -- "$real0")" >/dev/null && pwd -P) source "$thisdir/../repoconfig.sh" if test "${1-''}" = build; then - BUILD_ONLY=true + do_not_build=false + only_build=true # Enable shell debugging. set -x case "${2-''}" in @@ -40,14 +53,22 @@ if test "${1-''}" = build; then ;; esac else - BUILD_ONLY=false + case ${NO_BUILD-false} in + true | yes | y | 1) + do_not_build=true + ;; + *) + do_not_build=false + ;; + esac + only_build=false fi -NEED_NODEJS=$BUILD_ONLY +need_nodejs=$only_build case $@ in -*" start" | *" start "*) +start | *" start" | *" start "*) # We're starting the daemon, so we need Node.js. - NEED_NODEJS=true + need_nodejs=true ;; esac @@ -61,7 +82,7 @@ else SKIP_DOWNLOAD=true fi -if $NEED_NODEJS; then +if $need_nodejs; then # We need to get node at the script top level because it's used by the daemon # as well. case $(node --version 2>/dev/null) in @@ -93,12 +114,13 @@ fi ( # Send the build output to stderr unless we're only building. This prevents # the daemon's stdout from being polluted with build output. - $BUILD_ONLY || exec 1>&2 + $only_build || exec 1>&2 cd "$thisdir/.." + test -d "$STAMPS" || echo "Creating $STAMPS" 1>&2 mkdir -p "$STAMPS" - if $NEED_NODEJS; then + if $need_nodejs; then lazy_yarn() { yarn --version 2>/dev/null 1>&2 || { # Auto-download the Yarn version we need, if allowed. @@ -109,14 +131,8 @@ fi yarn "$@" } - # Check if any package.json is newer than the installation stamp. - stamp=$STAMPS/yarn-installed - if test -e "$stamp"; then - print=( -newer "$stamp" ) - else - print=() - fi - print+=( -print ) + # Check if any package.json sums are different. + sum=$STAMPS/yarn-installed.sum # Find the current list of package.jsons. files=( package.json ) @@ -125,13 +141,12 @@ fi done < <(lazy_yarn -s workspaces info | sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }') - src=$(find "${files[@]}" "${print[@]}" | head -1 || true) - test -z "$src" || { - echo "At least $src is newer than node_modules" - rm -f "$STAMPS/yarn-built" + diffsha1 "$sum" "${files[@]}" || { echo "$sum has changed"; $do_not_build; } || { + rm -f "$sum" "$STAMPS/yarn-built" lazy_yarn install - date > "$stamp" + mv "$sum.new" "$sum" } + rm -f "$sum.new" stamp=$STAMPS/yarn-built test -e "$stamp" || { @@ -141,17 +156,12 @@ fi } fi - stamp=$GOLANG_DAEMON - if test -e "$stamp"; then - print=( -newer "$stamp" ) - else - print=() - fi - print+=( -print ) - src=$(find "$GOLANG_DIR" \( ! -name '*_test.go' -name '*.go' -o -name '*.cc' -o -name 'go.*' \) "${print[@]}" | head -1 || true) - test -z "$src" || { - echo "At least $src is newer than $stamp" - + sum=$STAMPS/golang-built.sum + diffsha1 "$sum" "$GOLANG_DIR" \( \ + ! -name '*_test.go' -name '*.go' \ + -o -name '*.cc' \ + -o -name 'go.*' \) || { echo "$GOLANG_DIR $sum has changed"; $do_not_build; } || { + rm -f "$sum" ( # Run this build in another subshell in case we had to modify the path. case $(go version 2>/dev/null) in @@ -182,6 +192,8 @@ fi make ) } + test -f "$sum" || mv -f "$sum.new" "$sum" + rm -f "$sum.new" ) # the xsnap binary lives in a platform-specific directory @@ -196,7 +208,7 @@ esac xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n) [[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}" -if $BUILD_ONLY; then +if $only_build; then echo "Build complete." 1>&2 exit 0 fi diff --git a/scripts/smoketest-binaries.sh b/scripts/smoketest-binaries.sh index 126b7a60c25..3b866e38293 100755 --- a/scripts/smoketest-binaries.sh +++ b/scripts/smoketest-binaries.sh @@ -4,7 +4,7 @@ binaries=( "agd version" "agoric -V" "ag-chain-cosmos version" ) for binary in "${binaries[@]}"; do - echo "Checking $binary a" + echo "Checking $binary" $binary ec=$? if [[ $ec -ne 0 ]]; then From 1e8973b118e2a433d8fa5fa96318de5f05affca1 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sun, 4 Feb 2024 20:52:45 -0600 Subject: [PATCH 2/4] fix(registry.sh): use `build:types` and `clean:types` if they exist --- scripts/registry.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/registry.sh b/scripts/registry.sh index d34751fa228..309d1500302 100755 --- a/scripts/registry.sh +++ b/scripts/registry.sh @@ -64,12 +64,16 @@ publish() { yarn build git commit --allow-empty -am "chore: prepare for publishing" + yarn lerna run build:types + # Publish the packages to our local service. # without concurrency until https://github.com/Agoric/agoric-sdk/issues/8091 yarn lerna publish --concurrency 1 prerelease --exact \ --dist-tag="$DISTTAG" --preid=dev \ --no-push --no-git-reset --no-git-tag-version --no-verify-access --yes + yarn lerna run clean:types + # Change any version prefices to an exact match, and merge our versions. VERSIONSHASH=$(jq --argfile versions <(popd >/dev/null && git cat-file blob "$VERSIONSHASH") \ '[to_entries[] | { key: .key, value: (.value | sub("^[~^]"; "")) }] From 75b6ad505fa0c60a17d2e48d143602739eaf7d42 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Mon, 5 Feb 2024 20:40:27 -0600 Subject: [PATCH 3/4] build(golang): remove sources when building Docker image --- golang/cosmos/{src => }/index.cjs | 0 golang/cosmos/package.json | 2 +- packages/deployment/Dockerfile.sdk | 3 +++ 3 files changed, 4 insertions(+), 1 deletion(-) rename golang/cosmos/{src => }/index.cjs (100%) diff --git a/golang/cosmos/src/index.cjs b/golang/cosmos/index.cjs similarity index 100% rename from golang/cosmos/src/index.cjs rename to golang/cosmos/index.cjs diff --git a/golang/cosmos/package.json b/golang/cosmos/package.json index a1ebed63ecb..ad2c1f4799b 100644 --- a/golang/cosmos/package.json +++ b/golang/cosmos/package.json @@ -5,7 +5,7 @@ "parsers": { "js": "mjs" }, - "main": "src/index.cjs", + "main": "index.cjs", "engines": { "node": ">=14.15.0" }, diff --git a/packages/deployment/Dockerfile.sdk b/packages/deployment/Dockerfile.sdk index 71b0ad65307..0078481948a 100644 --- a/packages/deployment/Dockerfile.sdk +++ b/packages/deployment/Dockerfile.sdk @@ -63,6 +63,9 @@ RUN \ # Remove dev dependencies. RUN rm -rf packages/xsnap/moddable packages/xsnap/xsnap-native/build/tmp +RUN mkdir golang/tmp && \ + mv golang/cosmos/package.json golang/cosmos/build golang/cosmos/index.cjs golang/tmp/ && \ + rm -rf golang/cosmos && mv golang/tmp golang/cosmos # FIXME: This causes bundling differences. https://github.com/endojs/endo/issues/919 # RUN yarn install --frozen-lockfile --production --network-timeout 100000 From 6769b9f8bae540c1d6d294ff6f4808c457ed6da4 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Tue, 6 Feb 2024 13:05:36 -0600 Subject: [PATCH 4/4] fix(deployment): remove dependency on `HELPER_BINARY` --- packages/deployment/ansible/config.yml | 1 - packages/deployment/ansible/copy.yml | 1 - packages/deployment/ansible/prepare-cosmos.yml | 1 - .../deployment/ansible/roles/copy/tasks/main.yml | 14 ++++++-------- .../roles/cosmos-genesis/files/faucet-helper.sh | 2 +- packages/deployment/scripts/integration-test.sh | 4 +++- 6 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/deployment/ansible/config.yml b/packages/deployment/ansible/config.yml index 295cf7eb07d..ba5548028fb 100644 --- a/packages/deployment/ansible/config.yml +++ b/packages/deployment/ansible/config.yml @@ -7,7 +7,6 @@ vars: - service: ag-chain-cosmos - APPDIR: "{{lookup('pipe', 'pwd')}}/../.." - - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd" roles: - stop - copy diff --git a/packages/deployment/ansible/copy.yml b/packages/deployment/ansible/copy.yml index 6be79ff9aa1..1b816a78c65 100644 --- a/packages/deployment/ansible/copy.yml +++ b/packages/deployment/ansible/copy.yml @@ -7,6 +7,5 @@ vars: - service: ag-chain-cosmos - APPDIR: "{{lookup('pipe', 'pwd')}}/../.." - - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd" roles: - copy diff --git a/packages/deployment/ansible/prepare-cosmos.yml b/packages/deployment/ansible/prepare-cosmos.yml index 85692fafc41..8a46bdb158b 100644 --- a/packages/deployment/ansible/prepare-cosmos.yml +++ b/packages/deployment/ansible/prepare-cosmos.yml @@ -7,7 +7,6 @@ vars: - service: ag-chain-cosmos - CHAIN_NAME: "{{ lookup('file', SETUP_HOME + '/' + service + '/chain-name.txt') }}" - - HELPER_BINARY: "{{lookup('env', 'GOBIN') or ((lookup('env', 'GOPATH') or '/usr/local') + '/bin')}}/agd" - APPDIR: "{{lookup('pipe', 'pwd')}}/../.." roles: - copy diff --git a/packages/deployment/ansible/roles/copy/tasks/main.yml b/packages/deployment/ansible/roles/copy/tasks/main.yml index 75ab448daab..0b263cac04e 100644 --- a/packages/deployment/ansible/roles/copy/tasks/main.yml +++ b/packages/deployment/ansible/roles/copy/tasks/main.yml @@ -1,12 +1,4 @@ --- -- name: Synchronize helper - synchronize: - src: '{{HELPER_BINARY}}' - dest: /usr/local/bin/ - mode: push - use_ssh_args: true - when: HELPER_BINARY is defined - - name: 'Synchronize Agoric SDK {{ AGORIC_SDK }}' synchronize: src: '{{ AGORIC_SDK }}/' @@ -19,6 +11,12 @@ mode: push use_ssh_args: true +- name: 'Symlink agd' + file: + src: '/usr/src/agoric-sdk/bin/agd' + dest: '/usr/local/bin/agd' + state: link + - name: 'Symlink ag-chain-cosmos' file: src: '/usr/src/agoric-sdk/packages/cosmic-swingset/bin/ag-chain-cosmos' diff --git a/packages/deployment/ansible/roles/cosmos-genesis/files/faucet-helper.sh b/packages/deployment/ansible/roles/cosmos-genesis/files/faucet-helper.sh index daf8d812d32..e7c370e3a95 100755 --- a/packages/deployment/ansible/roles/cosmos-genesis/files/faucet-helper.sh +++ b/packages/deployment/ansible/roles/cosmos-genesis/files/faucet-helper.sh @@ -30,7 +30,7 @@ show-faucet-address) esac networkName=$(basename "$thisdir") -if netconfig=$(curl "https://$networkName.agoric.net/network-config"); then +if netconfig=$(curl -s "https://$networkName.agoric.net/network-config"); then chainName=$(echo "$netconfig" | jq -r .chainName) read -r -a origRpcAddrs <<<"$(echo "$netconfig" | jq -r .rpcAddrs[])" else diff --git a/packages/deployment/scripts/integration-test.sh b/packages/deployment/scripts/integration-test.sh index 59025808a07..d4714804b3e 100755 --- a/packages/deployment/scripts/integration-test.sh +++ b/packages/deployment/scripts/integration-test.sh @@ -29,6 +29,8 @@ else echo "Running chain without loadgen" >&2 fi +set -x + SOLO_ADDR= VAT_CONFIG= RESULTSDIR=${RESULTSDIR-"$NETWORK_NAME/results"} @@ -69,7 +71,7 @@ then cp ag-chain-cosmos/data/genesis.json "$RESULTSDIR/genesis.json" cp "$AG_SETUP_COSMOS_HOME/ag-chain-cosmos/data/genesis.json" "$RESULTSDIR/genesis.json" cd "$LOADGEN" - SOLO_COINS=40000000000uist PATH="$thisdir/../bin:$PATH" \ + SOLO_COINS=40000000000uist PATH="$thisdir/../bin:$SDK_SRC/bin:$PATH" \ "$AG_SETUP_COSMOS_HOME/faucet-helper.sh" add-egress loadgen "$SOLO_ADDR" SLOGSENDER=@agoric/telemetry/src/otel-trace.js SOLO_SLOGSENDER="" \ SLOGSENDER_FAIL_ON_ERROR=1 SLOGSENDER_AGENT=process \