From 01f07b3e42a558a29761e5210247264fd80a1430 Mon Sep 17 00:00:00 2001 From: Michael FIG Date: Sat, 6 Jan 2024 09:53:22 -0600 Subject: [PATCH] build(agd): rely on checksums, not timestamps --- .github/actions/restore-node/action.yml | 44 ++++++++++++++++++---- bin/agd | 49 ++++++++++++------------- 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/.github/actions/restore-node/action.yml b/.github/actions/restore-node/action.yml index bbc6d93b0b4..effedbbc0f5 100644 --- a/.github/actions/restore-node/action.yml +++ b/.github/actions/restore-node/action.yml @@ -72,13 +72,17 @@ runs: - name: Move Endo checkout outside the working directory id: endo-sha run: |- - set -e + set -ex if test -e ./replacement-endo; then mv ./replacement-endo ~/endo - echo "sha=$(cd ~/endo && git rev-parse HEAD)" >> $GITHUB_OUTPUT + sha=$(cd ~/endo && git rev-parse HEAD) else - echo "sha=NOPE" >> $GITHUB_OUTPUT + sha=NOPE fi + echo "sha=$sha" >> $GITHUB_OUTPUT + cd "${{ inputs.path }}" + echo "$sha" > endo-sha.txt + git add endo-sha.txt shell: bash - name: merge endo integration branch id: endo-integration-merge @@ -99,7 +103,9 @@ runs: with: node-version: ${{ inputs.node-version }} cache: yarn - cache-dependency-path: ${{ inputs.path }}/yarn.lock + cache-dependency-path: | + ${{ inputs.path }}/yarn.lock + ${{ inputs.path }}/endo-sha.txt - uses: kenchan0130/actions-system-info@master id: system-info - name: restore built files @@ -113,7 +119,7 @@ runs: - name: yarn install working-directory: ${{ inputs.path }} run: |- - set -e + set -ex if test "${{ inputs.xsnap-random-init }}" != 0 && test -d /etc/apt; then # Need libbsd-dev, as it's referenced in xsnapPlatform.h sudo apt-get update @@ -123,9 +129,33 @@ runs: if test -e ~/endo; then scripts/get-packed-versions.sh ~/endo | scripts/resolve-versions.sh fi + cp package.json package.json.orig yarn install + + if cmp -s package.json.orig package.json; then + # We need to generate a yarn-installed.sum that matches the + # unmodified package.json. As long as we don't explicitly + # `yarn install` anywhere other than in this action and in + # the bin/agd script, we should be able to reuse even + # Endo-overridden built caches successfully. + git checkout HEAD -- package.json + fi + rm package.json.orig + + function findsha1() { + find ${1+"$@"} -print0 | xargs -0 sha1sum | sort +1 || true + } + + # Find the current list of package.jsons. + files=( package.json ) + while IFS= read -r line; do + files+=( "$line" ) + done < <(yarn -s workspaces info | + sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }') + mkdir -p node_modules/.cache/agoric - date > node_modules/.cache/agoric/yarn-installed + findsha1 "${files[@]}" > node_modules/.cache/agoric/yarn-installed.sum + if test -e ~/endo; then # Stage the redirected `yarn install` consequences. git add package.json yarn.lock @@ -136,7 +166,7 @@ runs: - name: yarn build working-directory: ${{ inputs.path }} run: |- - set -e + set -ex yarn build mkdir -p node_modules/.cache/agoric date > node_modules/.cache/agoric/yarn-built diff --git a/bin/agd b/bin/agd index fb41d863896..35e412a79b3 100755 --- a/bin/agd +++ b/bin/agd @@ -19,6 +19,10 @@ function fatal() { exit 1 } +function findsha1() { + find ${1+"$@"} -print0 | xargs -0 sha1sum | sort +1 || true +} + GVM_URL=${GVM_URL-https://github.com/devnw/gvm/releases/download/latest/gvm} NVM_GIT_REPO=${NVM_GIT_REPO-https://github.com/nvm-sh/nvm.git} @@ -45,7 +49,7 @@ fi NEED_NODEJS=$BUILD_ONLY case $@ in -*" start" | *" start "*) +start | *" start" | *" start "*) # We're starting the daemon, so we need Node.js. NEED_NODEJS=true ;; @@ -96,6 +100,7 @@ fi $BUILD_ONLY || exec 1>&2 cd "$thisdir/.." + test -d "$STAMPS" || echo "Creating $STAMPS" 1>&2 mkdir -p "$STAMPS" if $NEED_NODEJS; then @@ -109,14 +114,8 @@ fi yarn "$@" } - # Check if any package.json is newer than the installation stamp. - stamp=$STAMPS/yarn-installed - if test -e "$stamp"; then - print=( -newer "$stamp" ) - else - print=() - fi - print+=( -print ) + # Check if any package.json sums are different. + sum=$STAMPS/yarn-installed.sum # Find the current list of package.jsons. files=( package.json ) @@ -125,13 +124,14 @@ fi done < <(lazy_yarn -s workspaces info | sed -ne '/"location":/{ s/.*": "//; s!",.*!/package.json!; p; }') - src=$(find "${files[@]}" "${print[@]}" | head -1 || true) - test -z "$src" || { - echo "At least $src is newer than node_modules" - rm -f "$STAMPS/yarn-built" + findsha1 "${files[@]}" > "$sum.new" || true + diff -u "$sum" "$sum.new" || { + echo "$sum has changed" + rm -f "$sum" "$STAMPS/yarn-built" lazy_yarn install - date > "$stamp" + mv "$sum.new" "$sum" } + rm -f "$sum.new" stamp=$STAMPS/yarn-built test -e "$stamp" || { @@ -141,17 +141,14 @@ fi } fi - stamp=$GOLANG_DAEMON - if test -e "$stamp"; then - print=( -newer "$stamp" ) - else - print=() - fi - print+=( -print ) - src=$(find "$GOLANG_DIR" \( ! -name '*_test.go' -name '*.go' -o -name '*.cc' -o -name 'go.*' \) "${print[@]}" | head -1 || true) - test -z "$src" || { - echo "At least $src is newer than $stamp" - + sum=$STAMPS/golang-built.sum + findsha1 "$GOLANG_DIR" \( \ + ! -name '*_test.go' -name '*.go' \ + -o -name '*.cc' \ + -o -name 'go.*' \) > "$sum.new" || true + diff -u "$sum" "$sum.new" || { + echo "$GOLANG_DIR $sum has changed" + rm -f "$sum" ( # Run this build in another subshell in case we had to modify the path. case $(go version 2>/dev/null) in @@ -181,7 +178,9 @@ fi cd "$GOLANG_DIR" make ) + mv "$sum.new" "$sum" } + rm -f "$sum.new" ) # the xsnap binary lives in a platform-specific directory