feat(deployment): add state-sync to upgrade 11 test

Agoric · Aug 15, 2023 · 38ac134 · 38ac134
1 parent a0389b8
commit 38ac134
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 2 deletions.
diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh
@@ -176,12 +176,24 @@ agops perf satisfaction --from "$USER2ADDR" --executeOffer "$OFFER" --keyring-ba
 
 # replicate state-sync of node
 # this will cause the swing-store to prune some data
+# we will save the pruned artifact for later
 killAgd
 EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-10-XXX)
 make_swing_store_snapshot $EXPORT_DIR || fail "Couldn't make swing-store snapshot"
 test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data"
+EXPORT_DIR_ALL_ARTIFACTS=$(mktemp -t -d swing-store-export-upgrade-10-all-artifacts-XXX)
+make_swing_store_snapshot $EXPORT_DIR_ALL_ARTIFACTS --export-mode archival || fail "Couldn't make swing-store snapshot for historical artifacts"
 restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot"
+(
+    cd $EXPORT_DIR_ALL_ARTIFACTS
+    mkdir $HOME/.agoric/data/agoric/swing-store-historical-artifacts
+    for i in *; do
+        [ -f $EXPORT_DIR/$i ] && continue
+        mv $i $HOME/.agoric/data/agoric/swing-store-historical-artifacts/
+    done
+)
 rm -rf $EXPORT_DIR
+rm -rf $EXPORT_DIR_ALL_ARTIFACTS
 startAgd
 
 # # TODO fully test bidding

diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh
@@ -109,7 +109,8 @@ submitDeliverInbound() {
 
 make_swing_store_snapshot() {( set -euo pipefail
   EXPORT_DIR="$1"
-  /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --include-export-data
+  shift
+  /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --include-export-data "$@"
 
   EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json"
   EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)"

diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh
@@ -2,9 +2,26 @@
 
 . ./upgrade-test-scripts/env_setup.sh
 
+# Enable debugging
+set -x
+
 # CWD is agoric-sdk
 upgrade11=./upgrade-test-scripts/agoric-upgrade-11
 
+# hacky restore of pruned artifacts
+killAgd
+EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX)
+make_swing_store_snapshot $EXPORT_DIR --artifact-mode debug || fail "Couldn't make swing-store snapshot"
+test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data"
+HISTORICAL_ARTIFACTS="$(cd $HOME/.agoric/data/agoric/swing-store-historical-artifacts/; for i in *; do echo -n "[\"$i\",\"$i\"],"; done)"
+mv -n $HOME/.agoric/data/agoric/swing-store-historical-artifacts/* $EXPORT_DIR || fail "some historical artifacts not pruned"
+mv $EXPORT_DIR/export-manifest.json $EXPORT_DIR/export-manifest-original.json
+cat $EXPORT_DIR/export-manifest-original.json | jq -r ".artifacts = .artifacts + [${HISTORICAL_ARTIFACTS%%,}] | del(.artifactMode)" > $EXPORT_DIR/export-manifest.json
+restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot"
+rmdir $HOME/.agoric/data/agoric/swing-store-historical-artifacts
+rm -rf $EXPORT_DIR
+startAgd
+
 ######################################################################
 # FIXME: remove this line when these tests don't hardcode bundle hashes.
 echo 1>&2 "FIXME: skipping zoe-full-upgrade tests"; return 0

diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh
@@ -83,4 +83,65 @@ pushPriceOnce () {
   else
     echo "ERROR: pushPrice failed (using $nextOracle)"
   fi
-}
+}
+
+export_genesis() {
+  HEIGHT_ARG=
+
+  if [ -n "$1" ]; then
+    HEIGHT_ARG="--height $1"
+    shift
+  fi
+
+  agd export $HEIGHT_ARG "$@"
+}
+
+make_swing_store_snapshot() {( set -euo pipefail
+  EXPORT_DIR="$1"
+  shift
+  /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --artifact-mode replay --export-data-mode all "$@"
+
+  EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json"
+  EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)"
+  EXPORT_DATA_UNTRUSTED_FILE="${EXPORT_DATA_FILE%.*}-untrusted.jsonl"
+  EXPORT_HEIGHT=$(cat "$EXPORT_MANIFEST_FILE" | jq -r .blockHeight)
+  EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)"
+
+  mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE"
+  export_genesis $EXPORT_HEIGHT | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE"
+
+  jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE"
+
+  echo "Successful swing-store export for block $EXPORT_HEIGHT"
+)}
+
+restore_swing_store_snapshot() {( set -euo pipefail
+  rm -f $HOME/.agoric/data/agoric/swingstore.sqlite
+  EXPORT_DIR="$1"
+  shift
+
+  /usr/src/agoric-sdk/packages/cosmic-swingset/src/import-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --artifact-mode replay --export-data-mode all "$@"
+)}
+
+compare_swing_store_export_data() {
+  EXPORT_DIR="$1"
+  EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json"
+  EXPORT_DATA_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)"
+  EXPORT_DATA_UNTRUSTED_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .untrustedData)"
+
+  if [ -z "$EXPORT_DATA_FILE" ]; then
+    echo "missing-export-data"
+    return
+  fi
+
+  if [ -z "$EXPORT_DATA_UNTRUSTED_FILE" ]; then
+    echo "missing-untrusted-export-data"
+    return
+  fi
+
+  diff <(cat "$EXPORT_DIR/$EXPORT_DATA_FILE" | sort) <(cat "$EXPORT_DIR/$EXPORT_DATA_UNTRUSTED_FILE" | sort) >&2 && {
+    echo "match"
+  } || {
+    echo "mismatch"
+  }
+}
diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh
@@ -8,6 +8,14 @@ waitForBlock 2
 # CWD is agoric-sdk
 upgrade11=./upgrade-test-scripts/agoric-upgrade-11
 
+# verify swing-store export-data is consistent
+killAgd
+EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX)
+make_swing_store_snapshot $EXPORT_DIR --artifact-mode none || fail "Couldn't make swing-store snapshot"
+test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store consistent state-sync"
+rm -rf $EXPORT_DIR
+startAgd
+
 # zoe vat is at incarnation 1
 echo "FIXME: bypassed zoe-full-upgrade validation"; return 0
 test_val "$(yarn --silent node $upgrade11/vat-status.mjs zoe)" "1" "zoe vat incarnation"