Merge pull request #8107 from Agoric/mhofman/8081-replicate-state-syn…

…c-issue

feat(deployment): replicate broken state sync in docker upgrade test

packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh

@@ -174,6 +174,16 @@ OFFER=$(mktemp -t agops.XXX)
 agops vaults close --vaultId vault2 --giveMinted 5.75 --from $USER2ADDR --keyring-backend="test" >|"$OFFER"
 agops perf satisfaction --from "$USER2ADDR" --executeOffer "$OFFER" --keyring-backend=test
 
-# # TODO test bidding
-# # TODO liquidations
-# # agops inter bid by-price --price 1 --give 1.0IST  --from $GOV1ADDR --keyring-backend test
+# replicate state-sync of node
+# this will cause the swing-store to prune some data
+killAgd
+EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-10-XXX)
+make_swing_store_snapshot $EXPORT_DIR || fail "Couldn't make swing-store snapshot"
+test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data"
+restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot"
+rm -rf $EXPORT_DIR
+startAgd
+
+# # TODO fully test bidding
+# # TODO test liquidations
+agops inter bid by-price --price 1 --give 1.0IST  --from $GOV1ADDR --keyring-backend test

packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh

@@ -106,3 +106,50 @@ submitDeliverInbound() {
       --chain-id="$CHAINID" -ojson --yes \
       --from="$sender" --keyring-backend=test -b block
 }
+
+make_swing_store_snapshot() {( set -euo pipefail
+  EXPORT_DIR="$1"
+  /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --include-export-data
+
+  EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json"
+  EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)"
+  EXPORT_DATA_UNTRUSTED_FILE="${EXPORT_DATA_FILE%.*}-untrusted.jsonl"
+  EXPORT_HEIGHT=$(cat "$EXPORT_MANIFEST_FILE" | jq -r .blockHeight)
+  EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)"
+
+  mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE"
+  agd export --height $EXPORT_HEIGHT | jq -cr '.app_state.vstorage.data[] | if .path | startswith("swingStore.") then [.path[11:],.value] else empty end' > "$EXPORT_DATA_FILE"
+
+  jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE"
+
+  echo "Successful swing-store export for block $EXPORT_HEIGHT"
+)}
+
+restore_swing_store_snapshot() {( set -euo pipefail
+  rm -f $HOME/.agoric/data/agoric/swingstore.sqlite
+
+  /usr/src/agoric-sdk/packages/cosmic-swingset/src/import-kernel-db.js --home "$HOME/.agoric" --export-dir "$1" --verbose
+)}
+
+compare_swing_store_export_data() {
+  EXPORT_DIR="$1"
+  EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json"
+  EXPORT_DATA_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)"
+  EXPORT_DATA_UNTRUSTED_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .untrustedData)"
+
+  if [ -z "$EXPORT_DATA_FILE" ]; then
+    echo "missing-export-data"
+    return
+  fi
+
+  if [ -z "$EXPORT_DATA_UNTRUSTED_FILE" ]; then
+    echo "missing-untrusted-export-data"
+    return
+  fi
+
+  diff <(cat "$EXPORT_DIR/$EXPORT_DATA_FILE" | sort) <(cat "$EXPORT_DIR/$EXPORT_DATA_UNTRUSTED_FILE" | sort) >&2 && {
+    echo "match"
+  } || {
+    echo "mismatch"
+  }
+}

packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/test.sh

@@ -35,3 +35,11 @@ test_val $(agoric follow -l -F :published.vaultFactory.managers.manager0.vaults.
 test_val $(agoric follow -l -F :published.vaultFactory.managers.manager0.vaults.vault2 -o jsonlines | jq -r '.vaultState') "closed" "vault2 is closed"
 test_val $(agoric follow -l -F :published.vaultFactory.managers.manager0.vaults.vault2 -o jsonlines | jq -r '.locked.value') "0" "vault2 contains no collateral"
 test_val $(agoric follow -l -F :published.vaultFactory.managers.manager0.vaults.vault2 -o jsonlines | jq -r '.debtSnapshot.debt.value') "0" "vault2 has no debt"
+
+# verify state-sync would be broken
+killAgd
+EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-10-XXX)
+make_swing_store_snapshot $EXPORT_DIR || fail "Couldn't make swing-store snapshot"
+test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "mismatch" "swing-store broken state-sync"
+rm -rf $EXPORT_DIR
+startAgd

packages/deployment/upgrade-test/upgrade-test-scripts/env_setup.sh

@@ -54,6 +54,26 @@ if [[ "$binary" == "agd" ]]; then
   sed -i 's/minSubmissionCount": 3/minSubmissionCount": 1/g' /usr/src/agoric-sdk/packages/vats/*.json
 fi
 
+startAgd() {
+  agd start --log_level warn "$@" &
+  AGD_PID=$!
+  echo $AGD_PID > $HOME/.agoric/agd.pid
+  wait_for_bootstrap
+  waitForBlock 2
+}
+
+killAgd() {
+  AGD_PID=$(cat $HOME/.agoric/agd.pid)
+  kill $AGD_PID
+  rm $HOME/.agoric/agd.pid
+  wait $AGD_PID || true
+}
+
+waitAgd() {
+  wait $(cat $HOME/.agoric/agd.pid)
+  rm $HOME/.agoric/agd.pid
+}
+
 provisionSmartWallet() {
   i="$1"
   amount="$2"

packages/deployment/upgrade-test/upgrade-test-scripts/start_to_to.sh

@@ -3,7 +3,7 @@
 grep -qF 'env_setup.sh' /root/.bashrc || echo ". ./upgrade-test-scripts/env_setup.sh" >> /root/.bashrc
 grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >> /root/.bashrc
 
-tmux -V || apt install -y tmux
+tmux -V 2>/dev/null || apt-get install -y tmux
 
 if [[ "$DEST" == "1" ]] && [[ "$TMUX" == "" ]]; then
   echo "launching entrypoint"
@@ -14,10 +14,7 @@ fi
 
 . ./upgrade-test-scripts/env_setup.sh
 
-agd start --log_level warn &
-AGD_PID=$!
-wait_for_bootstrap
-waitForBlock 2
+startAgd
 
 if ! test -f "$HOME/.agoric/runActions-${THIS_NAME}"; then
   runActions "pre_test"
@@ -61,9 +58,9 @@ if [[ "$DEST" != "1" ]]; then
   done
 
   sleep 2
-  kill $AGD_PID
+  killAgd
   echo "ready for upgrade to $UPGRADE_TO"
 else
 
-  wait $AGD_PID
+  waitAgd
 fi