From c249a3070d7b10df6a2ebf8907b6f8109c414122 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 05:49:00 +0000 Subject: [PATCH 1/7] fix(xsnap): force rebuild if build config changes --- .dockerignore | 1 + packages/xsnap/.gitignore | 1 + packages/xsnap/build.env | 2 +- packages/xsnap/src/build.js | 24 ++++++++++++++++++++++-- packages/xsnap/xsnap-native | 2 +- 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6e6aa7d3e00..97f2dd62b43 100644 --- a/.dockerignore +++ b/.dockerignore @@ -22,6 +22,7 @@ packages/stat-logger **/_agstate .vagrant endo-sha.txt +packages/xsnap/build.config.env # When changing/adding entries here, make sure to search the whole project for # `@@AGORIC_DOCKER_SUBMODULES@@` packages/xsnap/moddable diff --git a/packages/xsnap/.gitignore b/packages/xsnap/.gitignore index 04f5262f86f..2cee34ad100 100644 --- a/packages/xsnap/.gitignore +++ b/packages/xsnap/.gitignore @@ -1,4 +1,5 @@ build +build.config.env dist test/fixture-snap-pool/ test/fixture-snap-shot.xss diff --git a/packages/xsnap/build.env b/packages/xsnap/build.env index cf11308c6fc..b816a3002cd 100644 --- a/packages/xsnap/build.env +++ b/packages/xsnap/build.env @@ -1,4 +1,4 @@ MODDABLE_URL=https://github.com/agoric-labs/moddable.git MODDABLE_COMMIT_HASH=f6c5951fc055e4ca592b9166b9ae3cbb9cca6bf0 XSNAP_NATIVE_URL=https://github.com/agoric-labs/xsnap-pub -XSNAP_NATIVE_COMMIT_HASH=2d8ccb76b8508e490d9e03972bb4c64f402d5135 +XSNAP_NATIVE_COMMIT_HASH=eef9b67da5517ed18ff9e0073b842db20924eae3 diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 3bd1d4c0454..418355405e0 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -220,6 +220,7 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => { * existsSync: typeof import('fs').existsSync, * rmdirSync: typeof import('fs').rmdirSync, * readFile: typeof import('fs').promises.readFile, + * writeFile: typeof import('fs').promises.writeFile, * }, * os: { * type: typeof import('os').type, @@ -230,6 +231,21 @@ const makeXsnap = async ({ spawn, fs, os }) => { const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); + const configEnvs = [ + `XSNAP_VERSION=${pkg.version}`, + `CC=cc "-D__has_builtin(x)=1"`, + ]; + + const configEnvFile = asset('../build.config.env'); + const existingConfigEnvs = fs.existsSync(configEnvFile) + ? await fs.readFile(configEnvFile, 'utf-8') + : ''; + + const expectedConfigEnvs = configEnvs.concat('').join('\n'); + if (existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { + await fs.writeFile(configEnvFile, expectedConfigEnvs); + } + const platform = ModdableSDK.platforms[os.type()]; if (!platform) { throw Error(`Unsupported OS found: ${os.type()}`); @@ -241,8 +257,10 @@ const makeXsnap = async ({ spawn, fs, os }) => { [ `MODDABLE=${ModdableSDK.MODDABLE}`, `GOAL=${goal}`, - `XSNAP_VERSION=${pkg.version}`, - `CC=cc "-D__has_builtin(x)=1"`, + // Any other configuration variables that affect the build output + // should be placed in `configEnvs` to force a rebuild if they change + ...configEnvs, + `EXTRA_DEPS=${configEnvFile}`, '-f', 'xsnap-worker.mk', ], @@ -263,6 +281,7 @@ const makeXsnap = async ({ spawn, fs, os }) => { * existsSync: typeof import('fs').existsSync, * rmdirSync: typeof import('fs').rmdirSync, * readFile: typeof import('fs').promises.readFile, + * writeFile: typeof import('fs').promises.writeFile, * }, * os: { * type: typeof import('os').type, @@ -344,6 +363,7 @@ const run = () => spawn: childProcessTop.spawn, fs: { readFile: fsTop.promises.readFile, + writeFile: fsTop.promises.writeFile, existsSync: fsTop.existsSync, rmdirSync: fsTop.rmdirSync, }, diff --git a/packages/xsnap/xsnap-native b/packages/xsnap/xsnap-native index 2d8ccb76b85..eef9b67da55 160000 --- a/packages/xsnap/xsnap-native +++ b/packages/xsnap/xsnap-native @@ -1 +1 @@ -Subproject commit 2d8ccb76b8508e490d9e03972bb4c64f402d5135 +Subproject commit eef9b67da5517ed18ff9e0073b842db20924eae3 From 7f2f8e2e22f42cbdcb7333535d14e69243b7e41b Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Thu, 27 Jun 2024 17:01:23 -0500 Subject: [PATCH 2/7] fix(vow): include vat.js in package files --- packages/vow/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/vow/package.json b/packages/vow/package.json index 40e793d781b..591d2e4793f 100755 --- a/packages/vow/package.json +++ b/packages/vow/package.json @@ -45,6 +45,7 @@ "author": "Agoric", "license": "Apache-2.0", "files": [ + "*.js", "src" ], "publishConfig": { From f037aec0c604f7504dc4c2fb27b25602642a2682 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 05:50:08 +0000 Subject: [PATCH 3/7] fix(agd): check xsnap was rebuilt --- packages/xsnap/package.json | 2 ++ packages/xsnap/scripts/get_xsnap_version.sh | 14 ++++++++++++++ repoconfig.sh | 3 +-- scripts/agd-builder.sh | 15 +++------------ 4 files changed, 20 insertions(+), 14 deletions(-) create mode 100755 packages/xsnap/scripts/get_xsnap_version.sh diff --git a/packages/xsnap/package.json b/packages/xsnap/package.json index 3d67a418912..fd157a7be7d 100644 --- a/packages/xsnap/package.json +++ b/packages/xsnap/package.json @@ -16,6 +16,7 @@ "build:env": "node src/build.js --show-env > build.env", "build:from-env": "{ cat build.env; echo node src/build.js; } | xargs env", "build": "yarn build:bin && yarn build:env", + "check-version": "xsnap_version=$(./scripts/get_xsnap_version.sh); if test \"${npm_package_version}\" != \"${xsnap_version}\"; then echo \"xsnap version mismatch; expected '${npm_package_version}', got '${xsnap_version}'\"; exit 1; fi", "postinstall": "npm run build:from-env", "clean": "rm -rf xsnap-native/xsnap/build", "lint": "run-s --continue-on-error lint:*", @@ -56,6 +57,7 @@ "moddable/xs/makefiles", "moddable/xs/platforms/*.h", "moddable/xs/sources", + "scripts", "src", "xsnap-native/xsnap/makefiles", "xsnap-native/xsnap/sources" diff --git a/packages/xsnap/scripts/get_xsnap_version.sh b/packages/xsnap/scripts/get_xsnap_version.sh new file mode 100755 index 00000000000..065c3724ab2 --- /dev/null +++ b/packages/xsnap/scripts/get_xsnap_version.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +# the xsnap binary lives in a platform-specific directory +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) platform=lin ;; + Darwin*) platform=mac ;; + *) platform=win ;; +esac + +# extract the xsnap package version from the long version printed by xsnap-worker +"./xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -v | sed -e 's/^xsnap \([^ ]*\) (XS [^)]*)$/\1/g' diff --git a/repoconfig.sh b/repoconfig.sh index 14194defb79..231093a2c66 100644 --- a/repoconfig.sh +++ b/repoconfig.sh @@ -4,7 +4,6 @@ NODEJS_VERSION=v16 GOLANG_VERSION=1.20.3 GOLANG_DIR=golang/cosmos GOLANG_DAEMON=$GOLANG_DIR/build/agd -XSNAP_VERSION=agoric-upgrade-10 # Args are major, minor and patch version numbers golang_version_check() { @@ -12,7 +11,7 @@ golang_version_check() { [ "$1" -eq 1 ] && [ "$2" -eq 20 ] && [ "$3" -ge 2 ] && return 0 [ "$1" -eq 1 ] && [ "$2" -ge 21 ] && return 0 [ "$1" -ge 2 ] && return 0 - } 2>/dev/null + } 2> /dev/null echo 1>&2 "need go version 1.20.2+, 1.21+, or 2+" return 1 } diff --git a/scripts/agd-builder.sh b/scripts/agd-builder.sh index e55a4173c3e..39a1648dc44 100755 --- a/scripts/agd-builder.sh +++ b/scripts/agd-builder.sh @@ -219,21 +219,12 @@ $do_not_build || ( echo "At least $src is newer than gyp bindings" (cd "$GOLANG_DIR" && lazy_yarn build:gyp) } + + # check the built xsnap version against the package version it should be using + (cd "${thisdir}/../packages/xsnap" && npm run -s check-version) || exit 1 fi ) -# the xsnap binary lives in a platform-specific directory -unameOut="$(uname -s)" -case "${unameOut}" in - Linux*) platform=lin ;; - Darwin*) platform=mac ;; - *) platform=win ;; -esac - -# check the xsnap version against our baked-in notion of what version we should be using -xsnap_version=$("${thisdir}/../packages/xsnap/xsnap-native/xsnap/build/bin/${platform}/release/xsnap-worker" -n) -[[ "${xsnap_version}" == "${XSNAP_VERSION}" ]] || fatal "xsnap version mismatch; expected ${XSNAP_VERSION}, got ${xsnap_version}" - if $only_build; then echo "Build complete." 1>&2 exit 0 From 0c6bb37f1b4ac754859607a3ae221b9ce3183bfd Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Sun, 30 Jun 2024 08:08:30 +0000 Subject: [PATCH 4/7] feat(xsnap): force rebuild if binary version mismatch --- packages/xsnap/src/build.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/xsnap/src/build.js b/packages/xsnap/src/build.js index 418355405e0..23032349121 100644 --- a/packages/xsnap/src/build.js +++ b/packages/xsnap/src/build.js @@ -226,8 +226,10 @@ const updateSubmodules = async (showEnv, { env, stdout, spawn, fs }) => { * type: typeof import('os').type, * } * }} io + * @param {object} [options] + * @param {boolean} [options.forceBuild] */ -const makeXsnap = async ({ spawn, fs, os }) => { +const makeXsnap = async ({ spawn, fs, os }, { forceBuild = false } = {}) => { const pjson = await fs.readFile(asset('../package.json'), 'utf-8'); const pkg = JSON.parse(pjson); @@ -242,7 +244,7 @@ const makeXsnap = async ({ spawn, fs, os }) => { : ''; const expectedConfigEnvs = configEnvs.concat('').join('\n'); - if (existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { + if (forceBuild || existingConfigEnvs.trim() !== expectedConfigEnvs.trim()) { await fs.writeFile(configEnvFile, expectedConfigEnvs); } @@ -347,7 +349,18 @@ async function main(args, { env, stdout, spawn, fs, os }) { if (!showEnv) { if (hasSource) { - await makeXsnap({ spawn, fs, os }); + // Force a rebuild if for some reason the binary is out of date + // Since the make checks may not always detect that situation + let forceBuild = !hasBin; + if (hasBin) { + const npm = makeCLI('npm', { spawn }); + await npm + .run(['run', '-s', 'check-version'], { cwd: asset('..') }) + .catch(() => { + forceBuild = true; + }); + } + await makeXsnap({ spawn, fs, os }, { forceBuild }); } else if (!hasBin) { throw new Error( 'XSnap has neither sources nor a pre-built binary. Docker? .dockerignore? npm files?', From 377e093c1b9b4aefb8a70a4e7d24dd7cfc957c68 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman <86499+mhofman@users.noreply.github.com> Date: Fri, 28 Jun 2024 01:12:09 -0700 Subject: [PATCH 5/7] fix(cosmos): only allow snapshot export at latest height (#9601) closes: #9600 ## Description Check that an explicit height matches the latest height as exporting historical height is not supported by swing-store ### Security Considerations None ### Scaling Considerations None ### Documentation Considerations Release notes should make clear limits of new support ### Testing Considerations Manually tested explicit height ### Upgrade Considerations Would be good to include in u16 if we cut a new rc1 --- golang/cosmos/daemon/cmd/root.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/golang/cosmos/daemon/cmd/root.go b/golang/cosmos/daemon/cmd/root.go index 11af8c8c22e..043b1b5edcf 100644 --- a/golang/cosmos/daemon/cmd/root.go +++ b/golang/cosmos/daemon/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "errors" + "fmt" "io" "os" "path/filepath" @@ -452,7 +453,7 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) { replacedRunE := func(cmd *cobra.Command, args []string) error { ctx := server.GetServerContextFromCmd(cmd) - height, err := cmd.Flags().GetInt64("height") + heightFlag, err := cmd.Flags().GetInt64("height") if err != nil { return err } @@ -467,13 +468,15 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) { app := ac.newSnapshotsApp(ctx.Logger, db, nil, ctx.Viper) gaiaApp := app.(*gaia.GaiaApp) - if height == 0 { - height = app.CommitMultiStore().LastCommitID().Version + latestHeight := app.CommitMultiStore().LastCommitID().Version + + if heightFlag != 0 && latestHeight != heightFlag { + return fmt.Errorf("cannot export at height %d, only latest height %d is supported", heightFlag, latestHeight) } - cmd.Printf("Exporting snapshot for height %d\n", height) + cmd.Printf("Exporting snapshot for height %d\n", latestHeight) - err = gaiaApp.SwingSetSnapshotter.InitiateSnapshot(height) + err = gaiaApp.SwingSetSnapshotter.InitiateSnapshot(latestHeight) if err != nil { return err } @@ -488,7 +491,7 @@ func replaceCosmosSnapshotExportCommand(cmd *cobra.Command, ac appCreator) { return err } - snapshotHeight := uint64(height) + snapshotHeight := uint64(latestHeight) for _, snapshot := range snapshotList { if snapshot.Height == snapshotHeight { From e3de799ce65df0c25f1ae8e7bd2a0a0887c6f720 Mon Sep 17 00:00:00 2001 From: Mathieu Hofman Date: Mon, 1 Jul 2024 01:51:05 +0000 Subject: [PATCH 6/7] fix(agd): force own node.js version check --- repoconfig.sh | 12 +++++++++++- scripts/agd-builder.sh | 10 +++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/repoconfig.sh b/repoconfig.sh index 231093a2c66..81c33672d95 100644 --- a/repoconfig.sh +++ b/repoconfig.sh @@ -1,6 +1,6 @@ #! /bin/sh # shellcheck disable=SC2034 -NODEJS_VERSION=v16 +NODEJS_VERSION=v20 GOLANG_VERSION=1.20.3 GOLANG_DIR=golang/cosmos GOLANG_DAEMON=$GOLANG_DIR/build/agd @@ -15,3 +15,13 @@ golang_version_check() { echo 1>&2 "need go version 1.20.2+, 1.21+, or 2+" return 1 } + +# Args are major, minor and patch version numbers +nodejs_version_check() { + { + [ "$1" -eq 18 ] && [ "$2" -ge 12 ] && return 0 + [ "$1" -ge 20 ] && [ "$2" -ge 9 ] && return 0 + } 2> /dev/null + echo 1>&2 "need Node.js LTS version ^18.12 or ^20.9, found $1.$2.$3" + return 1 +} diff --git a/scripts/agd-builder.sh b/scripts/agd-builder.sh index 39a1648dc44..9b09c4bfca2 100755 --- a/scripts/agd-builder.sh +++ b/scripts/agd-builder.sh @@ -106,6 +106,12 @@ if $need_nodejs; then } 1>&2 ;; esac + + if nodeversion=$(node --version 2> /dev/null); then + noderegexp='v([0-9]+)\.([0-9]+)\.([0-9]+)' + [[ "$nodeversion" =~ $noderegexp ]] || fatal "illegible node version '$nodeversion'" + nodejs_version_check "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" || exit 1 + fi fi $do_not_build || ( @@ -194,7 +200,9 @@ $do_not_build || ( test -z "$src" || { echo "At least $src is newer than node_modules" rm -f "$STAMPS/yarn-built" - lazy_yarn install + # Ignore engines since we already checked officially supported versions above + # UNTIL https://github.com/Agoric/agoric-sdk/issues/9622 + lazy_yarn install --ignore-engines } stamp=$STAMPS/yarn-built From 972dd6f4f56ef480a02606539bc0025b7d2328ed Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 1 Jul 2024 13:00:19 -0400 Subject: [PATCH 7/7] fix: Disallow Node.js major version >20 --- repoconfig.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repoconfig.sh b/repoconfig.sh index 81c33672d95..8f7c7ca80b8 100644 --- a/repoconfig.sh +++ b/repoconfig.sh @@ -20,7 +20,7 @@ golang_version_check() { nodejs_version_check() { { [ "$1" -eq 18 ] && [ "$2" -ge 12 ] && return 0 - [ "$1" -ge 20 ] && [ "$2" -ge 9 ] && return 0 + [ "$1" -eq 20 ] && [ "$2" -ge 9 ] && return 0 } 2> /dev/null echo 1>&2 "need Node.js LTS version ^18.12 or ^20.9, found $1.$2.$3" return 1