From 610ab2a246d25da9dd6481fc5ab4698ec8e5bb2b Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 25 Jan 2024 09:38:16 -0800 Subject: [PATCH 1/7] fix: redundant install_deps --- .../synthetic-chain/src/cli/dockerfileGen.ts | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/packages/synthetic-chain/src/cli/dockerfileGen.ts b/packages/synthetic-chain/src/cli/dockerfileGen.ts index 00a45d7f..7da2158b 100755 --- a/packages/synthetic-chain/src/cli/dockerfileGen.ts +++ b/packages/synthetic-chain/src/cli/dockerfileGen.ts @@ -63,8 +63,6 @@ FROM use-${lastProposal.proposalName} as prepare-${proposalName} ENV UPGRADE_TO=${planName} UPGRADE_INFO=${JSON.stringify( encodeUpgradeInfo(upgradeInfo), )} -# base is a fresh sdk image so copy these supports -COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/ WORKDIR /usr/src/upgrade-test-scripts SHELL ["/bin/bash", "-c"] @@ -76,17 +74,25 @@ RUN ./start_to_to.sh * - Start agd with the SDK that has the upgradeHandler * - Run any core-evals associated with the proposal (either the ones specified in prepare, or straight from the proposal) */ - EXECUTE({ proposalName, sdkImageTag }: SoftwareUpgradeProposal) { + EXECUTE({ + proposalIdentifier, + proposalName, + sdkImageTag, + }: SoftwareUpgradeProposal) { return ` # EXECUTE ${proposalName} FROM ghcr.io/agoric/agoric-sdk:${sdkImageTag} as execute-${proposalName} -# base is a fresh sdk image so copy these supports -COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/ +WORKDIR /usr/src/upgrade-test-scripts + +# base is a fresh sdk image so set up the proposal and its dependencies +COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/src/proposals/${proposalIdentifier}:${proposalName} +COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/ +# XXX this hits the network each time because the fresh base lacks the global Yarn cache from the previous proposal's build +RUN ./install_deps.sh ${proposalIdentifier}:${proposalName} COPY --link --from=prepare-${proposalName} /root/.agoric /root/.agoric -WORKDIR /usr/src/upgrade-test-scripts SHELL ["/bin/bash", "-c"] RUN ./start_to_to.sh `; @@ -107,7 +113,7 @@ COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/s WORKDIR /usr/src/upgrade-test-scripts -# install using global cache +# First stage of this proposal so install its deps. COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/ RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName} @@ -128,15 +134,8 @@ RUN ./run_eval.sh ${proposalIdentifier}:${proposalName} # USE ${proposalName} FROM ${previousStage}-${proposalName} as use-${proposalName} -COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/src/proposals/${proposalIdentifier}:${proposalName} - WORKDIR /usr/src/upgrade-test-scripts -# TODO remove network dependencies in stages -# install using global cache -COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/ -RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName} - COPY --link --chmod=755 ./upgrade-test-scripts/run_use.sh /usr/src/upgrade-test-scripts/ SHELL ["/bin/bash", "-c"] RUN ./run_use.sh ${proposalIdentifier}:${proposalName} @@ -157,10 +156,6 @@ FROM use-${proposalName} as test-${proposalName} WORKDIR /usr/src/upgrade-test-scripts -# install using global cache -COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/ -RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName} - # copy run_test for this entrypoint and start_agd for optional debugging COPY --link --chmod=755 ./upgrade-test-scripts/run_test.sh ./upgrade-test-scripts/start_agd.sh /usr/src/upgrade-test-scripts/ SHELL ["/bin/bash", "-c"] From d9716b9efaadf1d4050ab9a676c765551b46dcc1 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 24 Jan 2024 17:02:38 -0800 Subject: [PATCH 2/7] build: valid main script --- packages/synthetic-chain/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/synthetic-chain/package.json b/packages/synthetic-chain/package.json index 69dbdfee..1b5512ce 100644 --- a/packages/synthetic-chain/package.json +++ b/packages/synthetic-chain/package.json @@ -3,7 +3,7 @@ "version": "0.0.3", "description": "Utilities to build a chain and test proposals atop it", "bin": "./cli.ts", - "main": "index.js", + "main": "cli.ts", "type": "module", "files": [ "index.js", From df72419987049750f6c54c5b9793210cd8cf99c5 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 25 Jan 2024 09:58:29 -0800 Subject: [PATCH 3/7] chore: disable yarn telemetry --- .../synthetic-chain/upgrade-test-scripts/install_deps.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/synthetic-chain/upgrade-test-scripts/install_deps.sh b/packages/synthetic-chain/upgrade-test-scripts/install_deps.sh index e0e6b11b..7f5a986b 100755 --- a/packages/synthetic-chain/upgrade-test-scripts/install_deps.sh +++ b/packages/synthetic-chain/upgrade-test-scripts/install_deps.sh @@ -9,7 +9,6 @@ PROPOSAL_PATH=$1 export YARN_IGNORE_NODE=1 corepack enable -yarn --version # Run where this script is cd "$(dirname "$(realpath -- "$0")")" @@ -17,6 +16,10 @@ cd "$(dirname "$(realpath -- "$0")")" # TODO consider yarn workspaces to install all in one command if [ -n "$PROPOSAL_PATH" ]; then cd "../proposals/$PROPOSAL_PATH" - # install only if the proposal has a yarn.lock - test -n "yarn.lock" && yarn install --immutable + + if test -f "yarn.lock"; then + yarn --version # only Berry supported, so next commands will fail on classic + yarn config set --home enableTelemetry 0 + yarn install --immutable + fi fi From c8372b3599763013ed2fabbbf8764053692bf1a3 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 24 Jan 2024 12:49:58 -0800 Subject: [PATCH 4/7] feat: doctor clear node_modules --- .github/workflows/ci.yml | 2 -- packages/synthetic-chain/src/cli/doctor.ts | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98056cd9..cf05b418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,8 +108,6 @@ jobs: # Enable corepack for packageManager config - run: corepack enable || sudo corepack enable - run: yarn install - # Set up docker-bake files used by docker/bake-action - - run: node_modules/.bin/synthetic-chain prepare-ci # Test before pushing the images. - name: Build and run proposal tests diff --git a/packages/synthetic-chain/src/cli/doctor.ts b/packages/synthetic-chain/src/cli/doctor.ts index 2012df11..8edfd353 100644 --- a/packages/synthetic-chain/src/cli/doctor.ts +++ b/packages/synthetic-chain/src/cli/doctor.ts @@ -27,6 +27,9 @@ const fixupProposal = (proposal: ProposalInfo) => { } // default to node-modules linker + // (The pnpm linker has little benefit because hard links can't cross + // volumes so each Docker layer will have copies of the deps anyway. The + // pnp linker might work but requires other changes.) const yarnRc = path.join(proposalPath, '.yarnrc.yml'); if (!fs.existsSync(yarnRc)) { console.log(`creating ${yarnRc} with node-modules linker`); @@ -34,6 +37,7 @@ const fixupProposal = (proposal: ProposalInfo) => { } // refresh install + execSync('rm -rf node_modules', { cwd: proposalPath }); execSync('yarn install', { cwd: proposalPath }); } } From 4d008fb40309975327b9caae20d72b7459b52f8b Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 24 Jan 2024 17:15:00 -0800 Subject: [PATCH 5/7] build: scripts compatible with pnpm there's no .bin dir anymore --- .github/workflows/ci.yml | 2 +- README.md | 6 +++--- package.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf05b418..7a5b3cdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,7 @@ jobs: # Test before pushing the images. - name: Build and run proposal tests if: ${{ matrix.platform == env.X86_PLATFORM }} - run: node_modules/.bin/synthetic-chain test + run: yarn test # Build a "use" image for each proposal. This uses Docker Bake's # matrix feature. We could have each "use" image built in a different runner diff --git a/README.md b/README.md index 16f08947..c50f9d7b 100644 --- a/README.md +++ b/README.md @@ -63,21 +63,21 @@ A known issue is that `yarn synthetic-chain` files with `Unknown file extension To build the test images, ``` -node_modules/.bin/synthetic-chain build +tsx packages/synthetic-chain build ``` To build the test images for particular proposals, ``` # build just upgrades -node_modules/.bin/synthetic-chain build --match upgrade +tsx packages/synthetic-chain build --match upgrade ``` To run the tests for particular proposals, ``` # build just upgrades -node_modules/.bin/synthetic-chain test --match upgrade +tsx packages/synthetic-chain test --match upgrade ``` ## Debugging diff --git a/package.json b/package.json index 7114342e..650a4c03 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "packages/*" ], "scripts": { - "build": "echo Use synthetic-chain to build proposal images", - "test": "echo Use synthetic-chain to test proposal images", + "build": "tsx packages/synthetic-chain build", + "test": "tsx packages/synthetic-chain test", "format": "prettier --write ." }, "dependencies": { From 161a066b8ec34c651a4c240098aeef9cfdd81e2e Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Fri, 26 Jan 2024 07:46:11 -0800 Subject: [PATCH 6/7] feat: reclaim disk space after each test --- packages/synthetic-chain/cli.ts | 31 ++++++++++++++++------- packages/synthetic-chain/src/cli/build.ts | 11 +++++--- packages/synthetic-chain/src/cli/run.ts | 14 +++++----- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/packages/synthetic-chain/cli.ts b/packages/synthetic-chain/cli.ts index 987c9a01..f728d335 100755 --- a/packages/synthetic-chain/cli.ts +++ b/packages/synthetic-chain/cli.ts @@ -1,20 +1,20 @@ #!/usr/bin/env tsx -import { parseArgs } from 'node:util'; -import path from 'node:path'; import { execSync } from 'node:child_process'; +import path from 'node:path'; +import { parseArgs } from 'node:util'; import { + bakeTarget, buildProposalSubmissions, - bakeImages, readBuildConfig, } from './src/cli/build.js'; import { writeBakefileProposals, writeDockerfile, } from './src/cli/dockerfileGen.js'; -import { matchOneProposal, readProposals } from './src/cli/proposals.js'; -import { debugTestImage, runTestImages } from './src/cli/run.js'; import { runDoctor } from './src/cli/doctor.js'; +import { imageNameForProposal, matchOneProposal, readProposals } from './src/cli/proposals.js'; +import { debugTestImage, runTestImage } from './src/cli/run.js'; const { positionals, values } = parseArgs({ options: { @@ -62,18 +62,31 @@ const prepareDockerBuild = () => { switch (cmd) { case 'build': { prepareDockerBuild(); - bakeImages('use', values.dry); + bakeTarget('use', values.dry); break; } case 'test': // Always rebuild all test images to keep it simple. With the "use" stages // cached, these are pretty fast building doesn't run agd. prepareDockerBuild(); - bakeImages('test', values.dry); + if (values.debug) { - debugTestImage(matchOneProposal(proposals, match!)); + const proposal = matchOneProposal(proposals, match!); + bakeTarget(imageNameForProposal(proposal, 'test').target, values.dry); + debugTestImage(proposal); + // don't bother to delete the test image because there's just one + // and the user probably wants to run it again. } else { - runTestImages(proposals); + for (const proposal of proposals) { + const image = imageNameForProposal(proposal, 'test'); + bakeTarget(image.target, values.dry); + runTestImage(proposal); + // delete the image to reclaim disk space. The next build + // will use the build cache. + execSync('docker system df', { stdio: 'inherit' }); + execSync(`docker rmi ${image.name}`, { stdio: 'inherit' }); + execSync('docker system df', { stdio: 'inherit' }); + } } break; case 'doctor': diff --git a/packages/synthetic-chain/src/cli/build.ts b/packages/synthetic-chain/src/cli/build.ts index 8843dda3..5753bb23 100755 --- a/packages/synthetic-chain/src/cli/build.ts +++ b/packages/synthetic-chain/src/cli/build.ts @@ -64,13 +64,16 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => { /** * Bake images using the docker buildx bake command. + * + * Note this uses `--load` which pushes the completed images to the builder, + * consuming 2-3 GB per image. + * @see {@link https://docs.docker.com/engine/reference/commandline/buildx_build/#load} * - * @param matrix - The group target + * @param target - The image or group target * @param [dry] - Whether to skip building and just print the build config. */ -export const bakeImages = (matrix: 'test' | 'use', dry = false) => { - // https://docs.docker.com/engine/reference/commandline/buildx_build/#load - const cmd = `docker buildx bake --load ${matrix} ${dry ? '--print' : ''}`; +export const bakeTarget = (target: string, dry = false) => { + const cmd = `docker buildx bake --load ${target} ${dry ? '--print' : ''}`; console.log(cmd); execSync(cmd, { stdio: 'inherit' }); }; diff --git a/packages/synthetic-chain/src/cli/run.ts b/packages/synthetic-chain/src/cli/run.ts index b4a5d7a1..511639b8 100755 --- a/packages/synthetic-chain/src/cli/run.ts +++ b/packages/synthetic-chain/src/cli/run.ts @@ -1,14 +1,12 @@ import { execSync } from 'node:child_process'; import { ProposalInfo, imageNameForProposal } from './proposals.js'; -export const runTestImages = (proposals: ProposalInfo[]) => { - for (const proposal of proposals) { - console.log(`Running test image for proposal ${proposal.proposalName}`); - const { name } = imageNameForProposal(proposal, 'test'); - // 'rm' to remove the container when it exits - const cmd = `docker run --rm ${name}`; - execSync(cmd, { stdio: 'inherit' }); - } +export const runTestImage = (proposal: ProposalInfo) => { + console.log(`Running test image for proposal ${proposal.proposalName}`); + const { name } = imageNameForProposal(proposal, 'test'); + // 'rm' to remove the container when it exits + const cmd = `docker run --rm ${name}`; + execSync(cmd, { stdio: 'inherit' }); }; export const debugTestImage = (proposal: ProposalInfo) => { From e8fce151e00f959fe1b975602c8669dfdc7318ec Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Wed, 24 Jan 2024 11:58:27 -0800 Subject: [PATCH 7/7] ci: log disk usage --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a5b3cdd..d677f16d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,11 +109,19 @@ jobs: - run: corepack enable || sudo corepack enable - run: yarn install + - run: docker system df + - run: docker buildx du --verbose + - run: df -h + # Test before pushing the images. - name: Build and run proposal tests if: ${{ matrix.platform == env.X86_PLATFORM }} run: yarn test + - run: docker system df + - run: docker buildx du --verbose + - run: df -h + # Build a "use" image for each proposal. This uses Docker Bake's # matrix feature. We could have each "use" image built in a different runner # by including https://github.com/docker/bake-action?tab=readme-ov-file#list-targets @@ -130,6 +138,10 @@ jobs: ${{ steps.meta.outputs.bake-file }} targets: use + - run: docker system df + - run: docker buildx du --verbose + - run: df -h + - name: Build and push default image uses: docker/build-push-action@v5 with: @@ -141,6 +153,10 @@ jobs: tags: ${{ steps.docker-tags.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + - run: docker system df + - run: docker buildx du --verbose + - run: df -h + # Merge the default image from each platform into one multi-arch image, # then publish that multiarch image. docker-publish-multiarch: