solve disk usage

.github/workflows/ci.yml

@@ -108,13 +108,19 @@ jobs:
       # Enable corepack for packageManager config
       - run: corepack enable || sudo corepack enable
       - run: yarn install
-      # Set up docker-bake files used by docker/bake-action
-      - run: node_modules/.bin/synthetic-chain prepare-ci
+
+      - run: docker system df
+      - run: docker buildx du --verbose
+      - run: df -h
 
       # Test before pushing the images.
       - name: Build and run proposal tests
         if: ${{ matrix.platform == env.X86_PLATFORM }}
-        run: node_modules/.bin/synthetic-chain test
+        run: yarn test
+
+      - run: docker system df
+      - run: docker buildx du --verbose
+      - run: df -h
 
       # Build a "use" image for each proposal. This uses Docker Bake's
       # matrix feature. We could have each "use" image built in a different runner
@@ -132,6 +138,10 @@ jobs:
             ${{ steps.meta.outputs.bake-file }}
           targets: use
 
+      - run: docker system df
+      - run: docker buildx du --verbose
+      - run: df -h
+
       - name: Build and push default image
         uses: docker/build-push-action@v5
         with:
@@ -143,6 +153,10 @@ jobs:
           tags: ${{ steps.docker-tags.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
+      - run: docker system df
+      - run: docker buildx du --verbose
+      - run: df -h
+
   # Merge the default image from each platform into one multi-arch image,
   # then publish that multiarch image.
   docker-publish-multiarch:

README.md

@@ -63,21 +63,21 @@ A known issue is that `yarn synthetic-chain` files with `Unknown file extension
 To build the test images,
 
 ```
-node_modules/.bin/synthetic-chain build
+tsx packages/synthetic-chain build
 ```
 
 To build the test images for particular proposals,
 
 ```
 # build just upgrades
-node_modules/.bin/synthetic-chain build --match upgrade
+tsx packages/synthetic-chain build --match upgrade
 ```
 
 To run the tests for particular proposals,
 
 ```
 # build just upgrades
-node_modules/.bin/synthetic-chain test --match upgrade
+tsx packages/synthetic-chain test --match upgrade
 ```
 
 ## Debugging

package.json

@@ -4,8 +4,8 @@
     "packages/*"
   ],
   "scripts": {
-    "build": "echo Use synthetic-chain to build proposal images",
-    "test": "echo Use synthetic-chain to test proposal images",
+    "build": "tsx packages/synthetic-chain build",
+    "test": "tsx packages/synthetic-chain test",
     "format": "prettier --write ."
   },
   "dependencies": {

packages/synthetic-chain/cli.ts

@@ -1,20 +1,20 @@
 #!/usr/bin/env tsx
 
-import { parseArgs } from 'node:util';
-import path from 'node:path';
 import { execSync } from 'node:child_process';
+import path from 'node:path';
+import { parseArgs } from 'node:util';
 import {
+  bakeTarget,
   buildProposalSubmissions,
-  bakeImages,
   readBuildConfig,
 } from './src/cli/build.js';
 import {
   writeBakefileProposals,
   writeDockerfile,
 } from './src/cli/dockerfileGen.js';
-import { matchOneProposal, readProposals } from './src/cli/proposals.js';
-import { debugTestImage, runTestImages } from './src/cli/run.js';
 import { runDoctor } from './src/cli/doctor.js';
+import { imageNameForProposal, matchOneProposal, readProposals } from './src/cli/proposals.js';
+import { debugTestImage, runTestImage } from './src/cli/run.js';
 
 const { positionals, values } = parseArgs({
   options: {
@@ -62,18 +62,31 @@ const prepareDockerBuild = () => {
 switch (cmd) {
   case 'build': {
     prepareDockerBuild();
-    bakeImages('use', values.dry);
+    bakeTarget('use', values.dry);
     break;
   }
   case 'test':
     // Always rebuild all test images to keep it simple. With the "use" stages
     // cached, these are pretty fast building doesn't run agd.
     prepareDockerBuild();
-    bakeImages('test', values.dry);
+
     if (values.debug) {
-      debugTestImage(matchOneProposal(proposals, match!));
+      const proposal = matchOneProposal(proposals, match!);
+      bakeTarget(imageNameForProposal(proposal, 'test').target, values.dry);
+      debugTestImage(proposal);
+      // don't bother to delete the test image because there's just one
+      // and the user probably wants to run it again.
     } else {
-      runTestImages(proposals);
+      for (const proposal of proposals) {
+        const image = imageNameForProposal(proposal, 'test');
+        bakeTarget(image.target, values.dry);
+        runTestImage(proposal);
+        // delete the image to reclaim disk space. The next build
+        // will use the build cache.
+        execSync('docker system df', { stdio: 'inherit' });
+        execSync(`docker rmi ${image.name}`, { stdio: 'inherit' });
+        execSync('docker system df', { stdio: 'inherit' });
+      }
     }
     break;
   case 'doctor':

packages/synthetic-chain/package.json

@@ -3,7 +3,7 @@
   "version": "0.0.3",
   "description": "Utilities to build a chain and test proposals atop it",
   "bin": "./cli.ts",
-  "main": "index.js",
+  "main": "cli.ts",
   "type": "module",
   "files": [
     "index.js",

packages/synthetic-chain/src/cli/build.ts

@@ -64,13 +64,16 @@ export const buildProposalSubmissions = (proposals: ProposalInfo[]) => {
 
 /**
  * Bake images using the docker buildx bake command.
+ * 
+ * Note this uses `--load` which pushes the completed images to the builder,
+ * consuming 2-3 GB per image.
+ * @see {@link https://docs.docker.com/engine/reference/commandline/buildx_build/#load}
  *
- * @param matrix - The group target
+ * @param target - The image or group target
  * @param [dry] - Whether to skip building and just print the build config.
  */
-export const bakeImages = (matrix: 'test' | 'use', dry = false) => {
-  // https://docs.docker.com/engine/reference/commandline/buildx_build/#load
-  const cmd = `docker buildx bake --load ${matrix} ${dry ? '--print' : ''}`;
+export const bakeTarget = (target: string, dry = false) => {
+  const cmd = `docker buildx bake --load ${target} ${dry ? '--print' : ''}`;
   console.log(cmd);
   execSync(cmd, { stdio: 'inherit' });
 };

packages/synthetic-chain/src/cli/dockerfileGen.ts

@@ -63,8 +63,6 @@ FROM use-${lastProposal.proposalName} as prepare-${proposalName}
 ENV UPGRADE_TO=${planName} UPGRADE_INFO=${JSON.stringify(
       encodeUpgradeInfo(upgradeInfo),
     )}
-# base is a fresh sdk image so copy these supports
-COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/
 
 WORKDIR /usr/src/upgrade-test-scripts
 SHELL ["/bin/bash", "-c"]
@@ -76,17 +74,25 @@ RUN ./start_to_to.sh
    * - Start agd with the SDK that has the upgradeHandler
    * - Run any core-evals associated with the proposal (either the ones specified in prepare, or straight from the proposal)
    */
-  EXECUTE({ proposalName, sdkImageTag }: SoftwareUpgradeProposal) {
+  EXECUTE({
+    proposalIdentifier,
+    proposalName,
+    sdkImageTag,
+  }: SoftwareUpgradeProposal) {
     return `
 # EXECUTE ${proposalName}
 FROM ghcr.io/agoric/agoric-sdk:${sdkImageTag} as execute-${proposalName}
 
-# base is a fresh sdk image so copy these supports
-COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/
+WORKDIR /usr/src/upgrade-test-scripts
+
+# base is a fresh sdk image so set up the proposal and its dependencies
+COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/src/proposals/${proposalIdentifier}:${proposalName}
+COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/start_to_to.sh ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
+# XXX this hits the network each time because the fresh base lacks the global Yarn cache from the previous proposal's build
+RUN ./install_deps.sh ${proposalIdentifier}:${proposalName}
 
 COPY --link --from=prepare-${proposalName} /root/.agoric /root/.agoric
 
-WORKDIR /usr/src/upgrade-test-scripts
 SHELL ["/bin/bash", "-c"]
 RUN ./start_to_to.sh
 `;
@@ -107,7 +113,7 @@ COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/s
 
 WORKDIR /usr/src/upgrade-test-scripts
 
-# install using global cache
+# First stage of this proposal so install its deps.
 COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
 RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName}
 
@@ -128,15 +134,8 @@ RUN ./run_eval.sh ${proposalIdentifier}:${proposalName}
 # USE ${proposalName}
 FROM ${previousStage}-${proposalName} as use-${proposalName}
 
-COPY --link --chmod=755 ./proposals/${proposalIdentifier}:${proposalName} /usr/src/proposals/${proposalIdentifier}:${proposalName}
-
 WORKDIR /usr/src/upgrade-test-scripts
 
-# TODO remove network dependencies in stages
-# install using global cache
-COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
-RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName}
-
 COPY --link --chmod=755 ./upgrade-test-scripts/run_use.sh /usr/src/upgrade-test-scripts/
 SHELL ["/bin/bash", "-c"]
 RUN ./run_use.sh ${proposalIdentifier}:${proposalName}
@@ -157,10 +156,6 @@ FROM use-${proposalName} as test-${proposalName}
 
 WORKDIR /usr/src/upgrade-test-scripts
 
-# install using global cache
-COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
-RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${proposalIdentifier}:${proposalName}
-
 # copy run_test for this entrypoint and start_agd for optional debugging
 COPY --link --chmod=755 ./upgrade-test-scripts/run_test.sh ./upgrade-test-scripts/start_agd.sh /usr/src/upgrade-test-scripts/
 SHELL ["/bin/bash", "-c"]

packages/synthetic-chain/src/cli/doctor.ts

@@ -27,13 +27,17 @@ const fixupProposal = (proposal: ProposalInfo) => {
       }
 
       // default to node-modules linker
+      // (The pnpm linker has little benefit because hard links can't cross
+      // volumes so each Docker layer will have copies of the deps anyway. The
+      // pnp linker might work but requires other changes.)
       const yarnRc = path.join(proposalPath, '.yarnrc.yml');
       if (!fs.existsSync(yarnRc)) {
         console.log(`creating ${yarnRc} with node-modules linker`);
         fs.writeFileSync(yarnRc, 'nodeLinker: node-modules\n');
       }
 
       // refresh install
+      execSync('rm -rf node_modules', { cwd: proposalPath });
       execSync('yarn install', { cwd: proposalPath });
     }
   }

packages/synthetic-chain/src/cli/run.ts

@@ -1,14 +1,12 @@
 import { execSync } from 'node:child_process';
 import { ProposalInfo, imageNameForProposal } from './proposals.js';
 
-export const runTestImages = (proposals: ProposalInfo[]) => {
-  for (const proposal of proposals) {
-    console.log(`Running test image for proposal ${proposal.proposalName}`);
-    const { name } = imageNameForProposal(proposal, 'test');
-    // 'rm' to remove the container when it exits
-    const cmd = `docker run --rm ${name}`;
-    execSync(cmd, { stdio: 'inherit' });
-  }
+export const runTestImage = (proposal: ProposalInfo) => {
+  console.log(`Running test image for proposal ${proposal.proposalName}`);
+  const { name } = imageNameForProposal(proposal, 'test');
+  // 'rm' to remove the container when it exits
+  const cmd = `docker run --rm ${name}`;
+  execSync(cmd, { stdio: 'inherit' });
 };
 
 export const debugTestImage = (proposal: ProposalInfo) => {

packages/synthetic-chain/upgrade-test-scripts/install_deps.sh

@@ -9,14 +9,17 @@ PROPOSAL_PATH=$1
 export YARN_IGNORE_NODE=1
 
 corepack enable
-yarn --version
 
 # Run where this script is
 cd "$(dirname "$(realpath -- "$0")")"
 
 # TODO consider yarn workspaces to install all in one command
 if [ -n "$PROPOSAL_PATH" ]; then
     cd "../proposals/$PROPOSAL_PATH"
-    # install only if the proposal has a yarn.lock
-    test -n "yarn.lock" && yarn install --immutable
+
+    if test -f "yarn.lock"; then
+        yarn --version # only Berry supported, so next commands will fail on classic
+        yarn config set --home enableTelemetry 0
+        yarn install --immutable
+    fi
 fi