Merge pull request #11 from Agoric/6-push-complete

feat: push completed image

.github/workflows/ci.yml

@@ -11,7 +11,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
+  # see https://docs.docker.com/build/ci/github-actions/test-before-push/
   test-proposals:
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -29,12 +34,49 @@ jobs:
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
           echo "=== After cleanup:"
           df -h
-      - uses: actions/checkout@v3
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        # see https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          registry: ${{ env.REGISTRY }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+        # The .ts scripts depend upon this
       - run: yarn global add tsx
+
       - name: build test images
-        run: ./buildTestImages.ts
+        run: |
+          docker info
+          ./buildTestImages.ts
       - name: run test images
         run: ./runTestImages.ts
+
+      # XXX this should be instant because all the stages were already built in the steps above
+      # but it's re-building the last stage. This is deemed good enough for now.
+      # see https://github.com/moby/moby/issues/34715
+      - name: Build and push complete image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          # push to registry on every repo push. A PR #2 will push with tag `pr-2` and `main` will have tag `main`.
+          # See https://github.com/docker/metadata-action?tab=readme-ov-file#basic.
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status

buildTestImages.ts

@@ -1,5 +1,4 @@
 #!/usr/bin/env tsx
-// @ts-check
 
 import { parseArgs } from 'node:util';
 import { execSync } from 'node:child_process';
@@ -11,7 +10,7 @@ refreshDockerfile();
 const options = {
   match: { short: 'm', type: 'string' },
   dry: { type: 'boolean' },
-};
+} as const;
 const { values } = parseArgs({ options });
 
 const { match, dry } = values;
@@ -27,10 +26,11 @@ for (const proposal of proposals) {
     console.log(`\nBuilding test image for proposal ${proposal.proposalName}`);
   }
   const { name, target } = imageNameForProposalTest(proposal);
-  const cmd = `docker build --tag ${name} --target ${target} .`;
+  // 'load' to ensure the images are output to the Docker client. Seems to be necessary
+  // for the CI docker/build-push-action to re-use the cached stages.
+  const cmd = `docker buildx build --load --tag ${name} --target ${target} .`;
   console.log(cmd);
   if (!dry) {
-    // TODO stream the output
-    execSync(cmd);
+    execSync(cmd, { stdio: 'inherit' });
   }
 }

makeDockerfile.ts

@@ -135,10 +135,22 @@ RUN ./run_use.sh ${proposalIdentifier}:${proposalName}
 # TEST ${proposalName}
 FROM use-${proposalName} as test-${proposalName}
 
-# XXX the test files were already copied in the "use" stage
 WORKDIR /usr/src/upgrade-test-scripts
 SHELL ["/bin/bash", "-c"]
 ENTRYPOINT ./run_test.sh ${proposalIdentifier}:${proposalName}
+`;
+  },
+  /**
+   * The last target in the file, for untargeted `docker build`
+   */
+  DEFAULT(lastProposal: ProposalInfo) {
+    return `
+# DEFAULT
+FROM use-${lastProposal.proposalName}
+
+WORKDIR /usr/src/upgrade-test-scripts
+SHELL ["/bin/bash", "-c"]
+ENTRYPOINT ./start_agd.sh
 `;
   },
 };
@@ -166,10 +178,13 @@ for (const proposal of readProposals()) {
     blocks.push(stage.EXECUTE(proposal));
   }
 
+  // The stages must be output in dependency order because if the builder finds a FROM
+  // that it hasn't built yet, it will search for it in the registry. But it won't be there!
   blocks.push(stage.USE(proposal));
   blocks.push(stage.TEST(proposal));
   previousProposal = proposal;
 }
+blocks.push(stage.DEFAULT(previousProposal!));
 
 export function refreshDockerfile() {
   const contents = blocks.join('\n');

runTestImages.ts

@@ -1,13 +1,12 @@
 #!/usr/bin/env tsx
-// @ts-check
 
 import { parseArgs } from 'node:util';
 import { execSync } from 'node:child_process';
 import { imageNameForProposalTest, readProposals } from './common';
 
 const options = {
   match: { short: 'm', type: 'string' },
-};
+} as const;
 const { values } = parseArgs({ options });
 
 const { match } = values;
@@ -23,6 +22,5 @@ for (const proposal of proposals) {
   const { name } = imageNameForProposalTest(proposal);
   // 'rm' to remove the container when it exits
   const cmd = `docker run --rm ${name}`;
-  // TODO stream the output
-  execSync(cmd);
+  execSync(cmd, { stdio: 'inherit' });
 }

upgrade-test-scripts/start_agd.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Start and leave agd running
+
+grep -qF 'env_setup.sh' /root/.bashrc || echo "source /usr/src/upgrade-test-scripts/env_setup.sh" >>/root/.bashrc
+grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc
+
+source ./env_setup.sh
+
+export SLOGFILE=slog.slog
+
+startAgd

upgrade-test-scripts/start_to_to.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# Prepare or execute an upgrade
 
 grep -qF 'env_setup.sh' /root/.bashrc || echo "source /usr/src/upgrade-test-scripts/env_setup.sh" >>/root/.bashrc
 grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc
@@ -9,52 +10,46 @@ export SLOGFILE=slog.slog
 
 startAgd
 
-if [[ "$DEST" != "1" ]]; then
-  #Destined for an upgrade
-  if [[ -z "${UPGRADE_TO}" ]]; then
-    echo "no upgrade set.  running for a few blocks and exiting"
-    waitForBlock 5
-    exit 0
-  fi
+if [[ -z "${UPGRADE_TO}" ]]; then
+  echo "no upgrade set.  running for a few blocks and exiting"
+  waitForBlock 5
+  exit 0
+fi
+
+voting_period_s=10
+latest_height=$(agd status | jq -r .SyncInfo.latest_block_height)
+height=$((latest_height + voting_period_s + 10))
+info=${UPGRADE_INFO-"{}"}
+if echo "$info" | jq .; then
+  echo "upgrade-info: $info"
+else
+  status=$?
+  echo "Upgrade info is not valid JSON: $info"
+  exit $status
+fi
+agd tx gov submit-proposal software-upgrade "$UPGRADE_TO" \
+  --upgrade-height="$height" --upgrade-info="$info" \
+  --title="Upgrade to ${UPGRADE_TO}" --description="upgrades" \
+  --from=validator --chain-id="$CHAINID" \
+  --yes --keyring-backend=test
+waitForBlock
 
-  voting_period_s=10
+voteLatestProposalAndWait
+
+echo "Chain in to-be-upgraded state for $UPGRADE_TO"
+
+while true; do
   latest_height=$(agd status | jq -r .SyncInfo.latest_block_height)
-  height=$((latest_height + voting_period_s + 10))
-  info=${UPGRADE_INFO-"{}"}
-  if echo "$info" | jq .; then
-    echo "upgrade-info: $info"
+  if [ "$latest_height" != "$height" ]; then
+    echo "Waiting for upgrade height for $UPGRADE_TO to be reached (need $height, have $latest_height)"
+    sleep 1
   else
-    status=$?
-    echo "Upgrade info is not valid JSON: $info"
-    exit $status
+    echo "Upgrade height for $UPGRADE_TO reached. Killing agd"
+    echo "(CONSENSUS FAILURE above for height $height is expected)"
+    break
   fi
-  agd tx gov submit-proposal software-upgrade "$UPGRADE_TO" \
-    --upgrade-height="$height" --upgrade-info="$info" \
-    --title="Upgrade to ${UPGRADE_TO}" --description="upgrades" \
-    --from=validator --chain-id="$CHAINID" \
-    --yes --keyring-backend=test
-  waitForBlock
-
-  voteLatestProposalAndWait
-
-  echo "Chain in to-be-upgraded state for $UPGRADE_TO"
-
-  while true; do
-    latest_height=$(agd status | jq -r .SyncInfo.latest_block_height)
-    if [ "$latest_height" != "$height" ]; then
-      echo "Waiting for upgrade height for $UPGRADE_TO to be reached (need $height, have $latest_height)"
-      sleep 1
-    else
-      echo "Upgrade height for $UPGRADE_TO reached. Killing agd"
-      echo "(CONSENSUS FAILURE above for height $height is expected)"
-      break
-    fi
-  done
-
-  sleep 2
-  killAgd
-  echo "state directory $HOME/.agoric ready for upgrade to $UPGRADE_TO"
-else
+done
 
-  waitAgd
-fi
+sleep 2
+killAgd
+echo "state directory $HOME/.agoric ready for upgrade to $UPGRADE_TO"