From ca3070faaeb203b04fb2280ad569ba78be3f1b2a Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Fri, 3 Nov 2023 15:35:41 -0700 Subject: [PATCH 1/3] refactor: split start_agd from start_to_to --- upgrade-test-scripts/start_agd.sh | 11 ++++ upgrade-test-scripts/start_to_to.sh | 83 ++++++++++++++--------------- 2 files changed, 50 insertions(+), 44 deletions(-) create mode 100644 upgrade-test-scripts/start_agd.sh diff --git a/upgrade-test-scripts/start_agd.sh b/upgrade-test-scripts/start_agd.sh new file mode 100644 index 00000000..5f620344 --- /dev/null +++ b/upgrade-test-scripts/start_agd.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Start and leave agd running + +grep -qF 'env_setup.sh' /root/.bashrc || echo "source /usr/src/upgrade-test-scripts/env_setup.sh" >>/root/.bashrc +grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc + +source ./env_setup.sh + +export SLOGFILE=slog.slog + +startAgd diff --git a/upgrade-test-scripts/start_to_to.sh b/upgrade-test-scripts/start_to_to.sh index d5b83da5..a58d64ee 100644 --- a/upgrade-test-scripts/start_to_to.sh +++ b/upgrade-test-scripts/start_to_to.sh @@ -1,4 +1,5 @@ #!/bin/bash +# Prepare or execute an upgrade grep -qF 'env_setup.sh' /root/.bashrc || echo "source /usr/src/upgrade-test-scripts/env_setup.sh" >>/root/.bashrc grep -qF 'printKeys' /root/.bashrc || echo "printKeys" >>/root/.bashrc @@ -9,52 +10,46 @@ export SLOGFILE=slog.slog startAgd -if [[ "$DEST" != "1" ]]; then - #Destined for an upgrade - if [[ -z "${UPGRADE_TO}" ]]; then - echo "no upgrade set. running for a few blocks and exiting" - waitForBlock 5 - exit 0 - fi +if [[ -z "${UPGRADE_TO}" ]]; then + echo "no upgrade set. running for a few blocks and exiting" + waitForBlock 5 + exit 0 +fi + +voting_period_s=10 +latest_height=$(agd status | jq -r .SyncInfo.latest_block_height) +height=$((latest_height + voting_period_s + 10)) +info=${UPGRADE_INFO-"{}"} +if echo "$info" | jq .; then + echo "upgrade-info: $info" +else + status=$? + echo "Upgrade info is not valid JSON: $info" + exit $status +fi +agd tx gov submit-proposal software-upgrade "$UPGRADE_TO" \ + --upgrade-height="$height" --upgrade-info="$info" \ + --title="Upgrade to ${UPGRADE_TO}" --description="upgrades" \ + --from=validator --chain-id="$CHAINID" \ + --yes --keyring-backend=test +waitForBlock - voting_period_s=10 +voteLatestProposalAndWait + +echo "Chain in to-be-upgraded state for $UPGRADE_TO" + +while true; do latest_height=$(agd status | jq -r .SyncInfo.latest_block_height) - height=$((latest_height + voting_period_s + 10)) - info=${UPGRADE_INFO-"{}"} - if echo "$info" | jq .; then - echo "upgrade-info: $info" + if [ "$latest_height" != "$height" ]; then + echo "Waiting for upgrade height for $UPGRADE_TO to be reached (need $height, have $latest_height)" + sleep 1 else - status=$? - echo "Upgrade info is not valid JSON: $info" - exit $status + echo "Upgrade height for $UPGRADE_TO reached. Killing agd" + echo "(CONSENSUS FAILURE above for height $height is expected)" + break fi - agd tx gov submit-proposal software-upgrade "$UPGRADE_TO" \ - --upgrade-height="$height" --upgrade-info="$info" \ - --title="Upgrade to ${UPGRADE_TO}" --description="upgrades" \ - --from=validator --chain-id="$CHAINID" \ - --yes --keyring-backend=test - waitForBlock - - voteLatestProposalAndWait - - echo "Chain in to-be-upgraded state for $UPGRADE_TO" - - while true; do - latest_height=$(agd status | jq -r .SyncInfo.latest_block_height) - if [ "$latest_height" != "$height" ]; then - echo "Waiting for upgrade height for $UPGRADE_TO to be reached (need $height, have $latest_height)" - sleep 1 - else - echo "Upgrade height for $UPGRADE_TO reached. Killing agd" - echo "(CONSENSUS FAILURE above for height $height is expected)" - break - fi - done - - sleep 2 - killAgd - echo "state directory $HOME/.agoric ready for upgrade to $UPGRADE_TO" -else +done - waitAgd -fi +sleep 2 +killAgd +echo "state directory $HOME/.agoric ready for upgrade to $UPGRADE_TO" From 890b2e27d5128d71b2219bb15ae3e470c3e34492 Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Mon, 6 Nov 2023 05:36:38 -0800 Subject: [PATCH 2/3] fix: stream stdio from Docker commands in script --- buildTestImages.ts | 6 ++---- runTestImages.ts | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/buildTestImages.ts b/buildTestImages.ts index 29b07fc6..555b0a9b 100755 --- a/buildTestImages.ts +++ b/buildTestImages.ts @@ -1,5 +1,4 @@ #!/usr/bin/env tsx -// @ts-check import { parseArgs } from 'node:util'; import { execSync } from 'node:child_process'; @@ -11,7 +10,7 @@ refreshDockerfile(); const options = { match: { short: 'm', type: 'string' }, dry: { type: 'boolean' }, -}; +} as const; const { values } = parseArgs({ options }); const { match, dry } = values; @@ -30,7 +29,6 @@ for (const proposal of proposals) { const cmd = `docker build --tag ${name} --target ${target} .`; console.log(cmd); if (!dry) { - // TODO stream the output - execSync(cmd); + execSync(cmd, { stdio: 'inherit' }); } } diff --git a/runTestImages.ts b/runTestImages.ts index f1722c13..b9dd11ba 100755 --- a/runTestImages.ts +++ b/runTestImages.ts @@ -1,5 +1,4 @@ #!/usr/bin/env tsx -// @ts-check import { parseArgs } from 'node:util'; import { execSync } from 'node:child_process'; @@ -7,7 +6,7 @@ import { imageNameForProposalTest, readProposals } from './common'; const options = { match: { short: 'm', type: 'string' }, -}; +} as const; const { values } = parseArgs({ options }); const { match } = values; @@ -23,6 +22,5 @@ for (const proposal of proposals) { const { name } = imageNameForProposalTest(proposal); // 'rm' to remove the container when it exits const cmd = `docker run --rm ${name}`; - // TODO stream the output - execSync(cmd); + execSync(cmd, { stdio: 'inherit' }); } From 7ac85a1bfda0e846fd2f9bcf81071ca8372238ae Mon Sep 17 00:00:00 2001 From: Turadg Aleahmad Date: Thu, 2 Nov 2023 17:47:17 -0700 Subject: [PATCH 3/3] feat: push completed image --- .github/workflows/ci.yml | 46 ++++++++++++++++++++++++++++++++++++++-- buildTestImages.ts | 4 +++- makeDockerfile.ts | 17 ++++++++++++++- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d94dc2c4..300fea60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,12 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + jobs: + # see https://docs.docker.com/build/ci/github-actions/test-before-push/ test-proposals: runs-on: ubuntu-latest timeout-minutes: 60 @@ -29,12 +34,49 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" echo "=== After cleanup:" df -h - - uses: actions/checkout@v3 + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + # see https://docs.github.com/en/actions/publishing-packages/publishing-docker-images + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ${{ env.REGISTRY }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # The .ts scripts depend upon this - run: yarn global add tsx + - name: build test images - run: ./buildTestImages.ts + run: | + docker info + ./buildTestImages.ts - name: run test images run: ./runTestImages.ts + + # XXX this should be instant because all the stages were already built in the steps above + # but it's re-building the last stage. This is deemed good enough for now. + # see https://github.com/moby/moby/issues/34715 + - name: Build and push complete image + uses: docker/build-push-action@v5 + with: + context: . + # push to registry on every repo push. A PR #2 will push with tag `pr-2` and `main` will have tag `main`. + # See https://github.com/docker/metadata-action?tab=readme-ov-file#basic. + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - name: notify on failure if: failure() && github.event_name != 'pull_request' uses: ./.github/actions/notify-status diff --git a/buildTestImages.ts b/buildTestImages.ts index 555b0a9b..873e48c7 100755 --- a/buildTestImages.ts +++ b/buildTestImages.ts @@ -26,7 +26,9 @@ for (const proposal of proposals) { console.log(`\nBuilding test image for proposal ${proposal.proposalName}`); } const { name, target } = imageNameForProposalTest(proposal); - const cmd = `docker build --tag ${name} --target ${target} .`; + // 'load' to ensure the images are output to the Docker client. Seems to be necessary + // for the CI docker/build-push-action to re-use the cached stages. + const cmd = `docker buildx build --load --tag ${name} --target ${target} .`; console.log(cmd); if (!dry) { execSync(cmd, { stdio: 'inherit' }); diff --git a/makeDockerfile.ts b/makeDockerfile.ts index 75a20f3f..d2b7b064 100755 --- a/makeDockerfile.ts +++ b/makeDockerfile.ts @@ -135,10 +135,22 @@ RUN ./run_use.sh ${proposalIdentifier}:${proposalName} # TEST ${proposalName} FROM use-${proposalName} as test-${proposalName} -# XXX the test files were already copied in the "use" stage WORKDIR /usr/src/upgrade-test-scripts SHELL ["/bin/bash", "-c"] ENTRYPOINT ./run_test.sh ${proposalIdentifier}:${proposalName} +`; + }, + /** + * The last target in the file, for untargeted `docker build` + */ + DEFAULT(lastProposal: ProposalInfo) { + return ` +# DEFAULT +FROM use-${lastProposal.proposalName} + +WORKDIR /usr/src/upgrade-test-scripts +SHELL ["/bin/bash", "-c"] +ENTRYPOINT ./start_agd.sh `; }, }; @@ -166,10 +178,13 @@ for (const proposal of readProposals()) { blocks.push(stage.EXECUTE(proposal)); } + // The stages must be output in dependency order because if the builder finds a FROM + // that it hasn't built yet, it will search for it in the registry. But it won't be there! blocks.push(stage.USE(proposal)); blocks.push(stage.TEST(proposal)); previousProposal = proposal; } +blocks.push(stage.DEFAULT(previousProposal!)); export function refreshDockerfile() { const contents = blocks.join('\n');