ci(test-boot): split up test jobs via AVA recipe (#9511)

_Incidental_ `test-boot` takes too long to run as a single job (almost 30 minutes). Instead, create a new strategy matrix dimension to indicate how to split the test jobs via AVA's support for https://github.com/avajs/ava/blob/main/docs/recipes/splitting-tests-ci.md#splitting-tests-on-github-actions Also, deduplicate and replace the older glob-based SwingSet-test splitting with the more general AVA splitting. Before: [32m 3s](https://github.com/Agoric/agoric-sdk/actions/runs/9530612824/usage) After: [15m 53s](https://github.com/Agoric/agoric-sdk/actions/runs/9532147567/usage)
Agoric · Jun 17, 2024 · 5375019 · 5375019
2 parents 7b30169 + 50229bd
commit 5375019
Show file tree

Hide file tree

Showing 10 changed files with 66 additions and 167 deletions.
diff --git a/.github/workflows/test-all-packages.yml b/.github/workflows/test-all-packages.yml
@@ -164,9 +164,6 @@ jobs:
       - name: yarn test (ERTP)
         if: (success() || failure())
         run: cd packages/ERTP && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
-      - name: yarn test (governance)
-        if: (success() || failure())
-        run: cd packages/governance && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: yarn test (import-manager)
         if: (success() || failure())
         run: cd packages/import-manager && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
@@ -197,6 +194,12 @@ jobs:
       - name: yarn test (vow)
         if: (success() || failure())
         run: cd packages/vow && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+      - name: yarn test (xsnap-lockdown)
+        if: (success() || failure())
+        run: cd packages/xsnap-lockdown && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+      - name: yarn test (xsnap)
+        if: (success() || failure())
+        run: cd packages/xsnap && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -334,14 +337,14 @@ jobs:
 
   ##############
   # Long-running tests are executed individually.
-  test-solo:
+  test-governance:
     # BEGIN-TEST-BOILERPLATE
     timeout-minutes: 30
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # test:xs is noop in solo/package.json
+        # test:xs is noop in governance/package.json
         engine: ['18.x', '20.x']
     steps:
       - name: set vars
@@ -356,9 +359,9 @@ jobs:
         with:
           node-version: ${{ steps.vars.outputs.node-version }}
       # END-TEST-BOILERPLATE
-      - name: yarn test (solo)
+      - name: yarn test (governance)
         if: (success() || failure())
-        run: cd packages/solo && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+        run: cd packages/governance && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -375,15 +378,15 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-cosmic-swingset:
+  test-solo:
     # BEGIN-TEST-BOILERPLATE
     timeout-minutes: 30
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # test:xs is noop in cosmic-swingset/package.json
-        engine: ['18.x', '20.x', 'xs']
+        # test:xs is noop in solo/package.json
+        engine: ['18.x', '20.x']
     steps:
       - name: set vars
         id: vars
@@ -397,12 +400,9 @@ jobs:
         with:
           node-version: ${{ steps.vars.outputs.node-version }}
       # END-TEST-BOILERPLATE
-      - uses: ./.github/actions/restore-golang
-        with:
-          go-version: '1.20'
-      - name: yarn test (cosmic-swingset)
+      - name: yarn test (solo)
         if: (success() || failure())
-        run: cd packages/cosmic-swingset && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+        run: cd packages/solo && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -419,15 +419,15 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-inter-protocol:
+  test-cosmic-swingset:
     # BEGIN-TEST-BOILERPLATE
     timeout-minutes: 30
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # test:xs is noop in inter-protocol/package.json
-        engine: ['18.x', '20.x']
+        # test:xs is noop in cosmic-swingset/package.json
+        engine: ['18.x', '20.x', 'xs']
     steps:
       - name: set vars
         id: vars
@@ -441,9 +441,12 @@ jobs:
         with:
           node-version: ${{ steps.vars.outputs.node-version }}
       # END-TEST-BOILERPLATE
-      - name: yarn test (inter-protocol)
+      - uses: ./.github/actions/restore-golang
+        with:
+          go-version: '1.20'
+      - name: yarn test (cosmic-swingset)
         if: (success() || failure())
-        run: cd packages/inter-protocol && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+        run: cd packages/cosmic-swingset && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -460,14 +463,15 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-boot:
+  test-inter-protocol:
     # BEGIN-TEST-BOILERPLATE
-    timeout-minutes: 40
+    timeout-minutes: 30
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        engine: ['18.x', '20.x', 'xs']
+        # test:xs is noop in inter-protocol/package.json
+        engine: ['18.x', '20.x']
     steps:
       - name: set vars
         id: vars
@@ -481,9 +485,9 @@ jobs:
         with:
           node-version: ${{ steps.vars.outputs.node-version }}
       # END-TEST-BOILERPLATE
-      - name: yarn test (boot)
+      - name: yarn test (inter-protocol)
         if: (success() || failure())
-        run: cd packages/boot && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+        run: cd packages/inter-protocol && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -500,33 +504,35 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  # The test-swingset* tests are split by alphabetical test name.
-  test-swingset:
+  test-boot:
     # BEGIN-TEST-BOILERPLATE
-    timeout-minutes: 30
+    timeout-minutes: 40
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
         engine: ['18.x', '20.x', 'xs']
+        # Run parallel testing jobs, each executing a subset.
+        node_index: [0, 1, 2, 3]
+        total_nodes: [4] # the total number of nodes
     steps:
       - name: set vars
         id: vars
         run: |
           echo "node-version=${{ matrix.engine == 'xs' && '18.x' || matrix.engine }}" >> $GITHUB_OUTPUT
           echo "test=${{ matrix.engine == 'xs' && 'test:xs' || 'test' }}" >> $GITHUB_OUTPUT
           echo "GH_ENGINE=${{ matrix.engine }}" >> $GITHUB_ENV
+          echo "CI_NODE_INDEX=${{ matrix.node_index }}" >> $GITHUB_ENV
+          echo "CI_NODE_TOTAL=${{ matrix.total_nodes }}" >> $GITHUB_ENV
 
       - uses: actions/checkout@v4
       - uses: ./.github/actions/restore-node
         with:
           node-version: ${{ steps.vars.outputs.node-version }}
-        id: restore-node
-      - run: echo "ENDO_BRANCH=${{ steps.restore-node.outputs.endo-branch }}" >> $GITHUB_ENV
       # END-TEST-BOILERPLATE
-      - name: yarn test (SwingSet)
+      - name: yarn test (boot)
         if: (success() || failure())
-        run: cd packages/SwingSet && yarn ${{ steps.vars.outputs.test }} 'test/**/[A-Da-d]*' | $TEST_COLLECT
+        run: cd packages/boot && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       - name: notify on failure
         if: failure() && github.event_name != 'pull_request'
         uses: ./.github/actions/notify-status
@@ -543,21 +549,26 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-swingset2:
+  test-swingset:
     # BEGIN-TEST-BOILERPLATE
     timeout-minutes: 30
     needs: build
     runs-on: ubuntu-latest
     strategy:
       matrix:
         engine: ['18.x', '20.x', 'xs']
+        # Run parallel testing jobs, each executing a subset.
+        node_index: [0, 1, 2, 3, 4]
+        total_nodes: [5] # the total number of nodes
     steps:
       - name: set vars
         id: vars
         run: |
           echo "node-version=${{ matrix.engine == 'xs' && '18.x' || matrix.engine }}" >> $GITHUB_OUTPUT
           echo "test=${{ matrix.engine == 'xs' && 'test:xs' || 'test' }}" >> $GITHUB_OUTPUT
           echo "GH_ENGINE=${{ matrix.engine }}" >> $GITHUB_ENV
+          echo "CI_NODE_INDEX=${{ matrix.node_index }}" >> $GITHUB_ENV
+          echo "CI_NODE_TOTAL=${{ matrix.total_nodes }}" >> $GITHUB_ENV
 
       - uses: actions/checkout@v4
       - uses: ./.github/actions/restore-node
@@ -568,13 +579,7 @@ jobs:
       # END-TEST-BOILERPLATE
       - name: yarn test (SwingSet)
         if: (success() || failure())
-        run: cd packages/SwingSet && yarn ${{ steps.vars.outputs.test }} 'test/**/[E-Ie-i]*' | $TEST_COLLECT
-      - name: yarn test (xsnap-lockdown)
-        if: (success() || failure())
-        run: cd packages/xsnap-lockdown && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
-      - name: yarn test (xsnap)
-        if: (success() || failure())
-        run: cd packages/xsnap && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
+        run: cd packages/SwingSet && yarn ${{ steps.vars.outputs.test }} | $TEST_COLLECT
       # explicitly test the XS worker, for visibility
       - name: yarn test (SwingSet XS Worker)
         if: (success() || failure()) && matrix.engine != 'xs'
@@ -595,89 +600,6 @@ jobs:
           datadog-token: ${{ secrets.DATADOG_API_KEY }}
           codecov-token: ${{ secrets.CODECOV_TOKEN }}
 
-  test-swingset3:
-    # BEGIN-TEST-BOILERPLATE
-    timeout-minutes: 30
-    needs: build
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        engine: ['18.x', '20.x', 'xs']
-    steps:
-      - name: set vars
-        id: vars
-        run: |
-          echo "node-version=${{ matrix.engine == 'xs' && '18.x' || matrix.engine }}" >> $GITHUB_OUTPUT
-          echo "test=${{ matrix.engine == 'xs' && 'test:xs' || 'test' }}" >> $GITHUB_OUTPUT
-          echo "GH_ENGINE=${{ matrix.engine }}" >> $GITHUB_ENV
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/restore-node
-        with:
-          node-version: ${{ steps.vars.outputs.node-version }}
-        id: restore-node
-      - run: echo "ENDO_BRANCH=${{ steps.restore-node.outputs.endo-branch }}" >> $GITHUB_ENV
-      # END-TEST-BOILERPLATE
-      - name: yarn test (SwingSet)
-        if: (success() || failure())
-        run: cd packages/SwingSet && yarn ${{ steps.vars.outputs.test }} 'test/**/[J-Rj-r]*' | $TEST_COLLECT
-      - name: notify on failure
-        if: failure() && github.event_name != 'pull_request'
-        uses: ./.github/actions/notify-status
-        with:
-          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
-          from: ${{ secrets.NOTIFY_EMAIL_FROM }}
-          to: ${{ secrets.NOTIFY_EMAIL_TO }}
-          password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }}
-      - uses: ./.github/actions/post-test
-        if: (success() || failure())
-        continue-on-error: true
-        timeout-minutes: 4
-        with:
-          datadog-token: ${{ secrets.DATADOG_API_KEY }}
-          codecov-token: ${{ secrets.CODECOV_TOKEN }}
-
-  test-swingset4:
-    # BEGIN-TEST-BOILERPLATE
-    timeout-minutes: 30
-    needs: build
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        engine: ['18.x', '20.x', 'xs']
-    steps:
-      - name: set vars
-        id: vars
-        run: |
-          echo "node-version=${{ matrix.engine == 'xs' && '18.x' || matrix.engine }}" >> $GITHUB_OUTPUT
-          echo "test=${{ matrix.engine == 'xs' && 'test:xs' || 'test' }}" >> $GITHUB_OUTPUT
-          echo "GH_ENGINE=${{ matrix.engine }}" >> $GITHUB_ENV
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/restore-node
-        with:
-          node-version: ${{ steps.vars.outputs.node-version }}
-        id: restore-node
-      - run: echo "ENDO_BRANCH=${{ steps.restore-node.outputs.endo-branch }}" >> $GITHUB_ENV
-      # END-TEST-BOILERPLATE
-
-      - name: yarn test (SwingSet)
-        if: (success() || failure())
-        run: cd packages/SwingSet && yarn ${{ steps.vars.outputs.test }} 'test/**/[S-Zs-z0-9.test]*.js' | $TEST_COLLECT
-      - name: notify on failure
-        if: failure() && github.event_name != 'pull_request'
-        uses: ./.github/actions/notify-status
-        with:
-          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
-          from: ${{ secrets.NOTIFY_EMAIL_FROM }}
-          to: ${{ secrets.NOTIFY_EMAIL_TO }}
-          password: ${{ secrets.NOTIFY_EMAIL_PASSWORD }}
-      - uses: ./.github/actions/post-test
-        if: (success() || failure())
-        continue-on-error: true
-        timeout-minutes: 4
-        with:
-          datadog-token: ${{ secrets.DATADOG_API_KEY }}
-          codecov-token: ${{ secrets.CODECOV_TOKEN }}
-
   test-zoe-unit:
     # BEGIN-TEST-BOILERPLATE
     timeout-minutes: 30

diff --git a/packages/boot/test/bootstrapTests/demo-config.test.ts b/packages/boot/test/bootstrapTests/demo-config.test.ts
@@ -9,11 +9,9 @@ import { keyArrayEqual, makeSwingsetTestKit } from '../../tools/supports.ts';
 const { keys } = Object;
 
 const makeDefaultTestContext = async t => {
-  const swingsetTestKit = await makeSwingsetTestKit(
-    t.log,
-    'bundles/demo-config',
-    { configSpecifier: '@agoric/vm-config/decentral-demo-config.json' },
-  );
+  const swingsetTestKit = await makeSwingsetTestKit(t.log, undefined, {
+    configSpecifier: '@agoric/vm-config/decentral-demo-config.json',
+  });
   return swingsetTestKit;
 };
 type DefaultTestContext = Awaited<ReturnType<typeof makeDefaultTestContext>>;

diff --git a/packages/boot/test/bootstrapTests/net-ibc-upgrade.test.ts b/packages/boot/test/bootstrapTests/net-ibc-upgrade.test.ts
@@ -31,7 +31,7 @@ export const makeTestContext = async t => {
   }) as Baggage;
   const zone = makeDurableZone(baggage);
 
-  const bundleDir = 'bundles/net-ibc-upgrade';
+  const bundleDir = 'bundles';
   const bundleCache = await makeNodeBundleCache(
     bundleDir,
     { cacheSourceMaps: false },

diff --git a/packages/boot/test/bootstrapTests/vats-restart.test.ts b/packages/boot/test/bootstrapTests/vats-restart.test.ts
@@ -23,13 +23,9 @@ const PLATFORM_CONFIG = '@agoric/vm-config/decentral-itest-vaults-config.json';
 
 export const makeTestContext = async t => {
   console.time('DefaultTestContext');
-  const swingsetTestKit = await makeSwingsetTestKit(
-    t.log,
-    'bundles/vats-restart',
-    {
-      configSpecifier: PLATFORM_CONFIG,
-    },
-  );
+  const swingsetTestKit = await makeSwingsetTestKit(t.log, undefined, {
+    configSpecifier: PLATFORM_CONFIG,
+  });
 
   const { runUtils, storage } = swingsetTestKit;
   console.timeLog('DefaultTestContext', 'swingsetTestKit');

diff --git a/packages/boot/test/bootstrapTests/vaults-integration.test.ts b/packages/boot/test/bootstrapTests/vaults-integration.test.ts
@@ -31,10 +31,7 @@ const likePayouts = (collateral, minted) => ({
 
 const makeDefaultTestContext = async t => {
   console.time('DefaultTestContext');
-  const swingsetTestKit = await makeSwingsetTestKit(
-    t.log,
-    'bundles/vaults-integration',
-  );
+  const swingsetTestKit = await makeSwingsetTestKit(t.log);
 
   const { runUtils, storage } = swingsetTestKit;
   console.timeLog('DefaultTestContext', 'swingsetTestKit');

diff --git a/packages/boot/test/bootstrapTests/vaults-upgrade.test.ts b/packages/boot/test/bootstrapTests/vaults-upgrade.test.ts
@@ -30,13 +30,9 @@ const makeDefaultTestContext = async (
   } = {},
 ) => {
   logTiming && console.time('DefaultTestContext');
-  const swingsetTestKit = await makeSwingsetTestKit(
-    t.log,
-    'bundles/vaults-upgrade',
-    {
-      storage,
-    },
-  );
+  const swingsetTestKit = await makeSwingsetTestKit(t.log, undefined, {
+    storage,
+  });
 
   const { readLatest, runUtils } = swingsetTestKit;
   ({ storage } = swingsetTestKit);