test(multichain-testing): long awaits use stir for no AVA timeout

Agoric · Oct 8, 2024 · 00686bd · 00686bd
1 parent fdb7004
commit 00686bd
Show file tree

Hide file tree

Showing 9 changed files with 267 additions and 35 deletions.
diff --git a/multichain-testing/package.json b/multichain-testing/package.json
@@ -58,8 +58,7 @@
       "**/*.test.ts"
     ],
     "concurrency": 1,
-    "serial": true,
-    "timeout": "125s"
+    "serial": true
   },
   "eslintConfig": {
     "root": true,

diff --git a/multichain-testing/test/stake-ica.test.ts b/multichain-testing/test/stake-ica.test.ts
@@ -243,7 +243,10 @@ const stakeScenario = test.macro(async (t, scenario: StakeIcaScenario) => {
 
   console.log('waiting for unbonding period');
   // XXX reference `120000` from chain state + `maxClockSkew`
-  await sleep(120000);
+  await sleep(120000, {
+    stir: description =>
+      t.pass(`stirring while waiting for unbonding period: ${description}`),
+  });
   const { balances: rewardsWithUndelegations } = await retryUntilCondition(
     () => queryClient.queryBalances(address),
     ({ balances }) => {

diff --git a/multichain-testing/test/stir.test.ts b/multichain-testing/test/stir.test.ts
@@ -0,0 +1,22 @@
+import test from '@endo/ses-ava/prepare-endo.js';
+import { sleep } from '../tools/sleep.js';
+
+test('sleep without tripping AVA timeout', async t => {
+  const stirred: string[] = [];
+  t.timeout(500);
+  const sleepTime = 4_000;
+  await sleep(sleepTime, {
+    log: t.log,
+    stirEveryMs: 300,
+    stir: description => {
+      stirred.push(description);
+      t.pass(description);
+    },
+  });
+  const stirs = new Array(13);
+  for (let i = 0; i < stirs.length; i += 1) {
+    stirs[i] = `stir #${i + 1}`;
+  }
+  const expected = [`sleeping for ${sleepTime}ms...`, ...stirs];
+  t.deepEqual(stirred, expected);
+});
diff --git a/multichain-testing/test/support.ts b/multichain-testing/test/support.ts
@@ -7,9 +7,14 @@ import { makeAgdTools } from '../tools/agd-tools.js';
 import { type E2ETools } from '../tools/e2e-tools.js';
 import { makeGetFile, makeSetupRegistry } from '../tools/registry.js';
 import { generateMnemonic } from '../tools/wallet.js';
-import { makeRetryUntilCondition } from '../tools/sleep.js';
+import {
+  makeRetryUntilCondition,
+  sleep,
+  stirUntilSettled,
+} from '../tools/sleep.js';
 import { makeDeployBuilder } from '../tools/deploy.js';
 import { makeHermes } from '../tools/hermes-tools.js';
+import type { ExecAsync, ExecSync } from '../tools/agd-lib.js';
 
 export const FAUCET_POUR = 10_000n * 1_000_000n;
 
@@ -53,16 +58,87 @@ const makeKeyring = async (
   return { setupTestKeys, deleteTestKeys };
 };
 
+const ambientSetTimeout = globalThis.setTimeout;
+const ambientFetch = globalThis.fetch;
+
+export const makeStirringOptions = (
+  t: ExecutionContext,
+  stir: (description: string) => void,
+  setTimeout = ambientSetTimeout,
+) => ({
+  log: t.log,
+  stir,
+  setTimeout,
+});
+
+const makeStirringPowers = (
+  t: ExecutionContext,
+  {
+    fetch = ambientFetch,
+    execFile = childProcess.execFile,
+    setTimeout = ambientSetTimeout,
+  } = {},
+) => {
+  const stirWith = (stir: (description: string) => void) =>
+    makeStirringOptions(t, stir, setTimeout);
+  const delay = (ms: number) =>
+    sleep(
+      ms,
+      stirWith(description => t.pass(`stirring in commonSetup ${description}`)),
+    );
+
+  const stirringFetch: typeof fetch = async (
+    ...args: Parameters<typeof fetch>
+  ) => {
+    const resultP = ambientFetch(...args);
+    return stirUntilSettled(
+      resultP,
+      stirWith(description => t.pass(`stirring during fetch ${description}`)),
+    );
+  };
+
+  const execFileAsync: ExecAsync = (file, args, opts) => {
+    const stdoutP = new Promise<string>((resolve, reject) => {
+      execFile(
+        file,
+        args,
+        { ...opts, encoding: 'utf-8' },
+        (error, stdout, _stderr) => {
+          if (error) {
+            reject(error);
+          } else {
+            resolve(stdout);
+          }
+        },
+      );
+    });
+    return stirUntilSettled(
+      stdoutP,
+      stirWith(description =>
+        t.pass(`stirring in execFileAsync ${description}`),
+      ),
+    );
+  };
+  return { delay, fetch: stirringFetch, execFileAsync };
+};
+
 export const commonSetup = async (t: ExecutionContext) => {
   const { useChain } = await setupRegistry();
-  const tools = await makeAgdTools(t.log, childProcess);
+  const stirringPowers = makeStirringPowers(t);
+  const tools = await makeAgdTools(t.log, {
+    execFileSync: childProcess.execFileSync as ExecSync,
+    ...stirringPowers,
+  });
   const keyring = await makeKeyring(tools);
   const deployBuilder = makeDeployBuilder(tools, fse.readJSON, execa);
-  const retryUntilCondition = makeRetryUntilCondition({
-    log: t.log,
-    setTimeout: globalThis.setTimeout,
+  const retryUntilCondition = makeRetryUntilCondition(
+    makeStirringOptions(t, description =>
+      t.pass(`stirring in retryUntilCondition ${description}`),
+    ),
+  );
+  const hermes = makeHermes({
+    execFileSync: childProcess.execFileSync as ExecSync,
   });
-  const hermes = makeHermes(childProcess);
 
   /**
    * Starts a contract if instance not found. Takes care of installing

diff --git a/multichain-testing/tools/agd-lib.js b/multichain-testing/tools/agd-lib.js
@@ -32,14 +32,37 @@ export const flags = record => {
  * @callback ExecSync
  * @param {string} file
  * @param {string[]} args
- * @param {{ encoding: 'utf-8' } & { [k: string]: unknown }} opts
+ * @param {{ [k: string]: unknown }} [opts]
  * @returns {string}
  */
 
 /**
- * @param {{ execFileSync: ExecSync }} io
+ * @typedef {(...args: Parameters<ExecSync>) => Promise<ReturnType<ExecSync>>} ExecAsync
  */
-export const makeAgd = ({ execFileSync }) => {
+
+/**
+ * @param {ExecAsync} [execFileAsync]
+ * @param {ExecSync} [execFileSync]
+ * @returns {ExecAsync}
+ */
+const makeExecFileAsync = (execFileAsync, execFileSync) => {
+  if (execFileAsync) {
+    return execFileAsync;
+  }
+  if (!execFileSync) {
+    throw TypeError('execFileAsync or execFileSync required');
+  }
+  return async (...args) => {
+    return execFileSync(...args);
+  };
+};
+
+/**
+ * @param {{ execFileSync: ExecSync, execFileAsync?: ExecAsync }} io
+ */
+export const makeAgd = ({ execFileSync, execFileAsync: rawExecFileAsync }) => {
+  const execFileAsync = makeExecFileAsync(rawExecFileAsync, execFileSync);
+
   /**
    * @param { {
    *       home?: string;
@@ -67,6 +90,15 @@ export const makeAgd = ({ execFileSync }) => {
       opts = { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] },
     ) => execFileSync(kubectlBinary, [...binaryArgs, ...args], opts);
 
+    /**
+     * @param {string[]} args
+     * @param {Parameters<ExecSync>[2]} [opts]
+     */
+    const execAsync = async (
+      args,
+      opts = { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] },
+    ) => execFileAsync(kubectlBinary, [...binaryArgs, ...args], opts);
+
     const outJson = flags({ output: 'json' });
 
     const ro = freeze({
@@ -132,7 +164,12 @@ export const makeAgd = ({ execFileSync }) => {
           ...outJson,
         ];
         console.log('$$$ agd', ...args);
-        const out = exec(args, { stdio: ['ignore', 'pipe', 'ignore'] });
+        // This is practically the only command that takes longer than the
+        // default AVA 10s timeout.
+        const out = await execAsync(args, {
+          encoding: 'utf-8',
+          stdio: ['ignore', 'pipe', 'ignore'],
+        });
         try {
           const detail = JSON.parse(out);
           if (detail.code !== 0) {
@@ -183,34 +220,38 @@ export const makeAgd = ({ execFileSync }) => {
 
 /** @typedef {ReturnType<makeAgd>} Agd */
 
-/** @param {{ execFileSync: typeof import('child_process').execFileSync, log: typeof console.log }} powers */
+/** @param {{execFileSync?: ExecSync, execFileAsync?: ExecAsync, log: typeof console.log }} powers */
 export const makeCopyFiles = (
-  { execFileSync, log },
+  { execFileSync, execFileAsync: rawExecFileAsync, log },
   {
     podName = 'agoriclocal-genesis-0',
     containerName = 'validator',
     destDir = '/tmp/contracts',
   } = {},
 ) => {
+  // Provide a default execFileAsync if it's not specified.
+  /** @type {ExecAsync} */
+  const execFileAsync = makeExecFileAsync(rawExecFileAsync, execFileSync);
+
   /** @param {string[]} paths } */
-  return paths => {
+  return async paths => {
     // Create the destination directory if it doesn't exist
-    execFileSync(
+    await execFileAsync(
       kubectlBinary,
       `exec -i ${podName} -c ${containerName} -- mkdir -p ${destDir}`.split(
         ' ',
       ),
       { stdio: ['ignore', 'pipe', 'ignore'] },
     );
     for (const path of paths) {
-      execFileSync(
+      await execFileAsync(
         kubectlBinary,
         `cp ${path} ${podName}:${destDir}/ -c ${containerName}`.split(' '),
         { stdio: ['ignore', 'pipe', 'ignore'] },
       );
       log(`Copied ${path} to ${destDir} in pod ${podName}`);
     }
-    const lsOutput = execFileSync(
+    const lsOutput = await execFileAsync(
       kubectlBinary,
       `exec -i ${podName} -c ${containerName}  -- ls ${destDir}`.split(' '),
       { stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf-8' },

diff --git a/multichain-testing/tools/agd-tools.ts b/multichain-testing/tools/agd-tools.ts
@@ -1,19 +1,28 @@
 import { unsafeMakeBundleCache } from '@agoric/swingset-vat/tools/bundleTool.js';
 import { makeE2ETools } from './e2e-tools.js';
+import type { ExecAsync, ExecSync } from './agd-lib.js';
 
 export const makeAgdTools = async (
   log: typeof console.log,
   {
-    execFile,
     execFileSync,
-  }: Pick<typeof import('child_process'), 'execFile' | 'execFileSync'>,
+    delay,
+    fetch = globalThis.fetch,
+    execFileAsync,
+  }: {
+    execFileSync: ExecSync;
+    delay?: (ms: number) => Promise<void>;
+    fetch?: typeof globalThis.fetch;
+    execFileAsync?: ExecAsync;
+  },
 ) => {
   const bundleCache = await unsafeMakeBundleCache('bundles');
   const tools = await makeE2ETools(log, bundleCache, {
     execFileSync,
-    execFile,
+    execFileAsync,
     fetch,
     setTimeout,
+    delay,
   });
   return tools;
 };

diff --git a/multichain-testing/tools/deploy.ts b/multichain-testing/tools/deploy.ts
@@ -22,7 +22,7 @@ export const makeDeployBuilder = (
     console.log(plan);
 
     console.log('copying files to container');
-    tools.copyFiles([
+    await tools.copyFiles([
       nodeRequire.resolve(`../${plan.script}`),
       nodeRequire.resolve(`../${plan.permit}`),
       ...plan.bundles.map((b: CoreEvalPlan['bundles'][0]) => b.fileName),

diff --git a/multichain-testing/tools/e2e-tools.js b/multichain-testing/tools/e2e-tools.js
@@ -1,4 +1,4 @@
-/** global harden */
+/** global harden, globalThis */
 import { assert } from '@endo/errors';
 import { E, Far } from '@endo/far';
 import { Nat } from '@endo/nat';
@@ -409,10 +409,11 @@ const runCoreEval = async (
  * @param {typeof console.log} log
  * @param {import('@agoric/swingset-vat/tools/bundleTool.js').BundleCache} bundleCache
  * @param {object} io
- * @param {typeof import('child_process').execFileSync} io.execFileSync
- * @param {typeof import('child_process').execFile} io.execFile
+ * @param {import('./agd-lib.js').ExecSync} io.execFileSync
+ * @param {import('./agd-lib.js').ExecAsync} [io.execFileAsync]
  * @param {typeof window.fetch} io.fetch
- * @param {typeof window.setTimeout} io.setTimeout
+ * @param {typeof window.setTimeout} [io.setTimeout]
+ * @param {(ms: number) => Promise<void>} [io.delay]
  * @param {string} [io.bundleDir]
  * @param {string} [io.rpcAddress]
  * @param {string} [io.apiAddress]
@@ -423,16 +424,19 @@ export const makeE2ETools = async (
   bundleCache,
   {
     execFileSync,
+    execFileAsync,
     fetch,
     setTimeout,
     rpcAddress = 'http://localhost:26657',
     apiAddress = 'http://localhost:1317',
+    delay = ms => new Promise(resolve => setTimeout(resolve, ms)),
   },
 ) => {
-  const agd = makeAgd({ execFileSync }).withOpts({ keyringBackend: 'test' });
+  const agd = makeAgd({ execFileSync, execFileAsync }).withOpts({
+    keyringBackend: 'test',
+  });
   const rpc = makeHttpClient(rpcAddress, fetch);
   const lcd = makeAPI(apiAddress, { fetch });
-  const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 
   const explainDelay = (ms, info) => {
     if (typeof info === 'object' && Object.keys(info).length > 0) {
@@ -504,7 +508,7 @@ export const makeE2ETools = async (
     return proposal;
   };
 
-  const copyFiles = makeCopyFiles({ execFileSync, log });
+  const copyFiles = makeCopyFiles({ execFileSync, execFileAsync, log });
 
   const vstorageClient = makeQueryKit(vstorage).query;