diff --git a/multichain-testing/package.json b/multichain-testing/package.json index 4da3c7459343..8e862ce27461 100644 --- a/multichain-testing/package.json +++ b/multichain-testing/package.json @@ -58,8 +58,7 @@ "**/*.test.ts" ], "concurrency": 1, - "serial": true, - "timeout": "125s" + "serial": true }, "eslintConfig": { "root": true, diff --git a/multichain-testing/test/stake-ica.test.ts b/multichain-testing/test/stake-ica.test.ts index f5fd41fd0203..eaef0e7f2e7b 100644 --- a/multichain-testing/test/stake-ica.test.ts +++ b/multichain-testing/test/stake-ica.test.ts @@ -243,7 +243,10 @@ const stakeScenario = test.macro(async (t, scenario: StakeIcaScenario) => { console.log('waiting for unbonding period'); // XXX reference `120000` from chain state + `maxClockSkew` - await sleep(120000); + await sleep(120000, { + stir: description => + t.pass(`stirring while waiting for unbonding period: ${description}`), + }); const { balances: rewardsWithUndelegations } = await retryUntilCondition( () => queryClient.queryBalances(address), ({ balances }) => { diff --git a/multichain-testing/test/stir.test.ts b/multichain-testing/test/stir.test.ts new file mode 100644 index 000000000000..863d05de22d3 --- /dev/null +++ b/multichain-testing/test/stir.test.ts @@ -0,0 +1,22 @@ +import test from '@endo/ses-ava/prepare-endo.js'; +import { sleep } from '../tools/sleep.js'; + +test('sleep without tripping AVA timeout', async t => { + const stirred: string[] = []; + t.timeout(500); + const sleepTime = 4_000; + await sleep(sleepTime, { + log: t.log, + stirEveryMs: 300, + stir: description => { + stirred.push(description); + t.pass(description); + }, + }); + const stirs = new Array(13); + for (let i = 0; i < stirs.length; i += 1) { + stirs[i] = `stir #${i + 1}`; + } + const expected = [`sleeping for ${sleepTime}ms...`, ...stirs]; + t.deepEqual(stirred, expected); +}); diff --git a/multichain-testing/test/support.ts b/multichain-testing/test/support.ts index 2267b8134367..3ab1b5583c3a 100644 --- a/multichain-testing/test/support.ts +++ b/multichain-testing/test/support.ts @@ -7,9 +7,14 @@ import { makeAgdTools } from '../tools/agd-tools.js'; import { type E2ETools } from '../tools/e2e-tools.js'; import { makeGetFile, makeSetupRegistry } from '../tools/registry.js'; import { generateMnemonic } from '../tools/wallet.js'; -import { makeRetryUntilCondition } from '../tools/sleep.js'; +import { + makeRetryUntilCondition, + sleep, + stirUntilSettled, +} from '../tools/sleep.js'; import { makeDeployBuilder } from '../tools/deploy.js'; import { makeHermes } from '../tools/hermes-tools.js'; +import type { ExecAsync, ExecSync } from '../tools/agd-lib.js'; export const FAUCET_POUR = 10_000n * 1_000_000n; @@ -53,16 +58,87 @@ const makeKeyring = async ( return { setupTestKeys, deleteTestKeys }; }; +const ambientSetTimeout = globalThis.setTimeout; +const ambientFetch = globalThis.fetch; + +export const makeStirringOptions = ( + t: ExecutionContext, + stir: (description: string) => void, + setTimeout = ambientSetTimeout, +) => ({ + log: t.log, + stir, + setTimeout, +}); + +const makeStirringPowers = ( + t: ExecutionContext, + { + fetch = ambientFetch, + execFile = childProcess.execFile, + setTimeout = ambientSetTimeout, + } = {}, +) => { + const stirWith = (stir: (description: string) => void) => + makeStirringOptions(t, stir, setTimeout); + const delay = (ms: number) => + sleep( + ms, + stirWith(description => t.pass(`stirring in commonSetup ${description}`)), + ); + + const stirringFetch: typeof fetch = async ( + ...args: Parameters + ) => { + const resultP = ambientFetch(...args); + return stirUntilSettled( + resultP, + stirWith(description => t.pass(`stirring during fetch ${description}`)), + ); + }; + + const execFileAsync: ExecAsync = (file, args, opts) => { + const stdoutP = new Promise((resolve, reject) => { + execFile( + file, + args, + { ...opts, encoding: 'utf-8' }, + (error, stdout, _stderr) => { + if (error) { + reject(error); + } else { + resolve(stdout); + } + }, + ); + }); + return stirUntilSettled( + stdoutP, + stirWith(description => + t.pass(`stirring in execFileAsync ${description}`), + ), + ); + }; + return { delay, fetch: stirringFetch, execFileAsync }; +}; + export const commonSetup = async (t: ExecutionContext) => { const { useChain } = await setupRegistry(); - const tools = await makeAgdTools(t.log, childProcess); + const stirringPowers = makeStirringPowers(t); + const tools = await makeAgdTools(t.log, { + execFileSync: childProcess.execFileSync as ExecSync, + ...stirringPowers, + }); const keyring = await makeKeyring(tools); const deployBuilder = makeDeployBuilder(tools, fse.readJSON, execa); - const retryUntilCondition = makeRetryUntilCondition({ - log: t.log, - setTimeout: globalThis.setTimeout, + const retryUntilCondition = makeRetryUntilCondition( + makeStirringOptions(t, description => + t.pass(`stirring in retryUntilCondition ${description}`), + ), + ); + const hermes = makeHermes({ + execFileSync: childProcess.execFileSync as ExecSync, }); - const hermes = makeHermes(childProcess); /** * Starts a contract if instance not found. Takes care of installing diff --git a/multichain-testing/tools/agd-lib.js b/multichain-testing/tools/agd-lib.js index a301945f1486..4a33e84dbf6b 100644 --- a/multichain-testing/tools/agd-lib.js +++ b/multichain-testing/tools/agd-lib.js @@ -32,14 +32,37 @@ export const flags = record => { * @callback ExecSync * @param {string} file * @param {string[]} args - * @param {{ encoding: 'utf-8' } & { [k: string]: unknown }} opts + * @param {{ [k: string]: unknown }} [opts] * @returns {string} */ /** - * @param {{ execFileSync: ExecSync }} io + * @typedef {(...args: Parameters) => Promise>} ExecAsync */ -export const makeAgd = ({ execFileSync }) => { + +/** + * @param {ExecAsync} [execFileAsync] + * @param {ExecSync} [execFileSync] + * @returns {ExecAsync} + */ +const makeExecFileAsync = (execFileAsync, execFileSync) => { + if (execFileAsync) { + return execFileAsync; + } + if (!execFileSync) { + throw TypeError('execFileAsync or execFileSync required'); + } + return async (...args) => { + return execFileSync(...args); + }; +}; + +/** + * @param {{ execFileSync: ExecSync, execFileAsync?: ExecAsync }} io + */ +export const makeAgd = ({ execFileSync, execFileAsync: rawExecFileAsync }) => { + const execFileAsync = makeExecFileAsync(rawExecFileAsync, execFileSync); + /** * @param { { * home?: string; @@ -67,6 +90,15 @@ export const makeAgd = ({ execFileSync }) => { opts = { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, ) => execFileSync(kubectlBinary, [...binaryArgs, ...args], opts); + /** + * @param {string[]} args + * @param {Parameters[2]} [opts] + */ + const execAsync = async ( + args, + opts = { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }, + ) => execFileAsync(kubectlBinary, [...binaryArgs, ...args], opts); + const outJson = flags({ output: 'json' }); const ro = freeze({ @@ -132,7 +164,12 @@ export const makeAgd = ({ execFileSync }) => { ...outJson, ]; console.log('$$$ agd', ...args); - const out = exec(args, { stdio: ['ignore', 'pipe', 'ignore'] }); + // This is practically the only command that takes longer than the + // default AVA 10s timeout. + const out = await execAsync(args, { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'ignore'], + }); try { const detail = JSON.parse(out); if (detail.code !== 0) { @@ -183,19 +220,23 @@ export const makeAgd = ({ execFileSync }) => { /** @typedef {ReturnType} Agd */ -/** @param {{ execFileSync: typeof import('child_process').execFileSync, log: typeof console.log }} powers */ +/** @param {{execFileSync?: ExecSync, execFileAsync?: ExecAsync, log: typeof console.log }} powers */ export const makeCopyFiles = ( - { execFileSync, log }, + { execFileSync, execFileAsync: rawExecFileAsync, log }, { podName = 'agoriclocal-genesis-0', containerName = 'validator', destDir = '/tmp/contracts', } = {}, ) => { + // Provide a default execFileAsync if it's not specified. + /** @type {ExecAsync} */ + const execFileAsync = makeExecFileAsync(rawExecFileAsync, execFileSync); + /** @param {string[]} paths } */ - return paths => { + return async paths => { // Create the destination directory if it doesn't exist - execFileSync( + await execFileAsync( kubectlBinary, `exec -i ${podName} -c ${containerName} -- mkdir -p ${destDir}`.split( ' ', @@ -203,14 +244,14 @@ export const makeCopyFiles = ( { stdio: ['ignore', 'pipe', 'ignore'] }, ); for (const path of paths) { - execFileSync( + await execFileAsync( kubectlBinary, `cp ${path} ${podName}:${destDir}/ -c ${containerName}`.split(' '), { stdio: ['ignore', 'pipe', 'ignore'] }, ); log(`Copied ${path} to ${destDir} in pod ${podName}`); } - const lsOutput = execFileSync( + const lsOutput = await execFileAsync( kubectlBinary, `exec -i ${podName} -c ${containerName} -- ls ${destDir}`.split(' '), { stdio: ['ignore', 'pipe', 'ignore'], encoding: 'utf-8' }, diff --git a/multichain-testing/tools/agd-tools.ts b/multichain-testing/tools/agd-tools.ts index 5b87be3e1ea4..d8851c2e1027 100644 --- a/multichain-testing/tools/agd-tools.ts +++ b/multichain-testing/tools/agd-tools.ts @@ -1,19 +1,28 @@ import { unsafeMakeBundleCache } from '@agoric/swingset-vat/tools/bundleTool.js'; import { makeE2ETools } from './e2e-tools.js'; +import type { ExecAsync, ExecSync } from './agd-lib.js'; export const makeAgdTools = async ( log: typeof console.log, { - execFile, execFileSync, - }: Pick, + delay, + fetch = globalThis.fetch, + execFileAsync, + }: { + execFileSync: ExecSync; + delay?: (ms: number) => Promise; + fetch?: typeof globalThis.fetch; + execFileAsync?: ExecAsync; + }, ) => { const bundleCache = await unsafeMakeBundleCache('bundles'); const tools = await makeE2ETools(log, bundleCache, { execFileSync, - execFile, + execFileAsync, fetch, setTimeout, + delay, }); return tools; }; diff --git a/multichain-testing/tools/deploy.ts b/multichain-testing/tools/deploy.ts index 5cd3fba5e53a..8d07023927ff 100755 --- a/multichain-testing/tools/deploy.ts +++ b/multichain-testing/tools/deploy.ts @@ -22,7 +22,7 @@ export const makeDeployBuilder = ( console.log(plan); console.log('copying files to container'); - tools.copyFiles([ + await tools.copyFiles([ nodeRequire.resolve(`../${plan.script}`), nodeRequire.resolve(`../${plan.permit}`), ...plan.bundles.map((b: CoreEvalPlan['bundles'][0]) => b.fileName), diff --git a/multichain-testing/tools/e2e-tools.js b/multichain-testing/tools/e2e-tools.js index df46fbd65b02..7db80d17457e 100644 --- a/multichain-testing/tools/e2e-tools.js +++ b/multichain-testing/tools/e2e-tools.js @@ -1,4 +1,4 @@ -/** global harden */ +/** global harden, globalThis */ import { assert } from '@endo/errors'; import { E, Far } from '@endo/far'; import { Nat } from '@endo/nat'; @@ -409,10 +409,11 @@ const runCoreEval = async ( * @param {typeof console.log} log * @param {import('@agoric/swingset-vat/tools/bundleTool.js').BundleCache} bundleCache * @param {object} io - * @param {typeof import('child_process').execFileSync} io.execFileSync - * @param {typeof import('child_process').execFile} io.execFile + * @param {import('./agd-lib.js').ExecSync} io.execFileSync + * @param {import('./agd-lib.js').ExecAsync} [io.execFileAsync] * @param {typeof window.fetch} io.fetch - * @param {typeof window.setTimeout} io.setTimeout + * @param {typeof window.setTimeout} [io.setTimeout] + * @param {(ms: number) => Promise} [io.delay] * @param {string} [io.bundleDir] * @param {string} [io.rpcAddress] * @param {string} [io.apiAddress] @@ -423,16 +424,19 @@ export const makeE2ETools = async ( bundleCache, { execFileSync, + execFileAsync, fetch, setTimeout, rpcAddress = 'http://localhost:26657', apiAddress = 'http://localhost:1317', + delay = ms => new Promise(resolve => setTimeout(resolve, ms)), }, ) => { - const agd = makeAgd({ execFileSync }).withOpts({ keyringBackend: 'test' }); + const agd = makeAgd({ execFileSync, execFileAsync }).withOpts({ + keyringBackend: 'test', + }); const rpc = makeHttpClient(rpcAddress, fetch); const lcd = makeAPI(apiAddress, { fetch }); - const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); const explainDelay = (ms, info) => { if (typeof info === 'object' && Object.keys(info).length > 0) { @@ -504,7 +508,7 @@ export const makeE2ETools = async ( return proposal; }; - const copyFiles = makeCopyFiles({ execFileSync, log }); + const copyFiles = makeCopyFiles({ execFileSync, execFileAsync, log }); const vstorageClient = makeQueryKit(vstorage).query; diff --git a/multichain-testing/tools/sleep.ts b/multichain-testing/tools/sleep.ts index 66251a87dcaa..722b85db55dc 100644 --- a/multichain-testing/tools/sleep.ts +++ b/multichain-testing/tools/sleep.ts @@ -1,21 +1,91 @@ const ambientSetTimeout = globalThis.setTimeout; +// Derived from the fact that AVA's "default timeout is 10 seconds." +// https://github.com/avajs/ava/blob/main/docs/07-test-timeouts.md#test-timeouts +export const DEFAULT_STIR_EVERY_MS = 8_000; + type Log = (...values: unknown[]) => void; type SleepOptions = { log?: Log; setTimeout?: typeof ambientSetTimeout; + stirEveryMs?: number; + /** + * Call every `stirEveryMs` during sleeping so that the sleeper isn't timed + * out by, e.g. a test runner. + * @param description - A bit of context as to why we're stirring. + */ + stir?: (description: string) => void; }; -export const sleep = ( +/** + * + * @param {number} ms Sleep duration in milliseconds + * @param {SleepOptions} opts + * @returns {Promise} + */ +const deepSleep = ( ms: number, - { log = () => {}, setTimeout = ambientSetTimeout }: SleepOptions = {}, + { setTimeout = ambientSetTimeout }: SleepOptions = {}, ) => new Promise(resolve => { - log(`Sleeping for ${ms}ms...`); setTimeout(resolve, ms); }); +export const stirUntilSettled = async ( + result: T, + { + log = () => {}, + setTimeout = ambientSetTimeout, + stirEveryMs = DEFAULT_STIR_EVERY_MS, + stir, + }: SleepOptions = {}, +): Promise> => { + const resultP = Promise.resolve(result); + if (!stir) { + return resultP; + } + let keepStirring = !!stir; + try { + // Ensure that we stop stirring when the result is settled. + resultP.finally(() => (keepStirring = false)); + + log(`start stirring`); + let nStirs = 0; + while (keepStirring) { + await new Promise(resolve => { + setTimeout(resolve, stirEveryMs); + }); + nStirs += 1; + await (keepStirring && stir(`stir #${nStirs}`)); + } + log(`done stirring`); + return resultP; + } finally { + keepStirring = false; + } +}; + +export const sleep = async ( + ms: number, + { + log = () => {}, + stir, + stirEveryMs = DEFAULT_STIR_EVERY_MS, + setTimeout = ambientSetTimeout, + }: SleepOptions = {}, +) => { + log(`Sleeping for ${ms}ms...`); + await (stir && stir(`sleeping for ${ms}ms...`)); + const slept = deepSleep(ms, { setTimeout }); + await stirUntilSettled(slept, { + log, + stir, + stirEveryMs, + setTimeout, + }); +}; + export type RetryOptions = { maxRetries?: number; retryIntervalMs?: number; @@ -29,6 +99,8 @@ const retryUntilCondition = async ( maxRetries = 6, retryIntervalMs = 3500, log = () => {}, + stirEveryMs, + stir, setTimeout = ambientSetTimeout, }: RetryOptions = {}, ): Promise => { @@ -37,7 +109,13 @@ const retryUntilCondition = async ( while (retries < maxRetries) { try { - const result = await operation(); + const resultP = operation(); + const result = await stirUntilSettled(resultP, { + log, + stirEveryMs, + stir, + setTimeout, + }); if (condition(result)) { return result; } @@ -53,7 +131,7 @@ const retryUntilCondition = async ( console.log( `Retry ${retries}/${maxRetries} - Waiting for ${retryIntervalMs}ms for ${message}...`, ); - await sleep(retryIntervalMs, { log, setTimeout }); + await sleep(retryIntervalMs, { log, setTimeout, stirEveryMs, stir }); } throw Error(`${message} condition failed after ${maxRetries} retries.`);