test: add retry logic to provisionSmartWallet

- motivated after observing "cannot read data of published.wallet.agoric1ujmk0492mauq2f2vrcn7ylq3w3x55k0ap9mt2p.current: fetch failed"
  in https://github.com/Agoric/agoric-sdk/actions/runs/12313012295/job/34369641775?pr=10638
- this likely indicates a race between `provision-one`, the vstorage update, or the RPCs view of vstorage
- to address this, retry the vstorage query with `retryUntilCondition`

multichain-testing/test/fast-usdc/fast-usdc.test.ts

@@ -50,17 +50,13 @@ test.before(async t => {
   const wallets = await setupTestKeys(accounts, values(oracleMnemonics));
 
   // provision oracle wallets first so invitation deposits don't fail
-  const oracleWdPs = keys(oracleMnemonics).map(n =>
-    provisionSmartWallet(wallets[n], {
-      BLD: 100n,
-    }),
+  const oracleWds = await Promise.all(
+    keys(oracleMnemonics).map(n =>
+      provisionSmartWallet(wallets[n], {
+        BLD: 100n,
+      }),
+    ),
   );
-  // execute sequentially, to avoid "published.wallet.${addr}.current: fetch failed"
-  const oracleWds: WalletDriver[] = [];
-  for (const p of oracleWdPs) {
-    const wd = await p;
-    oracleWds.push(wd);
-  }
 
   // calculate denomHash and channelId for privateArgs / builder opts
   const { getTransferChannelId, toDenomHash } = makeDenomTools(chainInfo);

multichain-testing/test/support.ts

@@ -79,13 +79,13 @@ export const commonSetup = async (t: ExecutionContext) => {
     console.error('setupRegistry failed', e);
     throw e;
   }
-  const tools = await makeAgdTools(t.log, childProcess);
-  const keyring = await makeKeyring(tools);
-  const deployBuilder = makeDeployBuilder(tools, fse.readJSON, execa);
   const retryUntilCondition = makeRetryUntilCondition({
     log: t.log,
     setTimeout: globalThis.setTimeout,
   });
+  const tools = await makeAgdTools(t.log, childProcess, retryUntilCondition);
+  const keyring = await makeKeyring(tools);
+  const deployBuilder = makeDeployBuilder(tools, fse.readJSON, execa);
   const hermes = makeHermes(childProcess);
   const nobleTools = makeNobleTools(childProcess);
   const assetInfo = makeAssetInfo(starshipChainInfo);

multichain-testing/tools/agd-tools.ts

@@ -1,18 +1,21 @@
 import { unsafeMakeBundleCache } from '@agoric/swingset-vat/tools/bundleTool.js';
 import { makeE2ETools } from './e2e-tools.js';
+import type { RetryUntilCondition } from './sleep.js';
 
 export const makeAgdTools = async (
   log: typeof console.log,
   {
     execFile,
     execFileSync,
   }: Pick<typeof import('child_process'), 'execFile' | 'execFileSync'>,
+  retryUntilCondition: RetryUntilCondition,
 ) => {
   const bundleCache = await unsafeMakeBundleCache('bundles');
   const tools = await makeE2ETools(log, bundleCache, {
     execFileSync,
     execFile,
     fetch,
+    retryUntilCondition,
     setTimeout,
   });
   return tools;

multichain-testing/tools/e2e-tools.js

@@ -9,7 +9,10 @@ import { makeHttpClient, makeAPI } from './makeHttpClient.js';
 import { dedup, makeQueryKit, poll } from './queryKit.js';
 import { makeVStorage } from './batchQuery.js';
 
-/** @import { EnglishMnemonic } from '@cosmjs/crypto'; */
+/**
+ * @import { EnglishMnemonic } from '@cosmjs/crypto';
+ * @import { RetryUntilCondition } from './sleep.js';
+ */
 
 const BLD = '000000ubld';
 
@@ -121,6 +124,7 @@ const installBundle = async (fullPath, opts) => {
  *   blockTool: BlockTool;
  *   lcd: import('./makeHttpClient.js').LCD;
  *   delay: (ms: number) => Promise<void>;
+ *   retryUntilCondition: RetryUntilCondition;
  *   chainId?: string;
  *   whale?: string;
  *   progress?: typeof console.log;
@@ -139,6 +143,7 @@ export const provisionSmartWallet = async (
     whale = 'faucet',
     progress = console.log,
     q = makeQueryKit(makeVStorage(lcd)).query,
+    retryUntilCondition,
   },
 ) => {
   // TODO: skip this query if balances is {}
@@ -187,7 +192,12 @@ export const provisionSmartWallet = async (
     { chainId, from: address, yes: true },
   );
 
-  const info = await q.queryData(`published.wallet.${address}.current`);
+  const info = await retryUntilCondition(
+    () => q.queryData(`published.wallet.${address}.current`),
+    result => !!result,
+    `wallet in vstorage ${address}`,
+    { log: () => {} }, // suppress logs as this is already noisy
+  );
   progress({
     provisioned: address,
     purses: info.purses.length,
@@ -424,6 +434,7 @@ const runCoreEval = async (
  * @param {typeof import('child_process').execFile} io.execFile
  * @param {typeof window.fetch} io.fetch
  * @param {typeof window.setTimeout} io.setTimeout
+ * @param {RetryUntilCondition} io.retryUntilCondition
  * @param {string} [io.bundleDir]
  * @param {string} [io.rpcAddress]
  * @param {string} [io.apiAddress]
@@ -438,6 +449,7 @@ export const makeE2ETools = async (
     setTimeout,
     rpcAddress = 'http://localhost:26657',
     apiAddress = 'http://localhost:1317',
+    retryUntilCondition,
   },
 ) => {
   const agd = makeAgd({ execFileSync }).withOpts({ keyringBackend: 'test' });
@@ -535,6 +547,7 @@ export const makeE2ETools = async (
         lcd,
         delay,
         q: vstorageClient,
+        retryUntilCondition,
       }),
     /**
      * @param {string} name

multichain-testing/tools/sleep.ts

@@ -28,11 +28,11 @@ const retryUntilCondition = async <T>(
   {
     maxRetries = 6,
     retryIntervalMs = 3500,
-    log = () => {},
+    log = console.log,
     setTimeout = ambientSetTimeout,
   }: RetryOptions = {},
 ): Promise<T> => {
-  console.log({ maxRetries, retryIntervalMs, message });
+  log({ maxRetries, retryIntervalMs, message });
   let retries = 0;
 
   while (retries < maxRetries) {
@@ -50,7 +50,7 @@ const retryUntilCondition = async <T>(
     }
 
     retries++;
-    console.log(
+    log(
       `Retry ${retries}/${maxRetries} - Waiting for ${retryIntervalMs}ms for ${message}...`,
     );
     await sleep(retryIntervalMs, { log, setTimeout });