From 6b6a241add95d139abcb57370a4d9c161c21b85c Mon Sep 17 00:00:00 2001
From: 0xPatrick <patrick@0xpatrick.dev>
Date: Fri, 13 Dec 2024 11:45:09 -0500
Subject: [PATCH] test: add retry logic to `provisionSmartWallet`

- motivated after observing "cannot read data of published.wallet.agoric1ujmk0492mauq2f2vrcn7ylq3w3x55k0ap9mt2p.current: fetch failed"
  in https://github.com/Agoric/agoric-sdk/actions/runs/12313012295/job/34369641775?pr=10638
- this likely indicates a race between `provision-one`, the vstorage update, or the RPCs view of vstorage
- to address this, retry the vstorage query with `retryUntilCondition`
---
 .../test/fast-usdc/fast-usdc.test.ts           | 16 ++++++----------
 multichain-testing/tools/e2e-tools.js          | 18 ++++++++++++++++--
 multichain-testing/tools/sleep.ts              |  6 +++---
 3 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/multichain-testing/test/fast-usdc/fast-usdc.test.ts b/multichain-testing/test/fast-usdc/fast-usdc.test.ts
index 3b3d7f79dce..8997384e031 100644
--- a/multichain-testing/test/fast-usdc/fast-usdc.test.ts
+++ b/multichain-testing/test/fast-usdc/fast-usdc.test.ts
@@ -50,17 +50,13 @@ test.before(async t => {
   const wallets = await setupTestKeys(accounts, values(oracleMnemonics));
 
   // provision oracle wallets first so invitation deposits don't fail
-  const oracleWdPs = keys(oracleMnemonics).map(n =>
-    provisionSmartWallet(wallets[n], {
-      BLD: 100n,
-    }),
+  const oracleWds = await Promise.all(
+    keys(oracleMnemonics).map(n =>
+      provisionSmartWallet(wallets[n], {
+        BLD: 100n,
+      }),
+    ),
   );
-  // execute sequentially, to avoid "published.wallet.${addr}.current: fetch failed"
-  const oracleWds: WalletDriver[] = [];
-  for (const p of oracleWdPs) {
-    const wd = await p;
-    oracleWds.push(wd);
-  }
 
   // calculate denomHash and channelId for privateArgs / builder opts
   const { getTransferChannelId, toDenomHash } = makeDenomTools(chainInfo);
diff --git a/multichain-testing/tools/e2e-tools.js b/multichain-testing/tools/e2e-tools.js
index 6ad5575acb7..4710761e014 100644
--- a/multichain-testing/tools/e2e-tools.js
+++ b/multichain-testing/tools/e2e-tools.js
@@ -8,8 +8,12 @@ import { flags, makeAgd, makeCopyFiles } from './agd-lib.js';
 import { makeHttpClient, makeAPI } from './makeHttpClient.js';
 import { dedup, makeQueryKit, poll } from './queryKit.js';
 import { makeVStorage } from './batchQuery.js';
+import { makeRetryUntilCondition } from './sleep.js';
 
-/** @import { EnglishMnemonic } from '@cosmjs/crypto'; */
+/**
+ * @import { EnglishMnemonic } from '@cosmjs/crypto';
+ * @import { RetryUntilCondition } from './sleep.js';
+ */
 
 const BLD = '000000ubld';
 
@@ -121,6 +125,7 @@ const installBundle = async (fullPath, opts) => {
  *   blockTool: BlockTool;
  *   lcd: import('./makeHttpClient.js').LCD;
  *   delay: (ms: number) => Promise<void>;
+ *   retryUntilCondition: RetryUntilCondition;
  *   chainId?: string;
  *   whale?: string;
  *   progress?: typeof console.log;
@@ -139,6 +144,7 @@ export const provisionSmartWallet = async (
     whale = 'faucet',
     progress = console.log,
     q = makeQueryKit(makeVStorage(lcd)).query,
+    retryUntilCondition,
   },
 ) => {
   // TODO: skip this query if balances is {}
@@ -187,7 +193,12 @@ export const provisionSmartWallet = async (
     { chainId, from: address, yes: true },
   );
 
-  const info = await q.queryData(`published.wallet.${address}.current`);
+  const info = await retryUntilCondition(
+    () => q.queryData(`published.wallet.${address}.current`),
+    result => !!result,
+    `wallet in vstorage ${address}`,
+    { log: () => {} }, // suppress logs as this is already noisy
+  );
   progress({
     provisioned: address,
     purses: info.purses.length,
@@ -428,6 +439,7 @@ const runCoreEval = async (
  * @param {string} [io.rpcAddress]
  * @param {string} [io.apiAddress]
  * @param {(...parts: string[]) => string} [io.join]
+ * * @param {RetryUntilCondition} [io.retryUntilCondition]
  */
 export const makeE2ETools = async (
   log,
@@ -438,6 +450,7 @@ export const makeE2ETools = async (
     setTimeout,
     rpcAddress = 'http://localhost:26657',
     apiAddress = 'http://localhost:1317',
+    retryUntilCondition = makeRetryUntilCondition({ log, setTimeout }),
   },
 ) => {
   const agd = makeAgd({ execFileSync }).withOpts({ keyringBackend: 'test' });
@@ -535,6 +548,7 @@ export const makeE2ETools = async (
         lcd,
         delay,
         q: vstorageClient,
+        retryUntilCondition,
       }),
     /**
      * @param {string} name
diff --git a/multichain-testing/tools/sleep.ts b/multichain-testing/tools/sleep.ts
index 1b0d1a58807..eca0392c1b7 100644
--- a/multichain-testing/tools/sleep.ts
+++ b/multichain-testing/tools/sleep.ts
@@ -28,11 +28,11 @@ const retryUntilCondition = async <T>(
   {
     maxRetries = 6,
     retryIntervalMs = 3500,
-    log = () => {},
+    log = console.log,
     setTimeout = ambientSetTimeout,
   }: RetryOptions = {},
 ): Promise<T> => {
-  console.log({ maxRetries, retryIntervalMs, message });
+  log({ maxRetries, retryIntervalMs, message });
   let retries = 0;
 
   while (retries < maxRetries) {
@@ -50,7 +50,7 @@ const retryUntilCondition = async <T>(
     }
 
     retries++;
-    console.log(
+    log(
       `Retry ${retries}/${maxRetries} - Waiting for ${retryIntervalMs}ms for ${message}...`,
     );
     await sleep(retryIntervalMs, { log, setTimeout });