diff --git a/packages/internal/src/upgrade-api.js b/packages/internal/src/upgrade-api.js index c2cadd7be165..a1b0fa9c640c 100644 --- a/packages/internal/src/upgrade-api.js +++ b/packages/internal/src/upgrade-api.js @@ -45,5 +45,23 @@ harden(makeUpgradeDisconnection); * @returns {reason is UpgradeDisconnection} */ export const isUpgradeDisconnection = reason => - isFrozen(reason) && matches(reason, UpgradeDisconnectionShape); + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, UpgradeDisconnectionShape); harden(isUpgradeDisconnection); + +/** + * Returns whether a reason is a 'vat terminated' error generated when an object + * is abandoned by a vat during an upgrade. + * + * @param {any} reason + * @returns {reason is Error} + */ +export const isAbandonedError = reason => + reason != null && // eslint-disable-line eqeqeq + isFrozen(reason) && + matches(reason, M.error()) && + // We're not using a constant here since this special value is already + // sprinkled throughout the SDK + reason.message === 'vat terminated'; +harden(isAbandonedError); diff --git a/packages/internal/test/upgrade-api.test.js b/packages/internal/test/upgrade-api.test.js index 5ffc38f9b29f..f8eee5d2d926 100644 --- a/packages/internal/test/upgrade-api.test.js +++ b/packages/internal/test/upgrade-api.test.js @@ -1,8 +1,11 @@ // @ts-check import test from 'ava'; +import { makeMarshal } from '@endo/marshal'; + import { makeUpgradeDisconnection, isUpgradeDisconnection, + isAbandonedError, } from '../src/upgrade-api.js'; test('isUpgradeDisconnection must recognize disconnection objects', t => { @@ -18,3 +21,15 @@ test('isUpgradeDisconnection must recognize original-format disconnection object }); t.true(isUpgradeDisconnection(disconnection)); }); + +test('isAbandonedError recognizes marshalled vat terminated errors', t => { + const { fromCapData, toCapData } = makeMarshal(undefined, undefined, { + serializeBodyFormat: 'smallcaps', + errorIdNum: 70_000, + marshalSaveError: () => {}, + }); + const error = harden(Error('vat terminated')); + const remoteError = fromCapData(toCapData(error)); + + t.true(isAbandonedError(remoteError)); +}); diff --git a/packages/orchestration/src/examples/stakeBld.contract.js b/packages/orchestration/src/examples/stakeBld.contract.js index 935c72b92263..d8cc34d18296 100644 --- a/packages/orchestration/src/examples/stakeBld.contract.js +++ b/packages/orchestration/src/examples/stakeBld.contract.js @@ -47,7 +47,7 @@ export const start = async (zcf, privateArgs, baggage) => { zcf, privateArgs.timerService, vowTools, - makeChainHub(privateArgs.agoricNames, vowTools), + makeChainHub(zone.subZone('chainHub'), privateArgs.agoricNames, vowTools), ); // ---------------- diff --git a/packages/orchestration/src/exos/chain-hub.js b/packages/orchestration/src/exos/chain-hub.js index 24b280085b3c..6687cf45db9d 100644 --- a/packages/orchestration/src/exos/chain-hub.js +++ b/packages/orchestration/src/exos/chain-hub.js @@ -4,7 +4,6 @@ import { M } from '@endo/patterns'; import { BrandShape } from '@agoric/ertp/src/typeGuards.js'; import { VowShape } from '@agoric/vow'; -import { makeHeapZone } from '@agoric/zone'; import { CosmosChainInfoShape, IBCConnectionInfoShape } from '../typeGuards.js'; /** @@ -169,18 +168,18 @@ const ChainHubI = M.interface('ChainHub', { }); /** - * Make a new ChainHub in the zone (or in the heap if no zone is provided). + * Make a new ChainHub in the zone. * * The resulting object is an Exo singleton. It has no precious state. It's only * state is a cache of queries to agoricNames and whatever info was provided in * registration calls. When you need a newer version you can simply make a hub * hub and repeat the registrations. * + * @param {Zone} zone * @param {Remote} agoricNames * @param {VowTools} vowTools */ -export const makeChainHub = (agoricNames, vowTools) => { - const zone = makeHeapZone(); +export const makeChainHub = (zone, agoricNames, vowTools) => { /** @type {MapStore} */ const chainInfos = zone.mapStore('chainInfos', { keyShape: M.string(), diff --git a/packages/orchestration/src/proposals/start-stakeAtom.js b/packages/orchestration/src/proposals/start-stakeAtom.js index 8035f4fda1c1..a9e0e9e3282a 100644 --- a/packages/orchestration/src/proposals/start-stakeAtom.js +++ b/packages/orchestration/src/proposals/start-stakeAtom.js @@ -46,8 +46,13 @@ export const startStakeAtom = async ({ const storageNode = await makeStorageNodeChild(chainStorage, VSTORAGE_PATH); const marshaller = await E(board).getPublishingMarshaller(); - const vt = prepareVowTools(makeHeapZone()); - const chainHub = makeChainHub(await agoricNames, vt); + const zone = makeHeapZone(); + const vt = prepareVowTools(zone.subZone('vows')); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + await agoricNames, + vt, + ); const [_, cosmoshub, connectionInfo] = await vt.when( chainHub.getChainsAndConnection('agoric', 'cosmoshub'), diff --git a/packages/orchestration/src/proposals/start-stakeOsmo.js b/packages/orchestration/src/proposals/start-stakeOsmo.js index 6fda8e812de2..083c4c6449b2 100644 --- a/packages/orchestration/src/proposals/start-stakeOsmo.js +++ b/packages/orchestration/src/proposals/start-stakeOsmo.js @@ -51,8 +51,13 @@ export const startStakeOsmo = async ({ const storageNode = await makeStorageNodeChild(chainStorage, VSTORAGE_PATH); const marshaller = await E(board).getPublishingMarshaller(); - const vt = prepareVowTools(makeHeapZone()); - const chainHub = makeChainHub(await agoricNames, vt); + const zone = makeHeapZone(); + const vt = prepareVowTools(zone.subZone('vows')); + const chainHub = makeChainHub( + zone.subZone('chainHub'), + await agoricNames, + vt, + ); const [_, osmosis, connectionInfo] = await vt.when( chainHub.getChainsAndConnection('agoric', 'osmosis'), diff --git a/packages/orchestration/src/utils/start-helper.js b/packages/orchestration/src/utils/start-helper.js index f065e3097aea..3c6a02cec819 100644 --- a/packages/orchestration/src/utils/start-helper.js +++ b/packages/orchestration/src/utils/start-helper.js @@ -62,6 +62,7 @@ export const provideOrchestration = ( /** for contract-provided names */ contract: zone.subZone('contract'), orchestration: zone.subZone('orchestration'), + chainHub: zone.subZone('chainHub'), vows: zone.subZone('vows'), zoe: zone.subZone('zoe'), }; @@ -71,7 +72,7 @@ export const provideOrchestration = ( const vowTools = prepareVowTools(zones.vows); - const chainHub = makeChainHub(agoricNames, vowTools); + const chainHub = makeChainHub(zones.chainHub, agoricNames, vowTools); const zoeTools = makeZoeTools(zones.zoe, { zcf, vowTools }); diff --git a/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.md b/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.md index ddff92fb013b..ae2462472f3c 100644 --- a/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.md @@ -24,11 +24,22 @@ Generated by [AVA](https://avajs.dev). flowForOutcomeVow: {}, unwrapMap: 'Alleged: weakMapStore', }, + chainHub: { + ChainHub_kindHandle: 'Alleged: kind', + ChainHub_singleton: 'Alleged: ChainHub', + chainInfos: {}, + connectionInfos: {}, + denom: {}, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', + }, contract: { 'ChainHub Admin_kindHandle': 'Alleged: kind', 'ChainHub Admin_singleton': 'Alleged: ChainHub Admin', 'Send PF_kindHandle': 'Alleged: kind', 'Send PF_singleton': 'Alleged: Send PF', + findBrandInVBank_kindHandle: 'Alleged: kind', }, orchestration: { 'Cosmos Orchestration Account Holder_kindHandle': 'Alleged: kind', @@ -52,9 +63,14 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retriableFlowForOutcomeVow: {}, + }, + zoe: { + localTransfer_kindHandle: 'Alleged: kind', }, - zoe: {}, } diff --git a/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.snap b/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.snap index 85e446a9dcb5..8f1084a0c1d1 100644 Binary files a/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.snap and b/packages/orchestration/test/examples/snapshots/sendAnywhere.test.ts.snap differ diff --git a/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.md b/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.md index f714aeab3a4c..4ccb13489b2c 100644 --- a/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.md +++ b/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.md @@ -24,6 +24,87 @@ Generated by [AVA](https://avajs.dev). flowForOutcomeVow: {}, unwrapMap: 'Alleged: weakMapStore', }, + chainHub: { + ChainHub_kindHandle: 'Alleged: kind', + ChainHub_singleton: 'Alleged: ChainHub', + chainInfos: { + agoric: { + chainId: 'agoric-3', + icqEnabled: false, + stakingTokens: [ + { + denom: 'ubld', + }, + ], + }, + omniflixhub: { + chainId: 'omniflixhub-1', + icqEnabled: false, + stakingTokens: [ + { + denom: 'uflix', + }, + ], + }, + stride: { + chainId: 'stride-1', + icqEnabled: false, + stakingTokens: [ + { + denom: 'ustrd', + }, + ], + }, + }, + connectionInfos: { + 'agoric-3_omniflixhub-1': { + client_id: '07-tendermint-73', + counterparty: { + client_id: '07-tendermint-47', + connection_id: 'connection-40', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-67', + state: 3, + transferChannel: { + channelId: 'channel-58', + counterPartyChannelId: 'channel-30', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + 'agoric-3_stride-1': { + client_id: '07-tendermint-74', + counterparty: { + client_id: '07-tendermint-129', + connection_id: 'connection-118', + prefix: { + key_prefix: 'FIXME', + }, + }, + id: 'connection-68', + state: 3, + transferChannel: { + channelId: 'channel-59', + counterPartyChannelId: 'channel-148', + counterPartyPortId: 'transfer', + ordering: 0, + portId: 'transfer', + state: 3, + version: 'ics20-1', + }, + }, + }, + denom: {}, + lookupChainInfo_kindHandle: 'Alleged: kind', + lookupChainsAndConnection_kindHandle: 'Alleged: kind', + lookupConnectionInfo_kindHandle: 'Alleged: kind', + }, contract: { publicFacet_kindHandle: 'Alleged: kind', publicFacet_singleton: 'Alleged: publicFacet', @@ -43,9 +124,14 @@ Generated by [AVA](https://avajs.dev). }, }, vows: { + AdminRetriableFlow_kindHandle: 'Alleged: kind', + AdminRetriableFlow_singleton: 'Alleged: AdminRetriableFlow', PromiseWatcher_kindHandle: 'Alleged: kind', VowInternalsKit_kindHandle: 'Alleged: kind', WatchUtils_kindHandle: 'Alleged: kind', + retriableFlowForOutcomeVow: {}, + }, + zoe: { + localTransfer_kindHandle: 'Alleged: kind', }, - zoe: {}, } diff --git a/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.snap b/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.snap index 55ffa320693b..8f52069a1c0a 100644 Binary files a/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.snap and b/packages/orchestration/test/examples/snapshots/unbondExample.test.ts.snap differ diff --git a/packages/orchestration/test/exos/chain-hub.test.ts b/packages/orchestration/test/exos/chain-hub.test.ts index e5cb3c05c07e..d0dc7c3dac96 100644 --- a/packages/orchestration/test/exos/chain-hub.test.ts +++ b/packages/orchestration/test/exos/chain-hub.test.ts @@ -43,9 +43,9 @@ const connection = { // fresh state for each test const setup = () => { const zone = provideDurableZone('root'); - const vt = prepareSwingsetVowTools(zone); + const vt = prepareSwingsetVowTools(zone.subZone('vows')); const { nameHub, nameAdmin } = makeNameHubKit(); - const chainHub = makeChainHub(nameHub, vt); + const chainHub = makeChainHub(zone.subZone('chainHub'), nameHub, vt); return { chainHub, nameAdmin, vt }; }; diff --git a/packages/orchestration/test/exos/make-test-loa-kit.ts b/packages/orchestration/test/exos/make-test-loa-kit.ts index 765e7d48f584..f7c53b0a803d 100644 --- a/packages/orchestration/test/exos/make-test-loa-kit.ts +++ b/packages/orchestration/test/exos/make-test-loa-kit.ts @@ -39,7 +39,7 @@ export const prepareMakeTestLOAKit = ( zcf, timer, vowTools, - makeChainHub(agoricNames, vowTools), + makeChainHub(rootZone.subZone('chainHub'), agoricNames, vowTools), ); return async ({ diff --git a/packages/vow/src/retriable.js b/packages/vow/src/retriable.js new file mode 100644 index 000000000000..c58e7621cfdf --- /dev/null +++ b/packages/vow/src/retriable.js @@ -0,0 +1,218 @@ +import { Fail } from '@endo/errors'; +import { M } from '@endo/patterns'; +import { PromiseWatcherI } from '@agoric/base-zone'; +import { toPassableCap, VowShape } from './vow-utils.js'; + +/** + * @import {WeakMapStore} from '@agoric/store' + * @import {Zone} from '@agoric/base-zone' + * @import {Vow, VowKit, IsRetryableReason} from './types.js' + * @import {Passable} from '@endo/pass-style' + */ + +/** + * @typedef {object} PreparationOptions + * @property {() => VowKit} makeVowKit + * @property {IsRetryableReason} isRetryableReason + */ + +/** + * @template {Passable[]} [TArgs=Passable[]] + * @template {any} [TRet=any] + * @typedef {(...args: TArgs) => Promise} RetriableFunc + */ + +const { defineProperties } = Object; + +const RetriableFlowIKit = harden({ + flow: M.interface('Flow', { + restart: M.call().returns(), + getOutcome: M.call().returns(VowShape), + }), + resultWatcher: PromiseWatcherI, +}); + +const AdminRetriableFlowI = M.interface('RetriableFlowAdmin', { + getFlowForOutcomeVow: M.call(VowShape).returns(M.opt(M.remotable('flow'))), +}); + +/** + * @param {Zone} outerZone + * @param {PreparationOptions} [outerOptions] + */ +export const prepareRetriableTools = (outerZone, outerOptions = {}) => { + const { makeVowKit, isRetryableReason } = outerOptions; + + /** + * So we can give out wrapper functions easily and recover flow objects + * for their activations later. + */ + const flowForOutcomeVowKey = outerZone.mapStore( + 'retriableFlowForOutcomeVow', + { + keyShape: M.remotable('toPassableCap'), + valueShape: M.remotable('flow'), // isDone === false + }, + ); + + /** + * @param {Zone} zone + * @param {string} tag + * @param {RetriableFunc} retriableFunc + */ + const prepareRetriableFlowKit = (zone, tag, retriableFunc) => { + typeof retriableFunc === 'function' || + Fail`retriableFunc must be a callable function ${retriableFunc}`; + + const internalMakeRetriableFlowKit = zone.exoClassKit( + tag, + RetriableFlowIKit, + activationArgs => { + harden(activationArgs); + + return { + activationArgs, // restarting the retriable function uses the original args + outcomeKit: makeVowKit(), // outcome of activation as vow + lastRetryReason: undefined, + runs: 0n, + isDone: false, // persistently done + }; + }, + { + flow: { + /** + * Calls the retriable function, either for the initial run or when + * the result of the previous run fails with a retriable reason. + */ + restart() { + const { state, facets } = this; + const { activationArgs, isDone } = state; + const { flow, resultWatcher } = facets; + + !isDone || + // separate line so I can set a breakpoint + Fail`Cannot restart a done retriable flow ${flow}`; + + const runId = state.runs + 1n; + state.runs = runId; + + let resultP; + try { + resultP = Promise.resolve(retriableFunc(...activationArgs)); + } catch (err) { + resultP = Promise.resolve(() => Promise.reject(err)); + } + + outerZone.watchPromise(harden(resultP), resultWatcher, runId); + }, + getOutcome() { + const { state } = this; + const { outcomeKit } = state; + return outcomeKit.vow; + }, + }, + resultWatcher: { + onFulfilled(value, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot resolve a done retriable flow ${this.facets.flow}`; + outcomeKit.resolver.resolve(value); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + }, + onRejected(reason, runId) { + const { state } = this; + const { runs, outcomeKit } = state; + if (runId !== runs) return; + !state.isDone || + Fail`Cannot reject a done retriable flow ${this.facets.flow}`; + const retryReason = isRetryableReason( + reason, + state.lastRetryReason, + ); + if (retryReason) { + state.lastRetryReason = retryReason; + this.facets.flow.restart(); + } else { + outcomeKit.resolver.reject(reason); + flowForOutcomeVowKey.delete(toPassableCap(outcomeKit.vow)); + state.isDone = true; + } + }, + }, + }, + ); + const makeRetriableFlowKit = activationArgs => { + const retriableKit = internalMakeRetriableFlowKit(activationArgs); + const { flow } = retriableKit; + + const vow = flow.getOutcome(); + flowForOutcomeVowKey.init(toPassableCap(vow), flow); + flow.restart(); + return retriableKit; + }; + return harden(makeRetriableFlowKit); + }; + + /** + * @template {RetriableFunc} F + * @param {Zone} zone + * @param {string} tag + * @param {F} retriableFunc + */ + const retriable = (zone, tag, retriableFunc) => { + const makeRetriableKit = prepareRetriableFlowKit(zone, tag, retriableFunc); + const wrapperFuncName = `${tag}_retriable`; + + const wrapperFunc = { + /** @type {(...args: Parameters) => Vow>>} */ + [wrapperFuncName](...args) { + const { flow } = makeRetriableKit(args); + return flow.getOutcome(); + }, + }[wrapperFuncName]; + defineProperties(wrapperFunc, { + length: { value: retriableFunc.length }, + }); + return harden(wrapperFunc); + }; + + const adminRetriableFlow = outerZone.exo( + 'AdminRetriableFlow', + AdminRetriableFlowI, + { + getFlowForOutcomeVow(outcomeVow) { + return flowForOutcomeVowKey.get(toPassableCap(outcomeVow)); + }, + }, + ); + + return harden({ + prepareRetriableFlowKit, + adminRetriableFlow, + retriable, + }); +}; +harden(prepareRetriableTools); + +/** + * @typedef {ReturnType} RetriableTools + */ + +/** + * @typedef {RetriableTools['adminRetriableFlow']} AdminRetriableFlow + */ + +/** + * @typedef {ReturnType} MakeRetriableFlowKit + */ + +/** + * @typedef {ReturnType} RetriableFlowKit + */ + +/** + * @typedef {RetriableFlowKit['flow']} RetriableFlow + */ diff --git a/packages/vow/src/tools.js b/packages/vow/src/tools.js index ec224cbe0789..905fa2606b04 100644 --- a/packages/vow/src/tools.js +++ b/packages/vow/src/tools.js @@ -3,6 +3,7 @@ import { makeAsVow } from './vow-utils.js'; import { prepareVowKit } from './vow.js'; import { prepareWatchUtils } from './watch-utils.js'; import { prepareWatch } from './watch.js'; +import { prepareRetriableTools } from './retriable.js'; import { makeWhen } from './when.js'; /** @@ -33,23 +34,10 @@ export const prepareVowTools = (zone, powers = {}) => { const watchUtils = makeWatchUtils(); const asVow = makeAsVow(makeVowKit); - /** - * TODO FIXME make this real - * Create a function that retries the given function if the underlying - * functions rejects due to upgrade disconnection. - * - * @template {(...args: any[]) => Promise} F - * @param {Zone} fnZone - the zone for the named function - * @param {string} name - * @param {F} fn - * @returns {F extends (...args: infer Args) => Promise ? (...args: Args) => Vow : never} - */ - const retriable = - (fnZone, name, fn) => - // @ts-expect-error cast - (...args) => { - return watch(fn(...args)); - }; + const { retriable } = prepareRetriableTools(zone, { + makeVowKit, + isRetryableReason, + }); /** * Vow-tolerant implementation of Promise.all. diff --git a/packages/vow/vat.js b/packages/vow/vat.js index cc1170eaa7e4..a0c24a90510d 100644 --- a/packages/vow/vat.js +++ b/packages/vow/vat.js @@ -5,7 +5,10 @@ /* global globalThis */ // @ts-check -import { isUpgradeDisconnection } from '@agoric/internal/src/upgrade-api.js'; +import { + isUpgradeDisconnection, + isAbandonedError, +} from '@agoric/internal/src/upgrade-api.js'; import { makeHeapZone } from '@agoric/base-zone/heap.js'; import { makeE, prepareVowTools as rawPrepareVowTools } from './src/index.js'; @@ -13,11 +16,16 @@ import { makeE, prepareVowTools as rawPrepareVowTools } from './src/index.js'; const isRetryableReason = (reason, priorRetryValue) => { if ( isUpgradeDisconnection(reason) && - (!priorRetryValue || + (!isUpgradeDisconnection(priorRetryValue) || reason.incarnationNumber > priorRetryValue.incarnationNumber) ) { return reason; } + // For abandoned errors there is no way to differentiate errors from + // consecutive upgrades + if (isAbandonedError(reason) && !isAbandonedError(priorRetryValue)) { + return reason; + } return undefined; };