Skip to content

Commit

Permalink
feat: auctioneer detects failing priceAuthority; requests new one (#8691
Browse files Browse the repository at this point in the history 
)

* feat: auctioneer/vaults detect failing priceAuthority & request new

If the quoteNotifier is broken, set updatingOracleQuote to null. If
updatingOracleQuote is null when attempting to capture the oracle
price, restart observeQuoteNotifier

Vaults detects failing priceAuthority and requests new one (#8696)

* feat: make vaultManager robust against failing priceAuthority

* chore: fix types, drop expect-error

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
@Chris-Hibbert @mergify
Chris-Hibbert and mergify[bot] authored Jan 9, 2024
1 parent 73364fa commit 8604b01
Show file tree
Hide file tree
Showing 12 changed files with 904 additions and 179 deletions.
2 changes: 2 additions & 0 deletions packages/builders/scripts/vats/restart-vats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ export const defaultProposalBuilder = async () => {
'feeDistributor',
// skip so vaultManager can have prices upon restart; these have been tested as restartable
'scaledPriceAuthority-ATOM',
// If this is killed, and the above is left alive, quoteNotifier throws
'ATOM-USD_price_feed',
];

return harden({
Expand Down
92 changes: 53 additions & 39 deletions packages/inter-protocol/src/auction/auctionBook.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import '@agoric/governance/exported.js';
import '@agoric/zoe/exported.js';
import '@agoric/zoe/src/contracts/exported.js';

import { AmountMath } from '@agoric/ertp';
import { AmountMath, RatioShape } from '@agoric/ertp';
import { mustMatch } from '@agoric/store';
import { M, prepareExoClassKit } from '@agoric/vat-data';

Expand Down Expand Up @@ -124,7 +124,7 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
bidHoldingSeat: M.any(),
bidAmountShape: M.any(),
priceAuthority: M.any(),
updatingOracleQuote: M.any(),
updatingOracleQuote: M.or(RatioShape, M.null()),
bookDataKit: M.any(),
priceBook: M.any(),
scaledBidBook: M.any(),
Expand All @@ -147,11 +147,6 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
*/
(bidBrand, collateralBrand, pAuthority, node) => {
assertAllDefined({ bidBrand, collateralBrand, pAuthority });
const zeroBid = makeEmpty(bidBrand);
const zeroRatio = makeRatioFromAmounts(
zeroBid,
AmountMath.make(collateralBrand, 1n),
);

// these don't have to be durable, since we're currently assuming that upgrade
// from a quiescent state is sufficient. When the auction is quiescent, there
Expand Down Expand Up @@ -188,7 +183,7 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
bidAmountShape,

priceAuthority: pAuthority,
updatingOracleQuote: zeroRatio,
updatingOracleQuote: /** @type {Ratio | null} */ (null),

bookDataKit,

Expand Down Expand Up @@ -468,6 +463,48 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
});
return state.bookDataKit.recorder.write(bookData);
},
observeQuoteNotifier() {
const { state, facets } = this;
const { collateralBrand, bidBrand, priceAuthority } = state;

trace('observing');

void E.when(
E(collateralBrand).getDisplayInfo(),
({ decimalPlaces = DEFAULT_DECIMALS }) => {
const quoteNotifier = E(priceAuthority).makeQuoteNotifier(
AmountMath.make(collateralBrand, 10n ** BigInt(decimalPlaces)),
bidBrand,
);
void observeNotifier(quoteNotifier, {
updateState: quote => {
trace(
`BOOK notifier ${priceFrom(quote).numerator.value}/${
priceFrom(quote).denominator.value
}`,
);
state.updatingOracleQuote = priceFrom(quote);
},
fail: reason => {
trace(
`Failure from quoteNotifier (${reason}) setting to null`,
);
// lack of quote will trigger restart
state.updatingOracleQuote = null;
},
finish: done => {
trace(
`quoteNotifier invoked finish(${done}). setting quote to null`,
);
// lack of quote will trigger restart
state.updatingOracleQuote = null;
},
});
},
);

void facets.helper.publishBookData();
},
},
self: {
/**
Expand Down Expand Up @@ -630,6 +667,12 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
const { facets, state } = this;

trace(`capturing oracle price `, state.updatingOracleQuote);
if (!state.updatingOracleQuote) {
// if the price has feed has died, try restarting it.
facets.helper.observeQuoteNotifier();
return;
}

state.capturedPriceForRound = state.updatingOracleQuote;
void facets.helper.publishBookData();
},
Expand Down Expand Up @@ -729,37 +772,8 @@ export const prepareAuctionBook = (baggage, zcf, makeRecorderKit) => {
finish: ({ state, facets }) => {
const { collateralBrand, bidBrand, priceAuthority } = state;
assertAllDefined({ collateralBrand, bidBrand, priceAuthority });
void E.when(
E(collateralBrand).getDisplayInfo(),
({ decimalPlaces = DEFAULT_DECIMALS }) => {
// TODO(#6946) use this to keep a current price that can be published in state.
const quoteNotifier = E(priceAuthority).makeQuoteNotifier(
AmountMath.make(collateralBrand, 10n ** BigInt(decimalPlaces)),
bidBrand,
);
void observeNotifier(quoteNotifier, {
updateState: quote => {
trace(
`BOOK notifier ${priceFrom(quote).numerator.value}/${
priceFrom(quote).denominator.value
}`,
);
state.updatingOracleQuote = priceFrom(quote);
},
fail: reason => {
throw Error(
`auction observer of ${collateralBrand} failed: ${reason}`,
);
},
finish: done => {
throw Error(
`auction observer for ${collateralBrand} died: ${done}`,
);
},
});
},
);
void facets.helper.publishBookData();

facets.helper.observeQuoteNotifier();
},
stateShape: AuctionBookStateShape,
},
Expand Down
123 changes: 98 additions & 25 deletions packages/inter-protocol/src/vaultFactory/vaultManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,38 @@ const { details: X, Fail, quote: q } = assert;

const trace = makeTracer('VM');

/**
* Watch a notifier that isn't expected to fail or finish unless the vat hosting
* the notifier is upgraded. This watcher supports that by providing a
* straightforward way to get a replacement if the notifier breaks.
*
* @template T notifier topic
* @template {any[]} [A=unknown[]] arbitrary arguments
* @param {ERef<LatestTopic<T>>} notifierP
* @param {import('@agoric/swingset-liveslots').PromiseWatcher<T, A>} watcher
* @param {A} args
*/
export const watchQuoteNotifier = async (notifierP, watcher, ...args) => {
await undefined;

let updateCount;
for (;;) {
let value;
try {
({ value, updateCount } = await E(notifierP).getUpdateSince(updateCount));
watcher.onFulfilled && watcher.onFulfilled(value, ...args);
} catch (e) {
watcher.onRejected && watcher.onRejected(e, ...args);
break;
}
if (updateCount === undefined) {
watcher.onRejected &&
watcher.onRejected(Error('stream finished'), ...args);
break;
}
}
};

/** @typedef {import('./storeUtils.js').NormalizedDebt} NormalizedDebt */
/** @typedef {import('@agoric/time').RelativeTime} RelativeTime */

Expand Down Expand Up @@ -170,7 +202,7 @@ const trace = makeTracer('VM');
* @type {(brand: Brand) => {
* prioritizedVaults: ReturnType<typeof makePrioritizedVaults>;
* storedQuotesNotifier: import('@agoric/notifier').StoredNotifier<PriceQuote>;
* storedCollateralQuote: PriceQuote;
* storedCollateralQuote: PriceQuote | null;
* }}
*/
// any b/c will be filled after start()
Expand Down Expand Up @@ -355,13 +387,7 @@ export const prepareVaultManagerKit = (
start() {
const { state, facets } = this;
trace(state.collateralBrand, 'helper.start()', state.vaultCounter);
const {
collateralBrand,
collateralUnit,
debtBrand,
storageNode,
unsettledVaults,
} = state;
const { collateralBrand, unsettledVaults } = state;

const ephemera = collateralEphemera(collateralBrand);
ephemera.prioritizedVaults = makePrioritizedVaults(unsettledVaults);
Expand Down Expand Up @@ -394,7 +420,17 @@ export const prepareVaultManagerKit = (
},
});

trace('helper.start() making quoteNotifier from', priceAuthority);
void facets.helper.observeQuoteNotifier();

trace('helper.start() done');
},
observeQuoteNotifier() {
const { state } = this;

const { collateralBrand, collateralUnit, debtBrand, storageNode } =
state;
const ephemera = collateralEphemera(collateralBrand);

const quoteNotifier = E(priceAuthority).makeQuoteNotifier(
collateralUnit,
debtBrand,
Expand All @@ -404,20 +440,29 @@ export const prepareVaultManagerKit = (
E(storageNode).makeChildNode('quotes'),
marshaller,
);
trace('helper.start() awaiting observe storedQuotesNotifier');
trace(
'helper.start() awaiting observe storedQuotesNotifier',
collateralBrand,
);
// NB: upon restart, there may not be a price for a while. If manager
// operations are permitted, ones the depend on price information will
// throw. See https://github.com/Agoric/agoric-sdk/issues/4317
void observeNotifier(quoteNotifier, {
updateState(value) {
trace('storing new quote', value.quoteAmount.value);
// operations are permitted, ones that depend on price information
// will throw. See https://github.com/Agoric/agoric-sdk/issues/4317
const quoteWatcher = harden({
onFulfilled(value) {
ephemera.storedCollateralQuote = value;
},
fail(reason) {
console.error('quoteNotifier failed to iterate', reason);
onRejected() {
// NOTE: drastic action, if the quoteNotifier fails, we don't know
// the value of the asset, nor do we know how long we'll be in
// ignorance. Best choice is to disable actions that require
// prices and restart when we have a new price. If we restart the
// notifier immediately, we'll trigger an infinite loop, so try
// to restart each time we get a request.

ephemera.storedCollateralQuote = null;
},
});
trace('helper.start() done');
void watchQuoteNotifier(quoteNotifier, quoteWatcher);
},
/** @param {Timestamp} updateTime */
async chargeAllVaults(updateTime) {
Expand Down Expand Up @@ -785,10 +830,15 @@ export const prepareVaultManagerKit = (
* @param {Amount<'nat'>} collateralAmount
*/
maxDebtFor(collateralAmount) {
const { collateralBrand } = this.state;
const { state, facets } = this;
const { collateralBrand } = state;
const { storedCollateralQuote } = collateralEphemera(collateralBrand);
if (!storedCollateralQuote)
throw Fail`maxDebtFor called before a collateral quote was available`;
if (!storedCollateralQuote) {
facets.helper.observeQuoteNotifier();

// it might take an arbitrary amount of time to get a new quote
throw Fail`maxDebtFor called before a collateral quote was available for ${collateralBrand}`;
}
// use the lower price to prevent vault adjustments that put them imminently underwater
const collateralPrice = minimumPrice(
storedCollateralQuote,
Expand Down Expand Up @@ -1025,11 +1075,17 @@ export const prepareVaultManagerKit = (
},

getCollateralQuote() {
const { state, facets } = this;
const { storedCollateralQuote } = collateralEphemera(
this.state.collateralBrand,
state.collateralBrand,
);
if (!storedCollateralQuote)
if (!storedCollateralQuote) {
facets.helper.observeQuoteNotifier();

// it might take an arbitrary amount of time to get a new quote
throw Fail`getCollateralQuote called before a collateral quote was available`;
}

return storedCollateralQuote;
},

Expand All @@ -1042,8 +1098,13 @@ export const prepareVaultManagerKit = (
const { storedCollateralQuote } = collateralEphemera(
state.collateralBrand,
);
if (!storedCollateralQuote)
if (!storedCollateralQuote) {
facets.helper.observeQuoteNotifier();

// it might take an arbitrary amount of time to get a new quote
throw Fail`lockOraclePrices called before a collateral quote was available for ${state.collateralBrand}`;
}

trace(
`lockOraclePrices`,
getAmountIn(storedCollateralQuote),
Expand Down Expand Up @@ -1078,6 +1139,15 @@ export const prepareVaultManagerKit = (
return;
}

const { storedCollateralQuote: collateralQuoteBefore } =
collateralEphemera(this.state.collateralBrand);
if (!collateralQuoteBefore) {
console.error(
'Skipping liquidation because collateralQuote is missing',
);
return;
}

const { prioritizedVaults } = collateralEphemera(collateralBrand);
prioritizedVaults || Fail`prioritizedVaults missing from ephemera`;

Expand Down Expand Up @@ -1141,7 +1211,10 @@ export const prepareVaultManagerKit = (
const { plan, vaultsInPlan } = helper.planProceedsDistribution(
proceeds,
totalDebt,
storedCollateralQuote,
// If a quote was available at the start of liquidation, but is no
// longer, using the earlier price is better than failing to
// distribute proceeds
storedCollateralQuote || collateralQuoteBefore,
vaultData,
totalCollateral,
);
Expand Down
11 changes: 7 additions & 4 deletions packages/inter-protocol/test/auction/test-auctionBook.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,19 @@ const assembleAuctionBook = async basics => {

test('states', async t => {
const basics = await setupBasics();
const { moolaKit, simoleanKit } = basics;
const { book } = await assembleAuctionBook(basics);
const { moolaKit, moola, simoleanKit, simoleans } = basics;
const { pa, book } = await assembleAuctionBook(basics);

pa.setPrice(makeRatioFromAmounts(moola(9n), simoleans(10n)));
await eventLoopIteration();

book.captureOraclePriceForRound();
book.setStartingRate(makeRatio(90n, moolaKit.brand, 100n));
t.deepEqual(
book.getCurrentPrice(),
makeRatioFromAmounts(
AmountMath.makeEmpty(moolaKit.brand),
AmountMath.make(simoleanKit.brand, 100n),
AmountMath.make(moolaKit.brand, 81_000_000_000n),
AmountMath.make(simoleanKit.brand, 100_000_000_000n),
),
);
});
Expand Down
Loading

0 comments on commit 8604b01

Please sign in to comment.