Operator attestation policy (#10721)

TODO in code

## Description
Update `TransactionFeed` to publish when a majority of operators to attest (previous required unanimous). Conflicts log a loud error and don't publish.

Also updates the contract test to have 3 operators (per Mainnet policy) and rely on unit tests for the other values.

### Security Considerations

Adds assertions to the deploy-config to help ensure that Mainnet has three unique operators, per security policy.

### Scaling Considerations
Each new attestation iterates over three stores for matching values.

### Documentation Considerations
Nothing end-user. OCW is documented elsewhere.

### Testing Considerations
CI

### Upgrade Considerations
not yet deployed.

contract is upgradable but changing the settlementAccountAddress will require creating a new TransactionFeed (as is the case for the Settler object.

packages/fast-usdc/src/exos/operator-kit.js

@@ -12,7 +12,7 @@ const trace = makeTracer('TxOperator');
 
 /**
  * @typedef {object} OperatorPowers
- * @property {(evidence: CctpTxEvidence, operatorKit: OperatorKit) => void} submitEvidence
+ * @property {(evidence: CctpTxEvidence, operatorId: string) => void} attest
  */
 
 /**
@@ -35,7 +35,7 @@ const OperatorKitI = {
   }),
 
   operator: M.interface('Operator', {
-    submitEvidence: M.call(CctpTxEvidenceShape).returns(M.promise()),
+    submitEvidence: M.call(CctpTxEvidenceShape).returns(),
     getStatus: M.call().returns(M.record()),
   }),
 };
@@ -87,7 +87,7 @@ export const prepareOperatorKit = (zone, staticPowers) =>
           const { operator } = this.facets;
           // TODO(bootstrap integration): cause this call to throw and confirm that it
           // shows up in the the smart-wallet UpdateRecord `error` property
-          await operator.submitEvidence(evidence);
+          operator.submitEvidence(evidence);
           return staticPowers.makeInertInvitation(
             'evidence was pushed in the invitation maker call',
           );
@@ -98,12 +98,12 @@ export const prepareOperatorKit = (zone, staticPowers) =>
          * submit evidence from this operator
          *
          * @param {CctpTxEvidence} evidence
+         * @returns {void}
          */
-        async submitEvidence(evidence) {
+        submitEvidence(evidence) {
           const { state } = this;
           !state.disabled || Fail`submitEvidence for disabled operator`;
-          const result = state.powers.submitEvidence(evidence, this.facets);
-          return result;
+          state.powers.attest(evidence, state.operatorId);
         },
         /** @returns {OperatorStatus} */
         getStatus() {

packages/fast-usdc/src/exos/transaction-feed.js

@@ -1,6 +1,7 @@
 import { makeTracer } from '@agoric/internal';
 import { prepareDurablePublishKit } from '@agoric/notifier';
-import { M } from '@endo/patterns';
+import { keyEQ, M } from '@endo/patterns';
+import { Fail } from '@endo/errors';
 import { CctpTxEvidenceShape } from '../type-guards.js';
 import { defineInertInvitation } from '../utils/zoe.js';
 import { prepareOperatorKit } from './operator-kit.js';
@@ -18,7 +19,7 @@ export const INVITATION_MAKERS_DESC = 'oracle operator invitation';
 
 const TransactionFeedKitI = harden({
   operatorPowers: M.interface('Transaction Feed Admin', {
-    submitEvidence: M.call(CctpTxEvidenceShape, M.any()).returns(),
+    attest: M.call(CctpTxEvidenceShape, M.string()).returns(),
   }),
   creator: M.interface('Transaction Feed Creator', {
     // TODO narrow the return shape to OperatorKit
@@ -118,23 +119,16 @@ export const prepareTransactionFeedKit = (zone, zcf) => {
         /**
          * Add evidence from an operator.
          *
+         * NB: the operatorKit is responsible for
+         *
          * @param {CctpTxEvidence} evidence
-         * @param {OperatorKit} operatorKit
+         * @param {string} operatorId
          */
-        submitEvidence(evidence, operatorKit) {
-          const { pending } = this.state;
-          trace(
-            'submitEvidence',
-            operatorKit.operator.getStatus().operatorId,
-            evidence,
-          );
-          const { operatorId } = operatorKit.operator.getStatus();
-
-          // TODO should this verify that the operator is one made by this exo?
-          // This doesn't work...
-          // operatorKit === operators.get(operatorId) ||
-          //   Fail`operatorKit mismatch`;
+        attest(evidence, operatorId) {
+          const { operators, pending } = this.state;
+          trace('submitEvidence', operatorId, evidence);
 
+          // TODO https://github.com/Agoric/agoric-sdk/pull/10720
           // TODO validate that it's a valid for Fast USDC before accepting
           // E.g. that the `recipientAddress` is the FU settlement account and that
           // the EUD is a chain supported by FU.
@@ -154,18 +148,46 @@ export const prepareTransactionFeedKit = (zone, zcf) => {
           const found = [...pending.values()].filter(store =>
             store.has(txHash),
           );
-          // TODO determine the real policy for checking agreement
-          if (found.length < pending.getSize()) {
-            // not all have seen it
+          const minAttestations = Math.ceil(operators.getSize() / 2);
+          trace(
+            'transaction',
+            txHash,
+            'has',
+            found.length,
+            'of',
+            minAttestations,
+            'necessary attestations',
+          );
+          if (found.length < minAttestations) {
             return;
           }
 
-          // TODO verify that all found deep equal
+          let lastEvidence;
+          for (const store of found) {
+            const next = store.get(txHash);
+            if (lastEvidence) {
+              if (keyEQ(lastEvidence, next)) {
+                lastEvidence = next;
+              } else {
+                trace(
+                  '🚨 conflicting evidence for',
+                  txHash,
+                  ':',
+                  lastEvidence,
+                  '!=',
+                  next,
+                );
+                Fail`conflicting evidence for ${txHash}`;
+              }
+            }
+            lastEvidence = next;
+          }
 
-          // all agree, so remove from pending and publish
-          for (const pendingStore of pending.values()) {
-            pendingStore.delete(txHash);
+          // sufficient agreement, so remove from pending and publish
+          for (const store of found) {
+            store.delete(txHash);
           }
+          trace('publishing evidence', evidence);
           publisher.publish(evidence);
         },
       },

packages/fast-usdc/src/utils/deploy-config.js

@@ -156,3 +156,11 @@ export const configurations = {
   },
 };
 harden(configurations);
+
+// Constraints on the configurations
+const MAINNET_EXPECTED_ORACLES = 3;
+assert(
+  new Set(Object.values(configurations.MAINNET.oracles)).size ===
+    MAINNET_EXPECTED_ORACLES,
+  `Mainnet must have exactly ${MAINNET_EXPECTED_ORACLES} oracles`,
+);

packages/fast-usdc/test/exos/transaction-feed.test.ts

@@ -47,50 +47,87 @@ test('happy aggregation', async t => {
   const evidenceSubscriber = feedKit.public.getEvidenceSubscriber();
 
   const { op1, op2, op3 } = await makeOperators(feedKit);
-  const evidence = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
-  const results = await Promise.all([
-    op1.operator.submitEvidence(evidence),
-    op2.operator.submitEvidence(evidence),
-    op3.operator.submitEvidence(evidence),
-  ]);
-  t.deepEqual(results, [undefined, undefined, undefined]);
 
+  const e1 = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
+  op1.operator.submitEvidence(e1);
+  op2.operator.submitEvidence(e1);
+
+  // Publishes with 2 of 3
   const accepted = await evidenceSubscriber.getUpdateSince(0);
   t.deepEqual(accepted, {
-    value: evidence,
+    value: e1,
     updateCount: 1n,
   });
 
-  // verify that it doesn't publish until three match
-  await Promise.all([
-    // once it publishes, it doesn't remember that it already saw these
-    op1.operator.submitEvidence(evidence),
-    op2.operator.submitEvidence(evidence),
-    // but this time the third is different
-    op3.operator.submitEvidence(MockCctpTxEvidences.AGORIC_PLUS_DYDX()),
-  ]);
+  // Now third operator catches up with same evidence already published
+  op3.operator.submitEvidence(e1);
   t.like(await evidenceSubscriber.getUpdateSince(0), {
-    // Update count is still 1
+    // The confirming evidence doesn't change anything
     updateCount: 1n,
   });
-  await op3.operator.submitEvidence(evidence);
+
+  const e2 = MockCctpTxEvidences.AGORIC_PLUS_DYDX();
+  assert(e1.txHash !== e2.txHash);
+  op1.operator.submitEvidence(e2);
+  t.like(await evidenceSubscriber.getUpdateSince(0), {
+    // op1 attestation insufficient
+    updateCount: 1n,
+  });
+});
+
+test('disagreement', async t => {
+  const feedKit = makeFeedKit();
+  const { op1, op2 } = await makeOperators(feedKit);
+  const e1 = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
+  const e1bad = { ...e1, tx: { ...e1.tx, amount: 999_999_999n } };
+  assert(e1.txHash === e1bad.txHash);
+  op1.operator.submitEvidence(e1);
+
+  t.throws(() => op2.operator.submitEvidence(e1bad), {
+    message:
+      'conflicting evidence for "0xc81bc6105b60a234c7c50ac17816ebcd5561d366df8bf3be59ff387552761702"',
+  });
+});
+
+test('disagreement after publishing', async t => {
+  const feedKit = makeFeedKit();
+  const evidenceSubscriber = feedKit.public.getEvidenceSubscriber();
+  const { op1, op2, op3 } = await makeOperators(feedKit);
+  const e1 = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
+  const e1bad = { ...e1, tx: { ...e1.tx, amount: 999_999_999n } };
+  assert(e1.txHash === e1bad.txHash);
+  op1.operator.submitEvidence(e1);
+  op2.operator.submitEvidence(e1);
+
+  t.like(await evidenceSubscriber.getUpdateSince(0), {
+    updateCount: 1n,
+  });
+
+  // it's simply ignored
+  t.notThrows(() => op3.operator.submitEvidence(e1bad));
+  t.like(await evidenceSubscriber.getUpdateSince(0), {
+    updateCount: 1n,
+  });
+
+  // now another op repeats the bad evidence, so it's published to the stream.
+  // It's the responsibility of the Advancer to fail because it has already processed that tx hash.
+  op1.operator.submitEvidence(e1bad);
   t.like(await evidenceSubscriber.getUpdateSince(0), {
     updateCount: 2n,
   });
 });
 
-// TODO: find a way to get this working
-test.skip('forged source', async t => {
+test('disabled operator', async t => {
   const feedKit = makeFeedKit();
   const { op1 } = await makeOperators(feedKit);
   const evidence = MockCctpTxEvidences.AGORIC_PLUS_OSMO();
 
-  // op1 is different than the facets object the evidence must come from
-  t.throws(() =>
-    feedKit.operatorPowers.submitEvidence(
-      evidence,
-      // @ts-expect-error XXX Types of property '[GET_INTERFACE_GUARD]' are incompatible.
-      op1,
-    ),
-  );
+  // works before disabling
+  op1.operator.submitEvidence(evidence);
+
+  op1.admin.disable();
+
+  t.throws(() => op1.operator.submitEvidence(evidence), {
+    message: 'submitEvidence for disabled operator',
+  });
 });

packages/fast-usdc/test/fast-usdc.contract.test.ts

@@ -1,8 +1,11 @@
 import { test as anyTest } from '@agoric/zoe/tools/prepare-test-env-ava.js';
 import type { ExecutionContext, TestFn } from 'ava';
 
+import {
+  decodeAddressHook,
+  encodeAddressHook,
+} from '@agoric/cosmic-proto/address-hooks.js';
 import { AmountMath } from '@agoric/ertp/src/amountMath.js';
-import { deeplyFulfilledObject } from '@agoric/internal';
 import {
   eventLoopIteration,
   inspectMapStore,
@@ -24,13 +27,9 @@ import {
 import type { Instance } from '@agoric/zoe/src/zoeService/utils.js';
 import { setUpZoeForTest } from '@agoric/zoe/tools/setup-zoe.js';
 import { E } from '@endo/far';
-import { matches, objectMap } from '@endo/patterns';
+import { matches } from '@endo/patterns';
 import { makePromiseKit } from '@endo/promise-kit';
 import path from 'path';
-import {
-  decodeAddressHook,
-  encodeAddressHook,
-} from '@agoric/cosmic-proto/address-hooks.js';
 import type { OperatorKit } from '../src/exos/operator-kit.js';
 import type { FastUsdcSF } from '../src/fast-usdc.contract.js';
 import { PoolMetricsShape } from '../src/type-guards.js';
@@ -56,10 +55,12 @@ const getInvitationProperties = async (
   return amount.value[0];
 };
 
+// Spec for Mainnet. Other values are covered in unit tests of TransactionFeed.
+const operatorQty = 3;
+
 type CommonSetup = Awaited<ReturnType<typeof commonSetup>>;
 const startContract = async (
   common: Pick<CommonSetup, 'brands' | 'commonPrivateArgs' | 'utils'>,
-  operatorQty = 1,
 ) => {
   const {
     brands: { usdc },
@@ -104,7 +105,7 @@ const makeTestContext = async (t: ExecutionContext) => {
   const common = await commonSetup(t);
   await E(common.mocks.ibcBridge).setAddressPrefix('noble');
 
-  const startKit = await startContract(common, 2);
+  const startKit = await startContract(common);
 
   const { transferBridge } = common.mocks;
   const evm = makeEVM();
@@ -227,12 +228,17 @@ const makeOracleOperator = async (
   ]);
   const { invitationMakers } = operatorKit;
 
+  let active = true;
+
   return harden({
     watch: () => {
       void observeIteration(subscribeEach(txSubscriber), {
-        updateState: tx =>
+        updateState: tx => {
+          if (!active) {
+            return;
+          }
           // KLUDGE: tx wouldn't include aux. OCW looks it up
-          E.when(
+          return E.when(
             E(invitationMakers).SubmitEvidence(tx),
             inv =>
               E.when(E(E(zoe).offer(inv)).getOfferResult(), res => {
@@ -242,13 +248,17 @@ const makeOracleOperator = async (
             reason => {
               failures.push(reason.message);
             },
-          ),
+          );
+        },
       });
     },
     getDone: () => done,
     getFailures: () => harden([...failures]),
     // operator only gets .invitationMakers
     getKit: () => operatorKit,
+    setActive: flag => {
+      active = flag;
+    },
   });
 };
 
@@ -760,10 +770,12 @@ test.serial('Settlement for unknown transaction (operator down)', async t => {
   } = t.context;
   const operators = await sync.ocw.promise;
 
+  // Simulate 2 of 3 operators being unavailable
+  operators[0].setActive(false);
+  operators[1].setActive(false);
+
   const opDown = makeCustomer('Otto', cctp, txPub.publisher, feeConfig);
 
-  // what removeOperator will do
-  await E(E.get(E(operators[1]).getKit()).admin).disable();
   const bridgePos = snapshot();
   const sent = await opDown.sendFast(t, 20_000_000n, 'osmo12345');
   await mint(sent);
@@ -788,9 +800,6 @@ test.serial('Settlement for unknown transaction (operator down)', async t => {
   t.deepEqual(bridgeTraffic.local, [], 'no IBC transfers');
 
   await transmitTransferAck();
-  t.deepEqual(await E(operators[1]).getFailures(), [
-    'submitEvidence for disabled operator',
-  ]);
 });
 
 test.todo(