feat: add APIs for tracking/debugging undesired object retention (aka…

… "leaks")

Closes #7318

Adds two and a half API features for to aid debugging and testing for leaks of
objects managed by LiveSlots:

* The `testHooks` object returned by liveslots now contains a function
`getRetentionStats` that will return a data object containing the counts of the
various non-singular objects that LiveSlots tracks internally.  (LiveSlots keeps
track of all these things in JavaScript Maps and Sets, so counting these objects
is simply a matter of returning the sizes of these various collections.  One
consequence of this is that `getRetentionStats` call will execute quickly
regardless of how much stuff LiveSlots is holding onto.)

* The `testHooks` object now also contains references to all these various Maps
and Sets directly.  Note that this is powerful and dangerous, but it's confined
to the `testHooks` object which is only exposed during testing.

* The data record that `getRetentionStats` produces is also returned as the
result of every `bringOutYourDead` operation.  From there it can be examined in
tests, but, more signficantly, it will be output as part of the delivery status
array that is written to the slog, so that a graph of object retention stats
over time can be produced from a running chain or a long running performance
test executing with a live swingset.

Collections passed in `testHooks` and counted by `getRetentionStats`:

Collection | Type | What
------------|:-----|-----
exportedRemotables | B | exported objects, to pin remotables; dropped on export drop
importedDevices | B | imported devices, to pin devices; grows monotonically
remotableRefCounts | B | objects ref'd from off vat (kernel or storage)
kernelRecognizableRemotables | C | exports recognizable by kernel; tracks vrefs known to kernel, drop on export retire, retire when dead or kernel retire
exportedVPIDs | C | promises exported; drop on resolve (vat is decider)
importedVPIDs | C | promises imported; drop on resolve (kernel is decider)
vrefRecognizers | C | vrefs used as keys in VirtualObjectAwareWeakMap/Set
definedDurableKinds | D | durable kinds that exist
kindInfoTable | D | info about kinds (durable + non-durable)
nextInstanceIDs | D | next id to allocate for kind once allocation has started
possiblyDeadSet | E | baseRefs to investigate for GC; leared on BOYD
possiblyRetiredSet | E | vrefs to investigate for retirement; cleared on BOYD
slotToVal | F | live objects with vrefs
valToSlot | E | live objects with vrefs

Types:
A - Keyed by strings referring to virtual/durable store collections; cardinality is the total number of collections
B - Keyed by direct references to explicitly in-memory objects; cardinality limited by RAM capacity
C - Keyed by strings referring to explicitly in-memory objects;  cardinality limited by RAM capacity
D - Keyed by kindID strings; cardinality is number of defined knids (sometimes only durable kinds)
E - Transient collections; note that these have no counters since weak collections are not countable
F - Keyed by vref strings referring to any kind of object currently addressable in memory

packages/SwingSet/src/kernel/vat-loader/manager-subprocess-xsnap.js

@@ -218,7 +218,7 @@ export function makeXsSubprocessFactory({
       // @ts-ignore I don't know how to appease tsc
       const deliverResult = harden([
         result.reply[0], // 'ok' or 'error'
-        result.reply[1] || null, // problem or null
+        result.reply[1] || null, // results or problem or null
         result.meterUsage || null, // meter usage statistics or null
       ]);
       insistVatDeliveryResult(deliverResult);

packages/SwingSet/src/lib/message.js

@@ -93,7 +93,6 @@ export function insistVatDeliveryResult(vdr) {
   const [type, problem, _usage] = vdr;
   switch (type) {
     case 'ok': {
-      assert.equal(problem, null);
       break;
     }
     case 'error': {

packages/SwingSet/src/supervisors/supervisor-helper.js

@@ -36,7 +36,7 @@ function makeSupervisorDispatch(dispatch) {
     return Promise.resolve(delivery)
       .then(dispatch)
       .then(
-        () => harden(['ok', null, null]),
+        res => harden(['ok', res, null]),
         err => {
           // TODO react more thoughtfully, maybe terminate the vat
           console.warn(`error during vat dispatch() of ${delivery}`, err);

packages/swingset-liveslots/src/collectionManager.js

@@ -940,7 +940,11 @@ export function makeCollectionManager(
     return collectionToWeakSetStore(reanimateCollection(vobjID));
   }
 
-  const testHooks = { obtainStoreKindID, storeSizeInternal, makeCollection };
+  const testHooks = {
+    obtainStoreKindID,
+    storeSizeInternal,
+    makeCollection,
+  };
 
   /**
    * @param {Pattern} baseKeyShape
@@ -1008,6 +1012,10 @@ export function makeCollectionManager(
   const makeScalarBigWeakSetStore = (label = 'weakSet', options = {}) =>
     makeBigWeakSetStore(label, narrowKeyShapeOption(M.scalar(), options));
 
+  function getRetentionStats() {
+    return {};
+  }
+
   return harden({
     initializeStoreKindInfo,
     deleteAllVirtualCollections,
@@ -1016,6 +1024,7 @@ export function makeCollectionManager(
     makeScalarBigSetStore,
     makeScalarBigWeakSetStore,
     provideBaggage,
+    getRetentionStats,
     testHooks,
   });
 }

packages/swingset-liveslots/src/liveslots.js

@@ -1340,12 +1340,39 @@ function build(
     WeakSet: vom.VirtualObjectAwareWeakSet,
   });
 
+  function getRetentionStats() {
+    return {
+      ...collectionManager.getRetentionStats(),
+      ...vrm.getRetentionStats(),
+      ...vom.getRetentionStats(),
+      exportedRemotables: exportedRemotables.size,
+      importedDevices: importedDevices.size,
+      kernelRecognizableRemotables: kernelRecognizableRemotables.size,
+      exportedVPIDs: exportedVPIDs.size,
+      importedVPIDs: importedVPIDs.size,
+      possiblyDeadSet: possiblyDeadSet.size,
+      possiblyRetiredSet: possiblyRetiredSet.size,
+      slotToVal: slotToVal.size,
+    };
+  }
+
   const testHooks = harden({
     ...vom.testHooks,
     ...vrm.testHooks,
     ...collectionManager.testHooks,
     setSyscallCapdataLimits,
     vatGlobals,
+
+    getRetentionStats,
+    exportedRemotables,
+    importedDevices,
+    kernelRecognizableRemotables,
+    exportedVPIDs,
+    importedVPIDs,
+    possiblyDeadSet,
+    possiblyRetiredSet,
+    slotToVal,
+    valToSlot,
   });
 
   function setVatOption(option, _value) {
@@ -1492,6 +1519,8 @@ function build(
     await scanForDeadObjects();
     // now flush all the vatstore changes (deletions) we made
     vom.flushStateCache();
+    // XXX TODO: make this conditional on a config setting
+    return getRetentionStats();
   }
 
   /**
@@ -1586,7 +1615,6 @@ function build(
   return harden({
     dispatch,
     m,
-    possiblyDeadSet,
     testHooks,
   });
 }

packages/swingset-liveslots/src/types.js

@@ -46,7 +46,7 @@
  *          } VatDeliveryObject
  *
  * @typedef { { compute: number } } MeterConsumption
- * @typedef { [tag: 'ok', message: null, usage: MeterConsumption | null] |
+ * @typedef { [tag: 'ok', results: any, usage: MeterConsumption | null] |
  *            [tag: 'error', message: string, usage: MeterConsumption | null] } VatDeliveryResult
  *
  *

packages/swingset-liveslots/src/virtualObjectManager.js

@@ -1078,6 +1078,9 @@ export function makeVirtualObjectManager(
 
   const testHooks = {
     countWeakKeysForCollection,
+
+    definedDurableKinds,
+    nextInstanceIDs,
   };
 
   const flushStateCache = () => {
@@ -1086,6 +1089,13 @@ export function makeVirtualObjectManager(
     }
   };
 
+  function getRetentionStats() {
+    return {
+      definedDurableKinds: definedDurableKinds.size,
+      nextInstanceIDs: nextInstanceIDs.size,
+    };
+  }
+
   return harden({
     initializeKindHandleKind,
     defineKind,
@@ -1097,6 +1107,7 @@ export function makeVirtualObjectManager(
     VirtualObjectAwareWeakMap,
     VirtualObjectAwareWeakSet,
     flushStateCache,
+    getRetentionStats,
     testHooks,
     canBeDurable,
   });

packages/swingset-liveslots/src/virtualReferences.js

@@ -651,8 +651,20 @@ export function makeVirtualReferenceManager(
   const testHooks = {
     getReachableRefCount,
     countCollectionsForWeakKey,
+
+    remotableRefCounts,
+    vrefRecognizers,
+    kindInfoTable,
   };
 
+  function getRetentionStats() {
+    return {
+      remotableRefCounts: remotableRefCounts.size,
+      vrefRecognizers: vrefRecognizers.size,
+      kindInfoTable: kindInfoTable.size,
+    };
+  }
+
   return harden({
     droppedCollectionRegistry,
     isDurable,
@@ -676,6 +688,7 @@ export function makeVirtualReferenceManager(
     possibleVirtualObjectDeath,
     ceaseRecognition,
     setDeleteCollectionEntry,
+    getRetentionStats,
     testHooks,
   });
 }

packages/swingset-liveslots/test/test-liveslots-mock-gc.js

@@ -5,7 +5,12 @@ import { Far } from '@endo/marshal';
 import { makeLiveSlots } from '../src/liveslots.js';
 import { kslot, kser } from './kmarshal.js';
 import { buildSyscall } from './liveslots-helpers.js';
-import { makeMessage, makeStartVat, makeBringOutYourDead } from './util.js';
+import {
+  makeMessage,
+  makeStartVat,
+  makeBringOutYourDead,
+  makeResolve,
+} from './util.js';
 import { makeMockGC } from './mock-gc.js';
 
 test('dropImports', async t => {
@@ -31,7 +36,8 @@ test('dropImports', async t => {
   const ls = makeLiveSlots(syscall, 'vatA', {}, {}, gcTools, undefined, () => ({
     buildRootObject: build,
   }));
-  const { dispatch, possiblyDeadSet } = ls;
+  const { dispatch, testHooks } = ls;
+  const { possiblyDeadSet } = testHooks;
   await dispatch(makeStartVat(kser()));
   const allFRs = gcTools.getAllFRs();
   t.is(allFRs.length, 2);
@@ -45,6 +51,7 @@ test('dropImports', async t => {
   // "COLLECTED" state
   t.deepEqual(possiblyDeadSet, new Set());
   t.is(FR.countCallbacks(), 1);
+
   FR.runOneCallback(); // moves to FINALIZED
   t.deepEqual(possiblyDeadSet, new Set(['o-1']));
   possiblyDeadSet.delete('o-1'); // pretend liveslots did syscall.dropImport
@@ -54,6 +61,7 @@ test('dropImports', async t => {
   t.deepEqual(possiblyDeadSet, new Set());
   t.is(FR.countCallbacks(), 0);
   await dispatch(makeMessage(rootA, 'free', []));
+
   t.deepEqual(possiblyDeadSet, new Set());
   t.is(FR.countCallbacks(), 1);
   FR.runOneCallback(); // moves to FINALIZED
@@ -147,3 +155,65 @@ test('dropImports', async t => {
   t.deepEqual(possiblyDeadSet, new Set());
   t.is(FR.countCallbacks(), 0);
 });
+
+test('retention counters', async t => {
+  const { syscall } = buildSyscall();
+  let held;
+  const gcTools = makeMockGC();
+
+  function buildRootObject(_vatPowers) {
+    const root = Far('root', {
+      hold(imp) {
+        held = imp;
+      },
+      exportRemotable() {
+        return Far('exported', {});
+      },
+    });
+    return root;
+  }
+
+  const makeNS = () => ({ buildRootObject });
+  const ls = makeLiveSlots(syscall, 'vatA', {}, {}, gcTools, undefined, makeNS);
+  const { dispatch, testHooks } = ls;
+  const { getRetentionStats } = testHooks;
+
+  const rootA = 'o+0';
+  const presenceVref = 'o-1';
+  const promiseVref = 'p-1';
+  const resultVref = 'p-2';
+
+  await dispatch(makeStartVat(kser()));
+  const count1 = await dispatch(makeBringOutYourDead());
+  t.deepEqual(count1, getRetentionStats());
+  t.is(count1.importedVPIDs, 0);
+  t.is(count1.exportedRemotables, 1);
+  t.is(count1.kernelRecognizableRemotables, 1);
+
+  await dispatch(makeMessage(rootA, 'hold', [kslot(presenceVref)]));
+  t.truthy(held);
+
+  const count2 = await dispatch(makeBringOutYourDead());
+  t.is(count2.slotToVal, count1.slotToVal + 1);
+
+  gcTools.kill(held);
+  gcTools.flushAllFRs();
+  const count3 = await dispatch(makeBringOutYourDead());
+  t.is(count3.slotToVal, count2.slotToVal - 1);
+
+  await dispatch(makeMessage(rootA, 'hold', [kslot(promiseVref)]));
+  const count4 = await dispatch(makeBringOutYourDead());
+  t.is(count4.slotToVal, count3.slotToVal + 1);
+  t.is(count4.importedVPIDs, 1);
+
+  await dispatch(makeResolve(promiseVref, kser(undefined)));
+  const count5 = await dispatch(makeBringOutYourDead());
+  t.is(count5.slotToVal, count4.slotToVal - 1);
+  t.is(count5.importedVPIDs, 0);
+
+  await dispatch(makeMessage(rootA, 'exportRemotable', [], resultVref));
+  const count6 = await dispatch(makeBringOutYourDead());
+  t.is(count6.exportedRemotables, 2);
+  t.is(count6.kernelRecognizableRemotables, 2);
+  t.is(count6.slotToVal, count5.slotToVal + 1);
+});

packages/swingset-liveslots/test/test-liveslots.js

@@ -428,6 +428,7 @@ test('liveslots vs symbols', async t => {
     });
   }
   const { dispatch } = await makeDispatch(syscall, build);
+
   log.length = 0; // assume pre-build vatstore operations are correct
   const rootA = 'o+0';
   const target = 'o-1';

packages/swingset-runner/demo/virtualObjectGC/swingset.json

@@ -5,10 +5,7 @@
       "sourceSpec": "bootstrap.js"
     },
     "bob": {
-      "sourceSpec": "vat-bob.js",
-      "creationOptions": {
-        "virtualObjectCacheSize": 0
-      }
+      "sourceSpec": "vat-bob.js"
     }
   }
 }

packages/swingset-xsnap-supervisor/lib/supervisor-helper.js

@@ -35,7 +35,7 @@ function makeSupervisorDispatch(dispatch) {
     return Promise.resolve(delivery)
       .then(dispatch)
       .then(
-        () => harden(['ok', null, null]),
+        res => harden(['ok', res, null]),
         err => {
           // TODO react more thoughtfully, maybe terminate the vat
           console.warn(`error during vat dispatch() of ${delivery}`, err);