feat(swingset): add controller.terminateVat(vatID, reason) (#9253)

This new API allows the host application to terminate any vat for which is knows the VatID (which must be gleaned manually from logs or the database). This might be useful if the normal vat code is unable or unwilling to terminate the vat, or if you need to trigger termination at some specific point in time.

closes #8687

packages/SwingSet/src/controller/controller.js

@@ -21,6 +21,7 @@ import { waitUntilQuiescent } from '@agoric/internal/src/lib-nodejs/waitUntilQui
 import { makeGcAndFinalize } from '@agoric/internal/src/lib-nodejs/gc-and-finalize.js';
 import { kslot, krefOf } from '@agoric/kmarshal';
 import { insistStorageAPI } from '../lib/storageAPI.js';
+import { insistCapData } from '../lib/capdata.js';
 import {
   buildKernelBundle,
   swingsetIsInitialized,
@@ -33,6 +34,10 @@ import {
 import { makeStartXSnap } from './startXSnap.js';
 import { makeStartSubprocessWorkerNode } from './startNodeSubprocess.js';
 
+/**
+ * @typedef { import('../types-internal.js').VatID } VatID
+ */
+
 const endoZipBase64Sha512Shape = harden({
   moduleFormat: 'endoZipBase64',
   endoZipBase64: M.string(harden({ stringLengthLimit: Infinity })),
@@ -441,6 +446,48 @@ export async function makeSwingsetController(
       // no emitCrankHashes here because queueToVatRoot did that
       return result;
     },
+
+    /**
+     * terminate a vat by ID
+     *
+     * This allows the host app to terminate any vat. The effect is
+     * equivalent to the holder of the vat's `adminNode` calling
+     * `E(adminNode).terminateWithFailure(reason)`, or the vat itself
+     * calling `vatPowers.exitVatWithFailure(reason)`. It accepts a
+     * reason capdata structure (use 'kser()' to build one), which
+     * will be included in rejection data for the promise available to
+     * `E(adminNode).done()`, just like the internal termination APIs.
+     * Note that no slots/krefs are allowed in 'reason' when
+     * terminating the vat externally.
+     *
+     * This is a superpower available only from the host app, not from
+     * within vats, since `vatID` is merely a string and can be forged
+     * trivially. The host app is responsible for supplying the right
+     * vatID to kill, by looking at the database or logs (note that
+     * vats do not know their own vatID, and `controller.vatNameToID`
+     * only works for static vats, not dynamic).
+     *
+     * This will cause state changes in the swing-store (specifically
+     * marking the vat as terminated, and rejection all its
+     * outstanding promises), which must be committed before they will
+     * be durable. Either call `hostStorage.commit()` immediately
+     * after calling this, or call `controller.run()` and *then*
+     * `hostStorage.commit()` as you would normally do in response to
+     * other I/O or timer activity.
+     *
+     * The first `controller.run()` after this call will delete all
+     * the old vat's state at once, unless you use a
+     * [`runPolicy`](../docs/run-policy.md) to rate-limit cleanups.
+     *
+     * @param {VatID} vatID
+     * @param {SwingSetCapData} reasonCD
+     */
+
+    terminateVat(vatID, reasonCD) {
+      insistCapData(reasonCD);
+      assert(reasonCD.slots.length === 0, 'no slots allowed in reason');
+      kernel.terminateVatExternally(vatID, reasonCD);
+    },
   });
 
   writeSlogObject({ type: 'kernel-init-finish' });

packages/SwingSet/src/kernel/kernel.js

@@ -2063,6 +2063,16 @@ export default function buildKernel(
     hooks[hookName] = hook;
   }
 
+  function terminateVatExternally(vatID, reasonCD) {
+    assert(started, 'must do kernel.start() before terminateVatExternally()');
+    insistCapData(reasonCD);
+    assert(reasonCD.slots.length === 0, 'no slots allowed in reason');
+    // this fires a promise when worker is dead, mostly for tests, so don't
+    // give it to external callers
+    void terminateVat(vatID, true, reasonCD);
+    console.log(`scheduled vatID ${vatID} for termination`);
+  }
+
   const kernel = harden({
     // these are meant for the controller
     installBundle,
@@ -2138,6 +2148,7 @@ export default function buildKernel(
     kpStatus,
     kpResolution,
     addDeviceHook,
+    terminateVatExternally,
   });
 
   return kernel;

packages/SwingSet/test/external-termination/bootstrap-external-termination.js

@@ -0,0 +1,27 @@
+import { Far, E } from '@endo/far';
+
+export function buildRootObject() {
+  let vatAdmin;
+  let bcap;
+  let root;
+  let adminNode;
+  let exitval;
+
+  return Far('root', {
+    bootstrap: async (vats, devices) => {
+      vatAdmin = await E(vats.vatAdmin).createVatAdminService(devices.vatAdmin);
+      bcap = await E(vatAdmin).getNamedBundleCap('doomed');
+      const res = await E(vatAdmin).createVat(bcap);
+      root = res.root;
+      adminNode = res.adminNode;
+      E(adminNode)
+        .done()
+        .then(
+          happy => (exitval = ['fulfill', happy]),
+          sad => (exitval = ['reject', sad]),
+        );
+    },
+    ping: async count => E(root).ping(count),
+    getExitVal: () => exitval,
+  });
+}

packages/SwingSet/test/external-termination/external-termination.test.js

@@ -0,0 +1,80 @@
+// eslint-disable-next-line import/order
+import { test } from '../../tools/prepare-test-env-ava.js';
+
+import { initSwingStore } from '@agoric/swing-store';
+import { kser, kunser } from '@agoric/kmarshal';
+import { initializeSwingset, makeSwingsetController } from '../../src/index.js';
+
+const bfile = name => new URL(name, import.meta.url).pathname;
+
+const testExternalTermination = async (t, defaultManagerType) => {
+  /** @type {SwingSetConfig} */
+  const config = {
+    defaultManagerType,
+    bootstrap: 'bootstrap',
+    vats: {
+      bootstrap: { sourceSpec: bfile('./bootstrap-external-termination.js') },
+    },
+    bundles: {
+      doomed: { sourceSpec: bfile('./vat-doomed.js') },
+    },
+  };
+
+  const kernelStorage = initSwingStore().kernelStorage;
+  await initializeSwingset(config, [], kernelStorage);
+  const c = await makeSwingsetController(kernelStorage);
+  t.teardown(c.shutdown);
+  c.pinVatRoot('bootstrap');
+  await c.run();
+
+  const getVatIDs = () => c.dump().vatTables.map(vt => vt.vatID);
+
+  // vat-doomed should now be running. We casually assume the new vat
+  // has the last ID
+  const vatIDs = getVatIDs();
+  const vatID = vatIDs[vatIDs.length - 1];
+
+  {
+    const kpid = c.queueToVatRoot('bootstrap', 'ping', [1]);
+    await c.run();
+    t.is(kunser(c.kpResolution(kpid)), 1);
+  }
+  {
+    const kpid = c.queueToVatRoot('bootstrap', 'getExitVal');
+    await c.run();
+    t.is(kunser(c.kpResolution(kpid)), undefined);
+  }
+
+  // The "vat has been terminated" flags are set synchronously during
+  // c.terminateVat(), as well as all the vat's promises being
+  // rejected. The deletion of state happens during the first cleanup
+  // crank, which (since we aren't limiting it with a runPolicy)
+  // cleans to completion during this c.run()
+
+  c.terminateVat(vatID, kser('doom!'));
+  await c.run();
+
+  t.false(getVatIDs().includes(vatID));
+
+  {
+    // this provokes noise: liveslots logs one RemoteError
+    const kpid = c.queueToVatRoot('bootstrap', 'ping', [1]);
+    await c.run();
+    t.is(c.kpStatus(kpid), 'rejected');
+    t.deepEqual(kunser(c.kpResolution(kpid)), Error('vat terminated'));
+  }
+
+  {
+    const kpid = c.queueToVatRoot('bootstrap', 'getExitVal');
+    await c.run();
+    t.deepEqual(kunser(c.kpResolution(kpid)), ['reject', 'doom!']);
+  }
+};
+
+test('external termination: local worker', async t => {
+  await testExternalTermination(t, 'local');
+});
+
+test('external termination: xsnap worker', async t => {
+  await testExternalTermination(t, 'xsnap');
+});

packages/SwingSet/test/external-termination/vat-doomed.js

@@ -0,0 +1,7 @@
+import { Far } from '@endo/far';
+
+export function buildRootObject() {
+  return Far('doomed', {
+    ping: count => count,
+  });
+}