From 09a7d009d7d181fcda3a302ec9a49cf611bf1ce7 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Thu, 6 Apr 2023 23:20:54 -0700 Subject: [PATCH] swingset-liveslots: merge '|label' into '|schemata' Virtual collections need to record a label and a keyShape/valueShape schema. Previously these were kept in two separate vatstore keys, `|label` and `|schemata`. With this change, they're both stored in the `|schemata` key. This should reduce the number of vatstore reads necessary to work with a collection. --- .../virtualObjects/test-representatives.js | 12 ++++------ .../src/collectionManager.js | 18 +++++---------- .../swingset-liveslots/src/vatstore-usage.md | 5 ++-- .../swingset-liveslots/test/gc-helpers.js | 1 - .../test/storeGC/test-lifecycle.js | 10 ++++---- .../swingset-liveslots/test/test-baggage.js | 5 +++- .../test/test-handled-promises.js | 12 ++++------ .../test/test-initial-vrefs.js | 23 +++++++++++-------- .../test/test-liveslots-mock-gc.js | 1 - .../swingset-liveslots/test/test-liveslots.js | 8 ++----- 10 files changed, 41 insertions(+), 54 deletions(-) diff --git a/packages/SwingSet/test/virtualObjects/test-representatives.js b/packages/SwingSet/test/virtualObjects/test-representatives.js index c39bdb4ebd78..b4214c889bb2 100644 --- a/packages/SwingSet/test/virtualObjects/test-representatives.js +++ b/packages/SwingSet/test/virtualObjects/test-representatives.js @@ -452,21 +452,17 @@ test('virtual object gc', async t => { [`${v}.vs.storeKindIDTable`]: '{"scalarMapStore":2,"scalarWeakMapStore":3,"scalarSetStore":4,"scalarWeakSetStore":5,"scalarDurableMapStore":6,"scalarDurableWeakMapStore":7,"scalarDurableSetStore":8,"scalarDurableWeakSetStore":9}', [`${v}.vs.vc.1.|entryCount`]: '0', - [`${v}.vs.vc.1.|label`]: 'baggage', [`${v}.vs.vc.1.|nextOrdinal`]: '1', - [`${v}.vs.vc.1.|schemata`]: vstr({ keyShape: M.string() }), + [`${v}.vs.vc.1.|schemata`]: vstr({ label: 'baggage', keyShape: M.string() }), [`${v}.vs.vc.2.|entryCount`]: '0', - [`${v}.vs.vc.2.|label`]: 'promiseRegistrations', [`${v}.vs.vc.2.|nextOrdinal`]: '1', - [`${v}.vs.vc.2.|schemata`]: vstr({ keyShape: M.scalar() }), + [`${v}.vs.vc.2.|schemata`]: vstr({ label: 'promiseRegistrations', keyShape: M.scalar() }), [`${v}.vs.vc.3.|entryCount`]: '0', - [`${v}.vs.vc.3.|label`]: 'promiseWatcherByKind', [`${v}.vs.vc.3.|nextOrdinal`]: '1', - [`${v}.vs.vc.3.|schemata`]: vstr({ keyShape: M.scalar() }), + [`${v}.vs.vc.3.|schemata`]: vstr({ label: 'promiseWatcherByKind', keyShape: M.scalar() }), [`${v}.vs.vc.4.|entryCount`]: '0', - [`${v}.vs.vc.4.|label`]: 'watchedPromises', [`${v}.vs.vc.4.|nextOrdinal`]: '1', - [`${v}.vs.vc.4.|schemata`]: vstr({ keyShape: M.and(M.scalar(), M.string()) }), + [`${v}.vs.vc.4.|schemata`]: vstr({ label: 'watchedPromises', keyShape: M.and(M.scalar(), M.string()) }), [`${v}.vs.vom.es.o+v10/3`]: 'r', [`${v}.vs.vom.o+v10/2`]: `{"label":${vstr('thing #2')}}`, [`${v}.vs.vom.o+v10/3`]: `{"label":${vstr('thing #3')}}`, diff --git a/packages/swingset-liveslots/src/collectionManager.js b/packages/swingset-liveslots/src/collectionManager.js index 15972f12733f..fb63f280ff4f 100644 --- a/packages/swingset-liveslots/src/collectionManager.js +++ b/packages/swingset-liveslots/src/collectionManager.js @@ -75,26 +75,20 @@ function makeSchemaCache(syscall, unserialize) { const schemataKey = prefixc(collectionID, '|schemata'); const schemataValue = syscall.vatstoreGet(schemataKey); const schemataCapData = JSON.parse(schemataValue); - const { keyShape, valueShape } = unserialize(schemataCapData); - const labelKey = prefixc(collectionID, '|label'); - const label = syscall.vatstoreGet(labelKey); + const { label, keyShape, valueShape } = unserialize(schemataCapData); return harden({ keyShape, valueShape, label, schemataCapData }); }; /** @type {(collectionID: string, value: SchemaCacheValue) => void } */ const writeBacking = (collectionID, value) => { - const { label, schemataCapData } = value; + const { schemataCapData } = value; const schemataKey = prefixc(collectionID, '|schemata'); const schemataValue = JSON.stringify(schemataCapData); syscall.vatstoreSet(schemataKey, schemataValue); - const labelKey = prefixc(collectionID, '|label'); - syscall.vatstoreSet(labelKey, label); }; /** @type {(collectionID: string) => void} */ const deleteBacking = collectionID => { const schemataKey = prefixc(collectionID, '|schemata'); - const labelKey = prefixc(collectionID, '|label'); syscall.vatstoreDelete(schemataKey); - syscall.vatstoreDelete(labelKey); }; return makeCache(readBacking, writeBacking, deleteBacking); } @@ -696,9 +690,9 @@ export function makeCollectionManager( syscall, prefixc(collectionID, '|'), )) { - // these two keys are owned by schemaCache, and will be deleted - // when schemaCache is flushed - if (dbKey.endsWith('|schemata') || dbKey.endsWith('|label')) { + // this key is owned by schemaCache, and will be deleted when + // schemaCache is flushed + if (dbKey.endsWith('|schemata')) { continue; } // but we must still delete the other keys (|nextOrdinal and @@ -726,7 +720,7 @@ export function makeCollectionManager( syscall.vatstoreSet(prefixc(collectionID, '|entryCount'), '0'); } - const schemata = {}; // don't populate 'undefined', keep it small + const schemata = { label }; // don't populate 'undefined', keep it small if (keyShape !== undefined) { schemata.keyShape = keyShape; } diff --git a/packages/swingset-liveslots/src/vatstore-usage.md b/packages/swingset-liveslots/src/vatstore-usage.md index 4695193e5b19..5872800ac302 100644 --- a/packages/swingset-liveslots/src/vatstore-usage.md +++ b/packages/swingset-liveslots/src/vatstore-usage.md @@ -180,11 +180,10 @@ c1.init('key4', foo2); Each collection stores a number of metadata keys in the vatstore, all with a prefix of `vc.${collectionID}.|` (note that the collection *type* is not a part of the key, only the collection *index*). The currently defined metadata keys (copied from the record for the "mylabel" Kind stored in `c1`) are: * `v6.vs.vc.2.|entryCount`: `4` (the size of the collection as a numeric string, incremented with each call to `c1.init` and decremented with each call to `c1.delete`) -* `v6.vs.vc.2.|label`: `mylabel` (a debugging label provided in the `make*Store` call that creates the collection) * `v6.vs.vc.2.|nextOrdinal`: `1` (a numeric string counter used to allocate index values for Objects used as keys, see `generateOrdinal` in [collectionManager.js](./collectionManager.js)) -* `v6.vs.vc.2.|schemata`: `{"body":"#[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}]","slots":[]}` +* `v6.vs.vc.2.|schemata`: `{"label":"mylabel","body":"#[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}]","slots":[]}` -The `schemata` is a capdata serialization of the Matcher constraints recorded for the collection. These constraints can limit keys to be just strings, or numbers, etc. (see [Patterns](https://github.com/endojs/endo/tree/master/packages/patterns)). The schemata consists of an array in which the first element is a schema for the keys and the second is a separate schema for the values. +The `schemata` is a capdata serialization of the debugging label provided in the `make*Store` call that created the collection, plus the Matcher constraints recorded for the collection. These constraints can limit keys to be just strings, or numbers, etc. (see [Patterns](https://github.com/endojs/endo/tree/master/packages/patterns)). The schemata consists of an array in which the first element is a schema for the keys and the second is a separate schema for the values. Each entry in the collection gets put into a single vatstore entry with a capdata-serialized value: diff --git a/packages/swingset-liveslots/test/gc-helpers.js b/packages/swingset-liveslots/test/gc-helpers.js index b85012531edd..8736a587895c 100644 --- a/packages/swingset-liveslots/test/gc-helpers.js +++ b/packages/swingset-liveslots/test/gc-helpers.js @@ -182,7 +182,6 @@ export function assertCollectionDeleted(v, baseref) { const { subid: cID } = parseVatSlot(baseref); const { t, fakestore } = v; t.is(fakestore.get(`vom.rc.${baseref}`), undefined); - t.is(fakestore.get(`vc.${cID}.|label`), undefined); t.is(fakestore.get(`vc.${cID}.|schemata`), undefined); t.is(fakestore.get(`vc.${cID}.|nextOrdinal`), undefined); // there should be no ordinal mappings: vc.${cid}.|${vref} diff --git a/packages/swingset-liveslots/test/storeGC/test-lifecycle.js b/packages/swingset-liveslots/test/storeGC/test-lifecycle.js index 72691cbf75d0..58a78e02931f 100644 --- a/packages/swingset-liveslots/test/storeGC/test-lifecycle.js +++ b/packages/swingset-liveslots/test/storeGC/test-lifecycle.js @@ -6,7 +6,7 @@ import { findSyscallsByType, } from '../liveslots-helpers.js'; import { buildRootObject, mainHeldIdx, mapRef } from '../gc-helpers.js'; -import { kslot } from '../kmarshal.js'; +import { kslot, kunser } from '../kmarshal.js'; import { parseVatSlot } from '../../src/parseVatSlots.js'; // These tests follow the model described in @@ -33,12 +33,13 @@ function assertState(v, vref, reachable, erv) { } const vdata = erv[2] === 'V'; const { t, fakestore } = v; + const get = key => fakestore.get(key); + const getLabel = key => kunser(JSON.parse(get(key))).label; const { subid: cID } = parseVatSlot(vref); if (reachable) { - t.is(fakestore.get(`vc.${cID}.|label`), 'store #6'); + t.is(getLabel(`vc.${cID}.|schemata`), 'store #6'); t.truthy(fakestore.get(`vc.${cID}.|entryCount`)); // exists even if 0 t.truthy(fakestore.get(`vc.${cID}.|nextOrdinal`)); - t.truthy(fakestore.get(`vc.${cID}.|schemata`)); if (vdata) { t.is(fakestore.get(`vom.rc.${vref}`), `1`); } else { @@ -52,10 +53,9 @@ function assertState(v, vref, reachable, erv) { t.is(fakestore.get(`vom.es.${vref}`), undefined); } } else { - t.is(fakestore.get(`vc.${cID}.|label`), undefined); + t.is(fakestore.get(`vc.${cID}.|schemata`), undefined); t.is(fakestore.get(`vc.${cID}.|entryCount`), undefined); t.is(fakestore.get(`vc.${cID}.|nextOrdinal`), undefined); - t.is(fakestore.get(`vc.${cID}.|schemata`), undefined); t.is(fakestore.get(`vom.rc.${vref}`), undefined); t.is(fakestore.get(`vom.es.${vref}`), undefined); } diff --git a/packages/swingset-liveslots/test/test-baggage.js b/packages/swingset-liveslots/test/test-baggage.js index e730b8397489..e33d068a7409 100644 --- a/packages/swingset-liveslots/test/test-baggage.js +++ b/packages/swingset-liveslots/test/test-baggage.js @@ -4,6 +4,7 @@ import '@endo/init/debug.js'; import { Far } from '@endo/marshal'; import { setupTestLiveslots } from './liveslots-helpers.js'; import { vstr } from './util.js'; +import { kunser } from './kmarshal.js'; import { parseVatSlot } from '../src/parseVatSlots.js'; function buildRootObject(vatPowers, vatParameters, baggage) { @@ -25,6 +26,8 @@ test.serial('exercise baggage', async t => { { forceGC: true }, ); const { fakestore } = v; + const get = key => fakestore.get(key); + const getLabel = key => kunser(JSON.parse(get(key))).label; const baggageVref = fakestore.get('baggageID'); const { subid } = parseVatSlot(baggageVref); @@ -32,7 +35,7 @@ test.serial('exercise baggage', async t => { const kindIDs = JSON.parse(fakestore.get('storeKindIDTable')); // baggage is the first collection created, a scalarDurableMapStore t.is(baggageVref, `o+d${kindIDs.scalarDurableMapStore}/1`); - t.is(fakestore.get(`vc.${baggageID}.|label`), 'baggage'); + t.is(getLabel(`vc.${baggageID}.|schemata`), 'baggage'); const outsideVal = fakestore.get(`vc.${baggageID}.soutside`); t.is(outsideVal, vstr('outer val')); t.is(fakestore.get(`vc.${baggageID}.|entryCount`), '1'); diff --git a/packages/swingset-liveslots/test/test-handled-promises.js b/packages/swingset-liveslots/test/test-handled-promises.js index 272093ea9b2d..ac88375d6752 100644 --- a/packages/swingset-liveslots/test/test-handled-promises.js +++ b/packages/swingset-liveslots/test/test-handled-promises.js @@ -138,21 +138,18 @@ const kvStoreDataV1 = Object.entries({ 'vc.1.sthe_DurablePromiseIgnorer': '{"body":"#\\"$0.Alleged: DurablePromiseIgnorer\\"","slots":["o+d10/1"]}', 'vc.1.|entryCount': '2', - 'vc.1.|label': 'baggage', 'vc.1.|nextOrdinal': '1', 'vc.1.|schemata': - '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}}","slots":[]}', + '{"label":"baggage","body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}}","slots":[]}', // non-durable // 'vc.2.sp+6': '{"body":"#\\"&0\\"","slots":["p+6"]}', // 'vc.2.|entryCount': '1', - // 'vc.2.|label': 'promiseRegistrations', // 'vc.2.|nextOrdinal': '1', - // 'vc.2.|schemata': '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', + // 'vc.2.|schemata': '{"label":"promiseRegistrations","body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', 'vc.3.|entryCount': '0', - 'vc.3.|label': 'promiseWatcherByKind', 'vc.3.|nextOrdinal': '1', 'vc.3.|schemata': - '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', + '{"label":"promiseWatcherByKind","body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', 'vc.4.sp+6': '{"body":"#[[\\"$0.Alleged: DurablePromiseIgnorer\\",\\"orphaned\\"]]","slots":["o+d10/1"]}', 'vc.4.sp-8': @@ -160,10 +157,9 @@ const kvStoreDataV1 = Object.entries({ 'vc.4.sp-9': '{"body":"#[[\\"$0.Alleged: DurablePromiseIgnorer\\",\\"late-rejected\\"]]","slots":["o+d10/1"]}', 'vc.4.|entryCount': '3', - 'vc.4.|label': 'watchedPromises', 'vc.4.|nextOrdinal': '1', 'vc.4.|schemata': - '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:and\\",\\"payload\\":[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]}}","slots":[]}', + '{"label":"watchedPromises","body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:and\\",\\"payload\\":[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]}}","slots":[]}', 'vom.dkind.10': '{"kindID":"10","tag":"DurablePromiseIgnorer","nextInstanceID":2,"unfaceted":true}', 'vom.o+d10/1': '{}', diff --git a/packages/swingset-liveslots/test/test-initial-vrefs.js b/packages/swingset-liveslots/test/test-initial-vrefs.js index acf7f7a8181f..48ba28a7fd57 100644 --- a/packages/swingset-liveslots/test/test-initial-vrefs.js +++ b/packages/swingset-liveslots/test/test-initial-vrefs.js @@ -56,6 +56,7 @@ test('initial vatstore contents', async t => { }); const { fakestore } = v; const get = key => fakestore.get(key); + const getLabel = key => kunser(JSON.parse(get(key))).label; // an empty buildRootObject should create 4 collections, and some metadata const kindIDs = JSON.parse(get('storeKindIDTable')); @@ -63,20 +64,19 @@ test('initial vatstore contents', async t => { // the first collection is baggage: a durable mapstore const baggageVref = `${dmsBase}/1`; t.is(get('baggageID'), baggageVref); - t.is(get(`vc.1.|label`), 'baggage'); t.is(get(`vc.1.|entryCount`), '0'); // no entries yet t.is(get(`vc.1.|nextOrdinal`), '1'); // no ordinals yet t.is(get(`vc.1.|entryCount`), '0'); - const stringSchema = { keyShape: M.string() }; + const stringSchema = { label: 'baggage', keyShape: M.string() }; t.deepEqual(kunser(JSON.parse(get(`vc.1.|schemata`))), stringSchema); // then three tables for the promise watcher (one virtual, two durable) - t.is(get(`vc.3.|label`), `promiseWatcherByKind`); // durable - t.is(get(`vc.4.|label`), `watchedPromises`); // durable + t.is(getLabel(`vc.3.|schemata`), 'promiseWatcherByKind'); // durable + t.is(getLabel(`vc.4.|schemata`), 'watchedPromises'); // durable // the promiseRegistrations table is not durable, and only gets vc.2 // on the first incarnation: it will get a new ID on subsequent // incarnations - t.is(get(`vc.2.|label`), `promiseRegistrations`); // virtual + t.is(getLabel(`vc.2.|schemata`), `promiseRegistrations`); // virtual const watcherTableVref = get('watcherTableID'); const watchedPromiseTableVref = get('watchedPromiseTableID'); @@ -89,11 +89,13 @@ test('initial vatstore contents', async t => { t.is(get(`vom.rc.${watchedPromiseTableVref}`), '1'); // promiseRegistrations and promiseWatcherByKind arbitrary scalars as keys - const scalarSchema = { keyShape: M.scalar() }; - t.deepEqual(kunser(JSON.parse(get(`vc.2.|schemata`))), scalarSchema); - t.deepEqual(kunser(JSON.parse(get(`vc.3.|schemata`))), scalarSchema); + const scalarSchema2 = { label: 'promiseRegistrations', keyShape: M.scalar() }; + const scalarSchema3 = { label: 'promiseWatcherByKind', keyShape: M.scalar() }; + t.deepEqual(kunser(JSON.parse(get(`vc.2.|schemata`))), scalarSchema2); + t.deepEqual(kunser(JSON.parse(get(`vc.3.|schemata`))), scalarSchema3); // watchedPromises uses vref (string) keys const scalarStringSchema = { + label: 'watchedPromises', keyShape: M.and(M.scalar(), M.string()), }; t.deepEqual(kunser(JSON.parse(get(`vc.4.|schemata`))), scalarStringSchema); @@ -108,6 +110,9 @@ test('vrefs', async t => { ); // const { fakestore, dumpFakestore } = v; const { fakestore } = v; + const get = key => fakestore.get(key); + const getLabel = key => kunser(JSON.parse(get(key))).label; + const kindIDID = JSON.parse(fakestore.get('kindIDID')); const initialKindIDs = JSON.parse(fakestore.get('storeKindIDTable')); const initialCounters = JSON.parse(fakestore.get(`idCounters`)); @@ -139,7 +144,7 @@ test('vrefs', async t => { // the liveslots-created collections consume vc.1 through vc.4, // leaving vc.5 for the first user-created collection - t.is(fakestore.get('vc.5.|label'), 'store1'); + t.is(getLabel('vc.5.|schemata'), 'store1'); const expectedStore1Vref = `o+v${initialKindIDs.scalarMapStore}/5`; const store1Vref = (await run('getStore1')).slots[0]; t.is(store1Vref, expectedStore1Vref); diff --git a/packages/swingset-liveslots/test/test-liveslots-mock-gc.js b/packages/swingset-liveslots/test/test-liveslots-mock-gc.js index 1ddb03633efa..20e8a5274c19 100644 --- a/packages/swingset-liveslots/test/test-liveslots-mock-gc.js +++ b/packages/swingset-liveslots/test/test-liveslots-mock-gc.js @@ -448,7 +448,6 @@ const doublefreetest = test.macro(async (t, mode) => { // all collection metadata should be gone const collectionID = String(parseVatSlot(vref).subid); t.is(fakestore.get(`vc.${collectionID}.|schemata`), undefined); - t.is(fakestore.get(`vc.${collectionID}.|label`), undefined); t.is(fakestore.get(`vc.${collectionID}.|nextOrdinal`), undefined); t.is(fakestore.get(`vc.${collectionID}.|entryCount`), undefined); } diff --git a/packages/swingset-liveslots/test/test-liveslots.js b/packages/swingset-liveslots/test/test-liveslots.js index ed6cb33bdd6c..4bbebcd4d36d 100644 --- a/packages/swingset-liveslots/test/test-liveslots.js +++ b/packages/swingset-liveslots/test/test-liveslots.js @@ -711,15 +711,11 @@ test('capdata size limit on syscalls', async t => { t.deepEqual(log, []); const gotSchema = () => { + const label = 'test'; t.deepEqual(log.shift(), { type: 'vatstoreGet', key: 'vc.5.|schemata', - result: JSON.stringify(kser({ keyShape: M.scalar() })), - }); - t.deepEqual(log.shift(), { - type: 'vatstoreGet', - key: 'vc.5.|label', - result: 'test', + result: JSON.stringify(kser({ label, keyShape: M.scalar() })), }); };