diff --git a/packages/SwingSet/test/virtualObjects/test-representatives.js b/packages/SwingSet/test/virtualObjects/test-representatives.js index 0336c6d0d228..c39bdb4ebd78 100644 --- a/packages/SwingSet/test/virtualObjects/test-representatives.js +++ b/packages/SwingSet/test/virtualObjects/test-representatives.js @@ -454,19 +454,19 @@ test('virtual object gc', async t => { [`${v}.vs.vc.1.|entryCount`]: '0', [`${v}.vs.vc.1.|label`]: 'baggage', [`${v}.vs.vc.1.|nextOrdinal`]: '1', - [`${v}.vs.vc.1.|schemata`]: vstr([M.string()]), + [`${v}.vs.vc.1.|schemata`]: vstr({ keyShape: M.string() }), [`${v}.vs.vc.2.|entryCount`]: '0', [`${v}.vs.vc.2.|label`]: 'promiseRegistrations', [`${v}.vs.vc.2.|nextOrdinal`]: '1', - [`${v}.vs.vc.2.|schemata`]: vstr([M.scalar()]), + [`${v}.vs.vc.2.|schemata`]: vstr({ keyShape: M.scalar() }), [`${v}.vs.vc.3.|entryCount`]: '0', [`${v}.vs.vc.3.|label`]: 'promiseWatcherByKind', [`${v}.vs.vc.3.|nextOrdinal`]: '1', - [`${v}.vs.vc.3.|schemata`]: vstr([M.scalar()]), + [`${v}.vs.vc.3.|schemata`]: vstr({ keyShape: M.scalar() }), [`${v}.vs.vc.4.|entryCount`]: '0', [`${v}.vs.vc.4.|label`]: 'watchedPromises', [`${v}.vs.vc.4.|nextOrdinal`]: '1', - [`${v}.vs.vc.4.|schemata`]: vstr([M.and(M.scalar(), M.string())]), + [`${v}.vs.vc.4.|schemata`]: vstr({ keyShape: M.and(M.scalar(), M.string()) }), [`${v}.vs.vom.es.o+v10/3`]: 'r', [`${v}.vs.vom.o+v10/2`]: `{"label":${vstr('thing #2')}}`, [`${v}.vs.vom.o+v10/3`]: `{"label":${vstr('thing #3')}}`, diff --git a/packages/swingset-liveslots/src/collectionManager.js b/packages/swingset-liveslots/src/collectionManager.js index 813b22efc32b..3c503f8dfd11 100644 --- a/packages/swingset-liveslots/src/collectionManager.js +++ b/packages/swingset-liveslots/src/collectionManager.js @@ -21,6 +21,7 @@ import { enumerateKeysStartEnd, enumerateKeysWithPrefix, } from './vatstore-iterators.js'; +import { makeCache } from './cache.js'; // XXX TODO: The following key length limit was put in place due to limitations // in LMDB. With the move away from LMDB, it is no longer relevant, but I'm @@ -48,6 +49,58 @@ function throwNotDurable(value, slotIndex, serializedValue) { ); } +function prefixc(collectionID, dbEntryKey) { + return `vc.${collectionID}.${dbEntryKey}`; +} + +/** + * @typedef {object} SchemaCacheValue + * @property {Pattern} keyShape + * @property {Pattern} valueShape + * @property {string} label + * @property {object} schemataCD + */ + +/* + * Build a cache that holds the schema for each collection. + * + * The cache maps colectionID to { keyShape, valueShape, label + * }. These are initialized when the collection is first constructed, + * and never modified afterwards. The values live in the vatstore, + * inside two keys, one for the [keyShape, valueShape] schemata, + * another for the label. + */ +function makeSchemaCache(syscall, unserialize) { + /** @type {(collectionID: string) => SchemaCacheValue} */ + const readBacking = collectionID => { + // this is only called once per crank + const schemataKey = prefixc(collectionID, '|schemata'); + const schemataValue = syscall.vatstoreGet(schemataKey); + const schemataCD = JSON.parse(schemataValue); + const { keyShape, valueShape } = unserialize(schemataCD); + const labelKey = prefixc(collectionID, '|label'); + const label = syscall.vatstoreGet(labelKey); + return harden({ keyShape, valueShape, label, schemataCD }); + }; + /** @type {(collectionID: string, value: SchemaCacheValue) => void } */ + const writeBacking = (collectionID, value) => { + const { label, schemataCD } = value; + const schemataKey = prefixc(collectionID, '|schemata'); + const schemataValue = JSON.stringify(schemataCD); + syscall.vatstoreSet(schemataKey, schemataValue); + const labelKey = prefixc(collectionID, '|label'); + syscall.vatstoreSet(labelKey, label); + }; + /** @type {(collectionID: string) => void} */ + const deleteBacking = collectionID => { + const schemataKey = prefixc(collectionID, '|schemata'); + const labelKey = prefixc(collectionID, '|label'); + syscall.vatstoreDelete(schemataKey); + syscall.vatstoreDelete(labelKey); + }; + return makeCache(readBacking, writeBacking, deleteBacking); +} + export function makeCollectionManager( syscall, vrm, @@ -62,6 +115,9 @@ export function makeCollectionManager( ) { const storeKindIDToName = new Map(); + /** @type { import('./cache.js').Cache} */ + const schemaCache = makeSchemaCache(syscall, unserialize); + const storeKindInfo = { scalarMapStore: { hasWeakKeys: false, @@ -121,10 +177,6 @@ export function makeCollectionManager( }, }; - function prefixc(collectionID, dbEntryKey) { - return `vc.${collectionID}.${dbEntryKey}`; - } - function initializeStoreKindInfo() { let storeKindIDs = {}; const rawTable = syscall.vatstoreGet('storeKindIDTable'); @@ -183,35 +235,33 @@ export function makeCollectionManager( } vrm.setDeleteCollectionEntry(deleteCollectionEntry); - function summonCollectionInternal( - _initial, - label, - collectionID, - kindName, - keyShape = M.any(), - valueShape, - ) { + function summonCollectionInternal(_initial, collectionID, kindName) { assert.typeof(kindName, 'string'); const kindInfo = storeKindInfo[kindName]; assert(kindInfo, `unknown collection kind ${kindName}`); const { hasWeakKeys, durable } = kindInfo; + const getSchema = () => schemaCache.get(collectionID); const dbKeyPrefix = `vc.${collectionID}.`; let currentGenerationNumber = 0; - const invalidKeyTypeMsg = `invalid key type for collection ${q(label)}`; - const invalidValueTypeMsg = `invalid value type for collection ${q(label)}`; + const makeInvalidKeyTypeMsg = label => + `invalid key type for collection ${q(label)}`; + const makeInvalidValueTypeMsg = label => + `invalid value type for collection ${q(label)}`; const serializeValue = value => { + const { valueShape, label } = getSchema(); if (valueShape !== undefined) { - mustMatch(value, valueShape, invalidValueTypeMsg); + mustMatch(value, valueShape, makeInvalidValueTypeMsg(label)); } return serialize(value); }; const unserializeValue = data => { + const { valueShape, label } = getSchema(); const value = unserialize(data); if (valueShape !== undefined) { - mustMatch(value, valueShape, invalidValueTypeMsg); + mustMatch(value, valueShape, makeInvalidValueTypeMsg(label)); } return value; }; @@ -279,6 +329,7 @@ export function makeCollectionManager( } function has(key) { + const { keyShape } = getSchema(); if (!matches(key, keyShape)) { return false; } @@ -290,7 +341,8 @@ export function makeCollectionManager( } function get(key) { - mustMatch(key, keyShape, invalidKeyTypeMsg); + const { keyShape, label } = getSchema(); + mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label)); if (passStyleOf(key) === 'remotable' && getOrdinal(key) === undefined) { throw Fail`key ${key} not found in collection ${q(label)}`; } @@ -312,7 +364,8 @@ export function makeCollectionManager( } const doInit = (key, value, precheckedHas) => { - mustMatch(key, keyShape, invalidKeyTypeMsg); + const { keyShape, label } = getSchema(); + mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label)); precheckedHas || !has(key) || Fail`key ${key} already registered in collection ${q(label)}`; @@ -352,7 +405,8 @@ export function makeCollectionManager( }; function set(key, value) { - mustMatch(key, keyShape, invalidKeyTypeMsg); + const { keyShape, label } = getSchema(); + mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label)); const after = serializeValue(harden(value)); assertAcceptableSyscallCapdataSize([after]); if (durable) { @@ -371,7 +425,8 @@ export function makeCollectionManager( } function deleteInternal(key) { - mustMatch(key, keyShape, invalidKeyTypeMsg); + const { keyShape, label } = getSchema(); + mustMatch(key, keyShape, makeInvalidKeyTypeMsg(label)); if (passStyleOf(key) === 'remotable' && getOrdinal(key) === undefined) { throw Fail`key ${key} not found in collection ${q(label)}`; } @@ -582,23 +637,9 @@ export function makeCollectionManager( }; } - function summonCollection( - initial, - label, - collectionID, - kindName, - keyShape, - valueShape, - ) { + function summonCollection(initial, collectionID, kindName) { const hasWeakKeys = storeKindInfo[kindName].hasWeakKeys; - const raw = summonCollectionInternal( - initial, - label, - collectionID, - kindName, - keyShape, - valueShape, - ); + const raw = summonCollectionInternal(initial, collectionID, kindName); const { has, get, init, addToSet, set, delete: del } = raw; const weakMethods = { @@ -642,12 +683,7 @@ export function makeCollectionManager( const kindName = storeKindIDToName.get(`${id}`); assert(kindName, `unknown kind ID ${id}`); const collectionID = `${subid}`; - const collection = summonCollectionInternal( - false, - 'test', - collectionID, - kindName, - ); + const collection = summonCollectionInternal(false, collectionID, kindName); return collection.sizeInternal(); } @@ -655,17 +691,19 @@ export function makeCollectionManager( const { id, subid } = parseVatSlot(vobjID); const kindName = storeKindIDToName.get(`${id}`); const collectionID = `${subid}`; - const collection = summonCollectionInternal( - false, - 'GC', - collectionID, - kindName, - ); + const collection = summonCollectionInternal(false, collectionID, kindName); const doMoreGC = collection.clearInternal(true); - for (const dbKey of enumerateKeysWithPrefix(syscall, prefixc(subid, '|'))) { + for (const dbKey of enumerateKeysWithPrefix( + syscall, + prefixc(collectionID, '|'), + )) { + if (dbKey.endsWith('|schemata') || dbKey.endsWith('|label')) { + continue; // will be deleted when schemaCache is flushed + } syscall.vatstoreDelete(dbKey); } + schemaCache.delete(collectionID); return doMoreGC; } @@ -673,10 +711,8 @@ export function makeCollectionManager( assert.typeof(label, 'string'); assert(storeKindInfo[kindName]); assertPattern(keyShape); - const schemata = [keyShape]; if (valueShape) { assertPattern(valueShape); - schemata.push(valueShape); } const collectionID = `${allocateCollectionID()}`; const kindID = obtainStoreKindID(kindName); @@ -687,23 +723,21 @@ export function makeCollectionManager( if (!hasWeakKeys) { syscall.vatstoreSet(prefixc(collectionID, '|entryCount'), '0'); } - syscall.vatstoreSet( - prefixc(collectionID, '|schemata'), - JSON.stringify(serialize(harden(schemata))), + + const schemata = {}; // don't populate 'undefined', keep it small + if (keyShape !== undefined) { + schemata.keyShape = keyShape; + } + if (valueShape !== undefined) { + schemata.valueShape = valueShape; + } + const schemataCD = serialize(harden(schemata)); + schemaCache.set( + collectionID, + harden({ keyShape, valueShape, label, schemataCD }), ); - syscall.vatstoreSet(prefixc(collectionID, '|label'), label); - - return [ - vobjID, - summonCollection( - true, - label, - collectionID, - kindName, - keyShape, - valueShape, - ), - ]; + + return [vobjID, summonCollection(true, collectionID, kindName)]; } function collectionToMapStore(collection) { @@ -901,19 +935,7 @@ export function makeCollectionManager( const { id, subid } = parseVatSlot(vobjID); const collectionID = `${subid}`; const kindName = storeKindIDToName.get(`${id}`); - const rawSchemata = JSON.parse( - syscall.vatstoreGet(prefixc(subid, '|schemata')), - ); - const [keyShape, valueShape] = unserialize(rawSchemata); - const label = syscall.vatstoreGet(prefixc(subid, '|label')); - return summonCollection( - false, - label, - collectionID, - kindName, - keyShape, - valueShape, - ); + return summonCollection(false, collectionID, kindName); } function reanimateMapStore(vobjID) { @@ -1000,6 +1022,8 @@ export function makeCollectionManager( const makeScalarBigWeakSetStore = (label = 'weakSet', options = {}) => makeBigWeakSetStore(label, narrowKeyShapeOption(M.scalar(), options)); + const flushSchemaCache = () => schemaCache.flush(); + return harden({ initializeStoreKindInfo, makeScalarBigMapStore, @@ -1007,6 +1031,7 @@ export function makeCollectionManager( makeScalarBigSetStore, makeScalarBigWeakSetStore, provideBaggage, + flushSchemaCache, testHooks, }); } diff --git a/packages/swingset-liveslots/src/liveslots.js b/packages/swingset-liveslots/src/liveslots.js index b26efe470a3a..65b7f9dc2622 100644 --- a/packages/swingset-liveslots/src/liveslots.js +++ b/packages/swingset-liveslots/src/liveslots.js @@ -1517,6 +1517,7 @@ function build( */ function afterDispatchActions() { flushIDCounters(); + collectionManager.flushSchemaCache(); vom.flushStateCache(); } diff --git a/packages/swingset-liveslots/test/test-gc-sensitivity.js b/packages/swingset-liveslots/test/test-gc-sensitivity.js index 4aaf486b63de..332602d73b8a 100644 --- a/packages/swingset-liveslots/test/test-gc-sensitivity.js +++ b/packages/swingset-liveslots/test/test-gc-sensitivity.js @@ -146,7 +146,7 @@ test('representative reanimation', async t => { t.deepEqual(noGCLog, yesGCLog); }); -test.failing('collection reanimation', async t => { +test('collection reanimation', async t => { const { syscall, log } = buildSyscall(); const gcTools = makeMockGC(); diff --git a/packages/swingset-liveslots/test/test-handled-promises.js b/packages/swingset-liveslots/test/test-handled-promises.js index db1aafbaed82..272093ea9b2d 100644 --- a/packages/swingset-liveslots/test/test-handled-promises.js +++ b/packages/swingset-liveslots/test/test-handled-promises.js @@ -141,18 +141,18 @@ const kvStoreDataV1 = Object.entries({ 'vc.1.|label': 'baggage', 'vc.1.|nextOrdinal': '1', 'vc.1.|schemata': - '{"body":"#[{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]","slots":[]}', + '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}}","slots":[]}', // non-durable // 'vc.2.sp+6': '{"body":"#\\"&0\\"","slots":["p+6"]}', // 'vc.2.|entryCount': '1', // 'vc.2.|label': 'promiseRegistrations', // 'vc.2.|nextOrdinal': '1', - // 'vc.2.|schemata': '{"body":"#[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}]","slots":[]}', + // 'vc.2.|schemata': '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', 'vc.3.|entryCount': '0', 'vc.3.|label': 'promiseWatcherByKind', 'vc.3.|nextOrdinal': '1', 'vc.3.|schemata': - '{"body":"#[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}]","slots":[]}', + '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"}}","slots":[]}', 'vc.4.sp+6': '{"body":"#[[\\"$0.Alleged: DurablePromiseIgnorer\\",\\"orphaned\\"]]","slots":["o+d10/1"]}', 'vc.4.sp-8': @@ -163,7 +163,7 @@ const kvStoreDataV1 = Object.entries({ 'vc.4.|label': 'watchedPromises', 'vc.4.|nextOrdinal': '1', 'vc.4.|schemata': - '{"body":"#[{\\"#tag\\":\\"match:and\\",\\"payload\\":[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]}]","slots":[]}', + '{"body":"#{\\"keyShape\\":{\\"#tag\\":\\"match:and\\",\\"payload\\":[{\\"#tag\\":\\"match:scalar\\",\\"payload\\":\\"#undefined\\"},{\\"#tag\\":\\"match:string\\",\\"payload\\":[]}]}}","slots":[]}', 'vom.dkind.10': '{"kindID":"10","tag":"DurablePromiseIgnorer","nextInstanceID":2,"unfaceted":true}', 'vom.o+d10/1': '{}', diff --git a/packages/swingset-liveslots/test/test-initial-vrefs.js b/packages/swingset-liveslots/test/test-initial-vrefs.js index 04ce20281a62..acf7f7a8181f 100644 --- a/packages/swingset-liveslots/test/test-initial-vrefs.js +++ b/packages/swingset-liveslots/test/test-initial-vrefs.js @@ -67,7 +67,7 @@ test('initial vatstore contents', async t => { t.is(get(`vc.1.|entryCount`), '0'); // no entries yet t.is(get(`vc.1.|nextOrdinal`), '1'); // no ordinals yet t.is(get(`vc.1.|entryCount`), '0'); - const stringSchema = [M.string()]; + const stringSchema = { keyShape: M.string() }; t.deepEqual(kunser(JSON.parse(get(`vc.1.|schemata`))), stringSchema); // then three tables for the promise watcher (one virtual, two durable) @@ -89,11 +89,13 @@ test('initial vatstore contents', async t => { t.is(get(`vom.rc.${watchedPromiseTableVref}`), '1'); // promiseRegistrations and promiseWatcherByKind arbitrary scalars as keys - const scalarSchema = [M.scalar()]; + const scalarSchema = { keyShape: M.scalar() }; t.deepEqual(kunser(JSON.parse(get(`vc.2.|schemata`))), scalarSchema); t.deepEqual(kunser(JSON.parse(get(`vc.3.|schemata`))), scalarSchema); // watchedPromises uses vref (string) keys - const scalarStringSchema = [M.and(M.scalar(), M.string())]; + const scalarStringSchema = { + keyShape: M.and(M.scalar(), M.string()), + }; t.deepEqual(kunser(JSON.parse(get(`vc.4.|schemata`))), scalarStringSchema); }); diff --git a/packages/swingset-liveslots/test/test-liveslots.js b/packages/swingset-liveslots/test/test-liveslots.js index 40313fee8b29..fa2092c4e4fe 100644 --- a/packages/swingset-liveslots/test/test-liveslots.js +++ b/packages/swingset-liveslots/test/test-liveslots.js @@ -5,6 +5,7 @@ import { E } from '@endo/eventual-send'; import { Far } from '@endo/marshal'; import { makePromiseKit } from '@endo/promise-kit'; import { Fail } from '@agoric/assert'; +import { M } from '@agoric/store'; import { makeLiveSlots, makeMarshaller } from '../src/liveslots.js'; import { kslot, kser, kunser } from './kmarshal.js'; import { buildSyscall, makeDispatch } from './liveslots-helpers.js'; @@ -708,8 +709,22 @@ test('capdata size limit on syscalls', async t => { expectStore(15); t.deepEqual(log, []); + const gotSchema = () => { + t.deepEqual(log.shift(), { + type: 'vatstoreGet', + key: 'vc.5.|schemata', + result: JSON.stringify(kser({ keyShape: M.scalar() })), + }); + t.deepEqual(log.shift(), { + type: 'vatstoreGet', + key: 'vc.5.|label', + result: 'test', + }); + }; + rp = nextRP(); await send('storeInitTooManySlots'); + gotSchema(); t.deepEqual(log.shift(), { type: 'vatstoreGet', key: 'vc.5.skey', @@ -721,6 +736,7 @@ test('capdata size limit on syscalls', async t => { rp = nextRP(); await send('storeInitBodyTooBig'); + gotSchema(); t.deepEqual(log.shift(), { type: 'vatstoreGet', key: 'vc.5.skey', @@ -732,12 +748,14 @@ test('capdata size limit on syscalls', async t => { rp = nextRP(); await send('storeSetTooManySlots'); + gotSchema(); expectFail(); expectVoidReturn(); t.deepEqual(log, []); rp = nextRP(); await send('storeSetBodyTooBig'); + gotSchema(); expectFail(); expectVoidReturn(); t.deepEqual(log, []);