From eeacc77e0aa8e0c244ee79b3d8ee65c1693c0ed6 Mon Sep 17 00:00:00 2001 From: "Mark S. Miller" Date: Wed, 31 May 2023 23:56:18 -0700 Subject: [PATCH] fix: switch to iterables --- .../marshal/src/builders/builder-types.js | 8 +- .../marshal/src/builders/justinBuilder.js | 132 ++++++ .../marshal/src/builders/smallcapsBuilder.js | 80 ++-- .../marshal/src/builders/subgraphBuilder.js | 67 ++- packages/marshal/src/marshal-justin.js | 4 +- .../test/builders/test-justin-builder.js | 57 +++ .../test/builders/test-smallcaps-builder.js | 2 +- .../test/test-marshal-justin-builder.js | 161 +++++++ .../test/test-marshal-smallcaps-builder.js | 419 ++++++++++++++++++ .../marshal/test/test-marshal-smallcaps.js | 38 +- 10 files changed, 887 insertions(+), 81 deletions(-) create mode 100644 packages/marshal/src/builders/justinBuilder.js create mode 100644 packages/marshal/test/builders/test-justin-builder.js create mode 100644 packages/marshal/test/test-marshal-justin-builder.js create mode 100644 packages/marshal/test/test-marshal-smallcaps-builder.js diff --git a/packages/marshal/src/builders/builder-types.js b/packages/marshal/src/builders/builder-types.js index 238b4a8499..5ca9478b32 100644 --- a/packages/marshal/src/builders/builder-types.js +++ b/packages/marshal/src/builders/builder-types.js @@ -4,7 +4,7 @@ * @template N * @template R * @typedef {object} Builder - * @property {(node: N) => R} buildRoot + * @property {(buildTopFn: () => N) => R} buildRoot * * @property {() => N} buildUndefined * @property {() => N} buildNull @@ -14,14 +14,14 @@ * @property {(str: string) => N} buildString * @property {(sym: symbol) => N} buildSymbol * - * @property {(builtEntries: [string, N][]) => N} buildRecord + * @property {(names: string[], buildValuesIter: Iterable) => N} buildRecord * The recognizer must pass the actual property names through. It is * up to the builder whether it wants to encode them. * It is up to the recognizer to sort the entries by their actual * property name first, and to encode their values in the resulting * sorted order. The builder may assume that sorted order. - * @property {(builtElements: N[]) => N} buildArray - * @property {(tagName: string, builtPayload: N) => N} buildTagged + * @property {(count: number, buildElementsIter: Iterable) => N} buildArray + * @property {(tagName: string, buildPayloadFn: () => N) => N} buildTagged * The recognizer must pass the actual tagName through. It is * up to the builder whether it wants to encode it. * diff --git a/packages/marshal/src/builders/justinBuilder.js b/packages/marshal/src/builders/justinBuilder.js new file mode 100644 index 0000000000..53310e8bde --- /dev/null +++ b/packages/marshal/src/builders/justinBuilder.js @@ -0,0 +1,132 @@ +/// + +import { Far, getInterfaceOf, nameForPassableSymbol } from '@endo/pass-style'; +import { + identPattern, + AtAtPrefixPattern, + makeNoIndenter, + makeYesIndenter, +} from '../marshal-justin.js'; + +const { stringify } = JSON; +const { Fail, quote: q } = assert; +const { is } = Object; + +export const makeJustinBuilder = (shouldIndent = false, _slots = []) => { + let out; + let slotIndex; + const outNextJSON = val => out.next(stringify(val)); + + /** @type {Builder} */ + const justinBuilder = Far('JustinBuilder', { + buildRoot: buildTopFn => { + const makeIndenter = shouldIndent ? makeYesIndenter : makeNoIndenter; + out = makeIndenter(); + slotIndex = -1; + buildTopFn(); + return out.done(); + }, + + buildUndefined: () => out.next('undefined'), + buildNull: () => out.next('null'), + buildBoolean: outNextJSON, + buildNumber: num => { + if (num === Infinity) { + return out.next('Infinity'); + } else if (num === -Infinity) { + return out.next('-Infinity'); + } else if (is(num, NaN)) { + return out.next('NaN'); + } else { + return out.next(stringify(num)); + } + }, + buildBigint: bigint => out.next(`${bigint}n`), + buildString: outNextJSON, + buildSymbol: sym => { + assert.typeof(sym, 'symbol'); + const name = nameForPassableSymbol(sym); + if (name === undefined) { + throw Fail`Symbol must be either registered or well known: ${q(sym)}`; + } + const registeredName = Symbol.keyFor(sym); + if (registeredName === undefined) { + const match = AtAtPrefixPattern.exec(name); + assert(match !== null); + const suffix = match[1]; + assert(Symbol[suffix] === sym); + assert(identPattern.test(suffix)); + return out.next(`Symbol.${suffix}`); + } + return out.next(`Symbol.for(${stringify(registeredName)})`); + }, + + buildRecord: (names, buildValuesIter) => { + if (names.length === 0) { + return out.next('{}'); + } + out.open('{'); + const iter = buildValuesIter[Symbol.iterator](); + for (const name of names) { + out.line(); + if (name === '__proto__') { + // JavaScript interprets `{__proto__: x, ...}` + // as making an object inheriting from `x`, whereas + // in JSON it is simply a property name. Preserve the + // JSON meaning. + out.next(`["__proto__"]:`); + } else if (identPattern.test(name)) { + out.next(`${name}:`); + } else { + out.next(`${stringify(name)}:`); + } + const { value: _, done } = iter.next(); + if (done) { + break; + } + out.next(','); + } + return out.close('}'); + }, + buildArray: (count, buildElementsIter) => { + if (count === 0) { + return out.next('[]'); + } + out.open('['); + const iter = buildElementsIter[Symbol.iterator](); + for (let i = 0; ; i += 1) { + if (i < count) { + out.line(); + } + const { value: _, done } = iter.next(); + if (done) { + break; + } + out.next(','); + } + return out.close(']'); + }, + buildTagged: (tagName, buildPayloadFn) => { + out.next(`makeTagged(${stringify(tagName)},`); + buildPayloadFn(); + return out.next(')'); + }, + + buildError: error => out.next(`${error.name}(${stringify(error.message)})`), + buildRemotable: remotable => { + slotIndex += 1; + return out.next( + `slot(${slotIndex},${stringify(getInterfaceOf(remotable))})`, + ); + }, + buildPromise: _promise => { + slotIndex += 1; + return out.next(`slot(${slotIndex})`); + }, + }); + return justinBuilder; +}; +harden(makeJustinBuilder); + +export const makeBuilder = () => makeJustinBuilder(); +harden(makeBuilder); diff --git a/packages/marshal/src/builders/smallcapsBuilder.js b/packages/marshal/src/builders/smallcapsBuilder.js index 0f77412508..36a2417f91 100644 --- a/packages/marshal/src/builders/smallcapsBuilder.js +++ b/packages/marshal/src/builders/smallcapsBuilder.js @@ -5,6 +5,7 @@ import { Far, getErrorConstructor, hasOwnPropertyOf, + mapIterable, nameForPassableSymbol, passableSymbolForName, } from '@endo/pass-style'; @@ -15,7 +16,7 @@ import { /** @typedef {import('../encodeToSmallcaps.js').SmallcapsEncoding} SmallcapsEncoding */ -const { is, fromEntries, entries } = Object; +const { is, fromEntries } = Object; const { isArray } = Array; const { ownKeys } = Reflect; const { quote: q, details: X, Fail } = assert; @@ -23,7 +24,7 @@ const { quote: q, details: X, Fail } = assert; const makeSmallcapsBuilder = () => { /** @type {Builder} */ const smallcapsBuilder = Far('SmallcapsBuilder', { - buildRoot: node => node, + buildRoot: buildTopFn => buildTopFn(), buildUndefined: () => '#undefined', buildNull: () => null, @@ -50,19 +51,17 @@ const makeSmallcapsBuilder = () => { const name = /** @type {string} */ (nameForPassableSymbol(sym)); return `%${name}`; }, - buildRecord: builtEntries => + buildRecord: (names, buildValuesIter) => { + const builtValues = [...buildValuesIter]; + assert(names.length === builtValues.length); // TODO Should be fromUniqueEntries, but utils needs to be // relocated first. - fromEntries( - builtEntries.map(([name, builtValue]) => [ - buildString(name), - builtValue, - ]), - ), - buildArray: builtElements => builtElements, - buildTagged: (tagName, builtPayload) => ({ + return fromEntries(names.map((name, i) => [name, builtValues[i]])); + }, + buildArray: (_count, buildElementsIter) => harden([...buildElementsIter]), + buildTagged: (tagName, buildPayloadFn) => ({ '#tag': buildString(tagName), - payload: builtPayload, + payload: buildPayloadFn(), }), // TODO slots and options and all that. Also errorId @@ -96,7 +95,12 @@ const recognizeString = str => { }; const makeSmallcapsRecognizer = () => { - const smallcapsRecognizer = (encoding, builder) => { + /** + * @param {SmallcapsEncoding} encoding + * @param {Builder} builder + * @returns {SmallcapsEncoding} + */ + const recognizeNode = (encoding, builder) => { switch (typeof encoding) { case 'boolean': { return builder.buildBoolean(encoding); @@ -115,9 +119,9 @@ const makeSmallcapsRecognizer = () => { return builder.buildString(encoding.slice(1)); } case '%': { - return builder.buildSymbol( - passableSymbolForName(encoding.slice(1)), - ); + const sym = passableSymbolForName(encoding.slice(1)); + assert(sym !== undefined); + return builder.buildSymbol(sym); } case '#': { switch (encoding) { @@ -134,7 +138,10 @@ const makeSmallcapsRecognizer = () => { return builder.buildNumber(-Infinity); } default: { - assert.fail(X`unknown constant "${q(encoding)}"`, TypeError); + throw assert.fail( + X`unknown constant "${q(encoding)}"`, + TypeError, + ); } } } @@ -163,20 +170,18 @@ const makeSmallcapsRecognizer = () => { } if (isArray(encoding)) { - const builtElements = encoding.map(val => - smallcapsRecognizer(val, builder), + const buildElementsIter = mapIterable(encoding, val => + recognizeNode(val, builder), ); - return builder.buildArray(builtElements); + return builder.buildArray(encoding.length, buildElementsIter); } if (hasOwnPropertyOf(encoding, '#tag')) { const { '#tag': tag, payload, ...rest } = encoding; ownKeys(rest).length === 0 || Fail`#tag record unexpected properties: ${q(ownKeys(rest))}`; - return builder.buildTagged( - recognizeString(tag), - smallcapsRecognizer(payload, builder), - ); + const buildPayloadFn = () => recognizeNode(payload, builder); + return builder.buildTagged(recognizeString(tag), buildPayloadFn); } if (hasOwnPropertyOf(encoding, '#error')) { @@ -190,21 +195,25 @@ const makeSmallcapsRecognizer = () => { return builder.buildError(error); } - const buildEntry = ([encodedName, encodedVal]) => { + const encodedNames = /** @type {string[]} */ (ownKeys(encoding)).sort(); + for (const encodedName of encodedNames) { typeof encodedName === 'string' || Fail`Property name ${q( encodedName, )} of ${encoding} must be a string`; !encodedName.startsWith('#') || Fail`Unrecognized record type ${q(encodedName)}: ${encoding}`; - const name = recognizeString(encodedName); - return [name, smallcapsRecognizer(encodedVal, builder)]; - }; - const builtEntries = entries(encoding).map(buildEntry); - return builder.buildRecord(builtEntries); + } + const buildValuesIter = mapIterable(encodedNames, encodedName => + recognizeNode(encoding[encodedName], builder), + ); + return builder.buildRecord( + encodedNames.map(recognizeString), + buildValuesIter, + ); } default: { - assert.fail( + throw assert.fail( X`internal: unrecognized JSON typeof ${q( typeof encoding, )}: ${encoding}`, @@ -213,7 +222,14 @@ const makeSmallcapsRecognizer = () => { } } }; - return smallcapsRecognizer; + /** + * @param {SmallcapsEncoding} encoding + * @param {Builder} builder + * @returns {SmallcapsEncoding} + */ + const recognizeSmallcaps = (encoding, builder) => + builder.buildRoot(() => recognizeNode(encoding, builder)); + return harden(recognizeSmallcaps); }; harden(makeSmallcapsRecognizer); diff --git a/packages/marshal/src/builders/subgraphBuilder.js b/packages/marshal/src/builders/subgraphBuilder.js index 075be8dd34..6bb1d7d2fe 100644 --- a/packages/marshal/src/builders/subgraphBuilder.js +++ b/packages/marshal/src/builders/subgraphBuilder.js @@ -1,6 +1,14 @@ /// -import { Far, getTag, makeTagged, passStyleOf } from '@endo/pass-style'; +import { + Far, + getTag, + makeTagged, + mapIterable, + passStyleOf, +} from '@endo/pass-style'; + +/** @typedef {import('@endo/pass-style').Passable} Passable */ const { fromEntries } = Object; const { ownKeys } = Reflect; @@ -9,9 +17,9 @@ const { quote: q, details: X } = assert; const makeSubgraphBuilder = () => { const ident = val => val; - /** @type {Builder} */ + /** @type {Builder} */ const subgraphBuilder = Far('SubgraphBuilder', { - buildRoot: ident, + buildRoot: buildTopFn => buildTopFn(), buildUndefined: () => undefined, buildNull: () => null, @@ -20,9 +28,15 @@ const makeSubgraphBuilder = () => { buildBigint: ident, buildString: ident, buildSymbol: ident, - buildRecord: builtEntries => harden(fromEntries(builtEntries)), - buildArray: ident, - buildTagged: (tagName, builtPayload) => makeTagged(tagName, builtPayload), + buildRecord: (names, buildValuesIter) => { + const builtValues = [...buildValuesIter]; + assert(names.length === builtValues.length); + const builtEntries = names.map((name, i) => [name, builtValues[i]]); + return harden(fromEntries(builtEntries)); + }, + buildArray: (_count, buildElementsIter) => harden([...buildElementsIter]), + buildTagged: (tagName, buildPayloadFn) => + makeTagged(tagName, buildPayloadFn()), buildError: ident, buildRemotable: ident, @@ -33,7 +47,12 @@ const makeSubgraphBuilder = () => { harden(makeSubgraphBuilder); const makeSubgraphRecognizer = () => { - const subgraphRecognizer = (passable, builder) => { + /** + * @param {Passable} passable + * @param {Builder} builder + * @returns {Passable} + */ + const recognizeNode = (passable, builder) => { // First we handle all primitives. Some can be represented directly as // JSON, and some must be encoded into smallcaps strings. const passStyle = passStyleOf(passable); @@ -60,26 +79,21 @@ const makeSubgraphRecognizer = () => { return builder.buildSymbol(passable); } case 'copyRecord': { - // copyRecord allows only string keys so this will - // work. - const names = ownKeys(passable).sort(); - return builder.buildRecord( - names.map(name => [ - name, - subgraphRecognizer(passable[name], builder), - ]), + const names = /** @type {string[]} */ (ownKeys(passable)).sort(); + const buildValuesIter = mapIterable(names, name => + recognizeNode(passable[name], builder), ); + return builder.buildRecord(names, buildValuesIter); } case 'copyArray': { - return builder.buildArray( - passable.map(el => subgraphRecognizer(el, builder)), + const buildElementsIter = mapIterable(passable, el => + recognizeNode(el, builder), ); + return builder.buildArray(passable.length, buildElementsIter); } case 'tagged': { - return builder.buildTagged( - getTag(passable), - subgraphRecognizer(passable.payload, builder), - ); + const buildPayloadFn = () => recognizeNode(passable.payload, builder); + return builder.buildTagged(getTag(passable), buildPayloadFn); } case 'remotable': { return builder.buildRemotable(passable); @@ -91,14 +105,21 @@ const makeSubgraphRecognizer = () => { return builder.buildError(passable); } default: { - assert.fail( + throw assert.fail( X`internal: Unrecognized passStyle ${q(passStyle)}`, TypeError, ); } } }; - return subgraphRecognizer; + /** + * @param {Passable} passable + * @param {Builder} builder + * @returns {Passable} + */ + const recognizeSubgraph = (passable, builder) => + builder.buildRoot(() => recognizeNode(passable, builder)); + return harden(recognizeSubgraph); }; harden(makeSubgraphRecognizer); diff --git a/packages/marshal/src/marshal-justin.js b/packages/marshal/src/marshal-justin.js index 9bb31553af..cbc23a319f 100644 --- a/packages/marshal/src/marshal-justin.js +++ b/packages/marshal/src/marshal-justin.js @@ -30,7 +30,7 @@ const { quote: q, details: X, Fail } = assert; * * @returns {Indenter} */ -const makeYesIndenter = () => { +export const makeYesIndenter = () => { const strings = []; let level = 0; let needSpace = false; @@ -85,7 +85,7 @@ const badPairPattern = /^(?:\w\w|<<|>>|\+\+|--|)$/; * * @returns {Indenter} */ -const makeNoIndenter = () => { +export const makeNoIndenter = () => { /** @type {string[]} */ const strings = []; return harden({ diff --git a/packages/marshal/test/builders/test-justin-builder.js b/packages/marshal/test/builders/test-justin-builder.js new file mode 100644 index 0000000000..cfd72456c6 --- /dev/null +++ b/packages/marshal/test/builders/test-justin-builder.js @@ -0,0 +1,57 @@ +import { test } from '../prepare-test-env-ava.js'; + +import * as js from '../../src/builders/subgraphBuilder.js'; +import { makeJustinBuilder } from '../../src/builders/justinBuilder.js'; +import { makeMarshal } from '../../src/marshal.js'; +import { decodeToJustin } from '../../src/marshal-justin.js'; +import { + fakeJustinCompartment, + justinPairs, +} from '../test-marshal-justin-builder.js'; + +// this only includes the tests that do not use liveSlots + +test('justin builder round trip pairs', t => { + const jsRecognizer = js.makeRecognizer(); + const { toCapData } = makeMarshal(undefined, undefined, { + // We're turning `errorTagging`` off only for the round trip tests, not in + // general. + errorTagging: 'off', + // TODO retire the old format in justin test cases + serializeBodyFormat: 'capdata', + }); + for (const [body, justinSrc, slots] of justinPairs) { + const c = fakeJustinCompartment(); + const encoding = JSON.parse(body); + const justinExpr = decodeToJustin(encoding, false, slots); + t.is(justinExpr, justinSrc); + const value = harden(c.evaluate(`(${justinExpr})`)); + const { body: newBody } = toCapData(value); + t.is(newBody, body); + + const justinBuilder = makeJustinBuilder(false, slots); + t.is(jsRecognizer(value, justinBuilder), justinExpr); + } +}); + +test('justin indented builder round trip pairs', t => { + const jsRecognizer = js.makeRecognizer(); + const { toCapData } = makeMarshal(undefined, undefined, { + // We're turning `errorTagging`` off only for the round trip tests, not in + // general. + errorTagging: 'off', + // TODO retire the old format in justin test cases + serializeBodyFormat: 'capdata', + }); + for (const [body, _, slots] of justinPairs) { + const c = fakeJustinCompartment(); + const encoding = JSON.parse(body); + const justinExpr = decodeToJustin(encoding, true, slots); + const value = harden(c.evaluate(`(${justinExpr})`)); + const { body: newBody } = toCapData(value); + t.is(newBody, body); + + const justinBuilder = makeJustinBuilder(true, slots); + t.is(jsRecognizer(value, justinBuilder), justinExpr); + } +}); diff --git a/packages/marshal/test/builders/test-smallcaps-builder.js b/packages/marshal/test/builders/test-smallcaps-builder.js index b27e8e0726..846845af4f 100644 --- a/packages/marshal/test/builders/test-smallcaps-builder.js +++ b/packages/marshal/test/builders/test-smallcaps-builder.js @@ -3,7 +3,7 @@ import { test } from '../prepare-test-env-ava.js'; import * as sc from '../../src/builders/smallcapsBuilder.js'; import * as js from '../../src/builders/subgraphBuilder.js'; import { roundTripPairs } from '../test-marshal-capdata.js'; -import { makeSmallcapsTestMarshal } from '../test-marshal-smallcaps.js'; +import { makeSmallcapsTestMarshal } from '../test-marshal-smallcaps-builder.js'; const { isFrozen } = Object; diff --git a/packages/marshal/test/test-marshal-justin-builder.js b/packages/marshal/test/test-marshal-justin-builder.js new file mode 100644 index 0000000000..a194dda641 --- /dev/null +++ b/packages/marshal/test/test-marshal-justin-builder.js @@ -0,0 +1,161 @@ +import { test } from './prepare-test-env-ava.js'; + +// eslint-disable-next-line import/order +import { Remotable, makeTagged } from '@endo/pass-style'; +import { makeMarshal } from '../src/marshal.js'; +import { decodeToJustin } from '../src/marshal-justin.js'; + +// this only includes the tests that do not use liveSlots + +/** + * Based on roundTripPairs from test-marshal-capdata.js + * + * A list of `[body, justinSrc]` pairs, where the body parses into + * an encoding that decodes to a Justin expression that evaluates to something + * that has the same encoding. + * + * @type {([string, string] | [string, string, unknown[]])[]} + */ +export const justinPairs = harden([ + // Justin is the same as the JSON encoding but without unnecessary quoting + ['[1,2]', '[1,2]'], + ['{"foo":1}', '{foo:1}'], + ['{"a":1,"b":2}', '{a:1,b:2}'], + ['{"a":1,"b":{"c":3}}', '{a:1,b:{c:3}}'], + ['true', 'true'], + ['1', '1'], + ['"abc"', '"abc"'], + ['null', 'null'], + + // Primitives not representable in JSON + ['{"@qclass":"undefined"}', 'undefined'], + ['{"@qclass":"NaN"}', 'NaN'], + ['{"@qclass":"Infinity"}', 'Infinity'], + ['{"@qclass":"-Infinity"}', '-Infinity'], + ['{"@qclass":"bigint","digits":"4"}', '4n'], + ['{"@qclass":"bigint","digits":"9007199254740993"}', '9007199254740993n'], + ['{"@qclass":"symbol","name":"@@asyncIterator"}', 'Symbol.asyncIterator'], + ['{"@qclass":"symbol","name":"@@match"}', 'Symbol.match'], + ['{"@qclass":"symbol","name":"foo"}', 'Symbol.for("foo")'], + ['{"@qclass":"symbol","name":"@@@@foo"}', 'Symbol.for("@@foo")'], + + // Arrays and objects + ['[{"@qclass":"undefined"}]', '[undefined]'], + ['{"foo":{"@qclass":"undefined"}}', '{foo:undefined}'], + ['{"@qclass":"error","message":"","name":"Error"}', 'Error("")'], + [ + '{"@qclass":"error","message":"msg","name":"ReferenceError"}', + 'ReferenceError("msg")', + ], + + // The one case where JSON is not a semantic subset of JS + ['{"__proto__":8}', '{["__proto__"]:8}'], + + // The Hilbert Hotel is always tricky + ['{"@qclass":"hilbert","original":8}', '{"@qclass":8}'], + ['{"@qclass":"hilbert","original":"@qclass"}', '{"@qclass":"@qclass"}'], + [ + '{"@qclass":"hilbert","original":{"@qclass":"hilbert","original":8}}', + '{"@qclass":{"@qclass":8}}', + ], + [ + '{"@qclass":"hilbert","original":{"@qclass":"hilbert","original":8,"rest":{"foo":"foo1"}},"rest":{"bar":{"@qclass":"hilbert","original":{"@qclass":"undefined"}}}}', + '{"@qclass":{"@qclass":8,foo:"foo1"},bar:{"@qclass":undefined}}', + ], + + // tagged + ['{"@qclass":"tagged","tag":"x","payload":8}', 'makeTagged("x",8)'], + [ + '{"@qclass":"tagged","tag":"x","payload":{"@qclass":"undefined"}}', + 'makeTagged("x",undefined)', + ], + + // TODO Make Justin Slots work + // // Slots + // [ + // '[{"@qclass":"slot","iface":"Alleged: for testing Justin","index":0}]', + // '[slot(0,"Alleged: for testing Justin")]', + // ], + // // More Slots + // [ + // '[{"@qclass":"slot","iface":"Alleged: for testing Justin","index":0},{"@qclass":"slot","iface":"Remotable","index":1}]', + // '[slotToVal("hello","Alleged: for testing Justin"),slotToVal(null,"Remotable")]', + // ['hello', null], + // ], + // // Tests https://github.com/endojs/endo/issues/1185 fix + // [ + // '[{"@qclass":"slot","iface":"Alleged: for testing Justin","index":0},{"@qclass":"slot","index":0}]', + // '[slot(0,"Alleged: for testing Justin"),slot(0)]', + // ], +]); + +export const fakeJustinCompartment = () => { + const slots = []; + const slotVals = new Map(); + const populateSlot = (index, iface) => { + assert.typeof(iface, 'string'); // Assumes not optional the first time + const r = Remotable(iface, undefined, { getIndex: () => index }); + const s = `s${index}`; + slotVals.set(s, r); + slots[index] = s; + return r; + }; + const slot = (index, iface = undefined) => { + if (slots[index] !== undefined) { + assert(iface === undefined); // Assumes backrefs omit iface + return slotVals.get(slots[index]); + } + return populateSlot(index, iface); + }; + const slotToVal = (s, iface = undefined) => { + if (slotVals.has(s)) { + assert(iface === undefined); // Assumes backrefs omit iface + return slotVals.get(s); + } + return populateSlot(slots.length, iface); + }; + return new Compartment({ slot, slotToVal, makeTagged }); +}; + +test('serialize decodeToJustin eval round trip pairs', t => { + const { toCapData } = makeMarshal(undefined, undefined, { + // We're turning `errorTagging`` off only for the round trip tests, not in + // general. + errorTagging: 'off', + // TODO make Justin work with smallcaps + serializeBodyFormat: 'capdata', + }); + for (const [body, justinSrc, slots] of justinPairs) { + const c = fakeJustinCompartment(); + const encoding = JSON.parse(body); + const justinExpr = decodeToJustin(encoding, false, slots); + t.is(justinExpr, justinSrc); + const value = harden(c.evaluate(`(${justinExpr})`)); + const { body: newBody } = toCapData(value); + t.is(newBody, body); + } +}); + +// Like "serialize decodeToJustin eval round trip pairs" but uses the indented +// representation *without* checking its specific whitespace decisions. +// Just checks that it has equivalent evaluation, and +// that the decoder passes the extra `level` balancing diagnostic in +// `makeYesIndenter`. +test('serialize decodeToJustin indented eval round trip', t => { + const { toCapData } = makeMarshal(undefined, undefined, { + // We're turning `errorTagging`` off only for the round trip tests, not in + // general. + errorTagging: 'off', + // TODO make Justin work with smallcaps + serializeBodyFormat: 'capdata', + }); + for (const [body, _, slots] of justinPairs) { + const c = fakeJustinCompartment(); + t.log(body); + const encoding = JSON.parse(body); + const justinExpr = decodeToJustin(encoding, true, slots); + const value = harden(c.evaluate(`(${justinExpr})`)); + const { body: newBody } = toCapData(value); + t.is(newBody, body); + } +}); diff --git a/packages/marshal/test/test-marshal-smallcaps-builder.js b/packages/marshal/test/test-marshal-smallcaps-builder.js new file mode 100644 index 0000000000..6d44ee0fc2 --- /dev/null +++ b/packages/marshal/test/test-marshal-smallcaps-builder.js @@ -0,0 +1,419 @@ +import { test } from './prepare-test-env-ava.js'; + +// eslint-disable-next-line import/order +import { Far, makeTagged, passStyleOf } from '@endo/pass-style'; +import { makeMarshal } from '../src/marshal.js'; + +import { roundTripPairs } from './test-marshal-capdata.js'; + +const { freeze, isFrozen, create, prototype: objectPrototype } = Object; + +// this only includes the tests that do not use liveSlots + +/** + * @param {import('../src/types.js').MakeMarshalOptions} [opts] + */ +export const makeSmallcapsTestMarshal = (opts = { errorTagging: 'off' }) => + makeMarshal(undefined, undefined, { + serializeBodyFormat: 'smallcaps', + marshalSaveError: _err => {}, + ...opts, + }); + +test('smallcaps serialize unserialize round trip half pairs', t => { + const { toCapData, fromCapData } = makeSmallcapsTestMarshal(); + for (const [plain, _] of roundTripPairs) { + const { body } = toCapData(plain); + const decoding = fromCapData({ body, slots: [] }); + t.deepEqual(decoding, plain); + t.assert(isFrozen(decoding)); + } +}); + +test('smallcaps serialize static data', t => { + const { toCapData } = makeSmallcapsTestMarshal(); + const ser = val => toCapData(val); + + // @ts-ignore `isFake` purposely omitted from type + if (!harden.isFake) { + t.throws(() => ser([1, 2]), { + message: /Cannot pass non-frozen objects like/, + }); + } + // -0 serialized as 0 + t.deepEqual(ser(0), { body: '#0', slots: [] }); + t.deepEqual(ser(-0), { body: '#0', slots: [] }); + t.deepEqual(ser(-0), ser(0)); + // unregistered symbols + t.throws(() => ser(Symbol('sym2')), { + // An anonymous symbol is not Passable + message: /Only registered symbols or well-known symbols are passable:/, + }); + + const cd = ser(harden([1, 2])); + t.is(isFrozen(cd), true); + t.is(isFrozen(cd.slots), true); +}); + +test('smallcaps unserialize static data', t => { + const { fromCapData } = makeSmallcapsTestMarshal(); + const uns = body => fromCapData({ body, slots: [] }); + + // should be frozen + const arr = uns('#[1,2]'); + t.truthy(isFrozen(arr)); + const a = uns('#{"b":{"c":{"d": []}}}'); + t.truthy(isFrozen(a)); + t.truthy(isFrozen(a.b)); + t.truthy(isFrozen(a.b.c)); + t.truthy(isFrozen(a.b.c.d)); +}); + +test('smallcaps serialize errors', t => { + const { toCapData } = makeSmallcapsTestMarshal(); + const ser = val => toCapData(val); + + t.deepEqual(ser(harden(Error())), { + body: '#{"#error":"","name":"Error"}', + slots: [], + }); + + t.deepEqual(ser(harden(ReferenceError('msg'))), { + body: '#{"#error":"msg","name":"ReferenceError"}', + slots: [], + }); + + t.deepEqual(ser(harden(ReferenceError('#msg'))), { + body: '#{"#error":"!#msg","name":"ReferenceError"}', + slots: [], + }); + + const myError = harden({ + __proto__: Error.prototype, + message: 'mine', + }); + t.deepEqual(ser(myError), { + body: '#{"#error":"mine","name":"Error"}', + slots: [], + }); + + // TODO Once https://github.com/Agoric/SES-shim/issues/579 is merged + // do a golden of the error notes generated by the following + + // Extra properties + const errExtra = Error('has extra properties'); + // @ts-ignore Check dynamic consequences of type violation + errExtra.foo = []; + freeze(errExtra); + t.assert(isFrozen(errExtra)); + // @ts-ignore `isFake` purposely omitted from type + if (!harden.isFake) { + // @ts-ignore Check dynamic consequences of type violation + t.falsy(isFrozen(errExtra.foo)); + } + t.deepEqual(ser(errExtra), { + body: '#{"#error":"has extra properties","name":"Error"}', + slots: [], + }); + // @ts-ignore `isFake` purposely omitted from type + if (!harden.isFake) { + // @ts-ignore Check dynamic consequences of type violation + t.falsy(isFrozen(errExtra.foo)); + } + + // Bad prototype and bad "message" property + const nonErrorProto1 = { __proto__: Error.prototype, name: 'included' }; + const nonError1 = { __proto__: nonErrorProto1, message: [] }; + t.deepEqual(ser(harden(nonError1)), { + body: '#{"#error":"","name":"included"}', + slots: [], + }); +}); + +test('smallcaps unserialize errors', t => { + const { fromCapData } = makeSmallcapsTestMarshal(); + const uns = body => fromCapData({ body, slots: [] }); + + const em1 = uns('#{"#error":"msg","name":"ReferenceError"}'); + t.truthy(em1 instanceof ReferenceError); + t.is(em1.message, 'msg'); + t.truthy(isFrozen(em1)); + + const em2 = uns('#{"#error":"msg2","name":"TypeError"}'); + t.truthy(em2 instanceof TypeError); + t.is(em2.message, 'msg2'); + + const em3 = uns('#{"#error":"msg3","name":"Unknown"}'); + t.truthy(em3 instanceof Error); + t.is(em3.message, 'msg3'); +}); + +test('smallcaps mal-formed @qclass', t => { + const { fromCapData } = makeSmallcapsTestMarshal(); + const uns = body => fromCapData({ body, slots: [] }); + t.throws(() => uns('#{"#foo": 0}'), { + message: 'Unrecognized record type "#foo": {"#foo":0}', + }); +}); + +test('smallcaps records', t => { + const fauxPresence = harden({}); + const { toCapData: ser, fromCapData: unser } = makeMarshal( + _val => 'slot', + _slot => fauxPresence, + { + errorTagging: 'off', + serializeBodyFormat: 'smallcaps', + }, + ); + + const emptyData = { body: '#{}', slots: [] }; + + function build(descriptors) { + const o = create(objectPrototype, descriptors); + return harden(o); + } + + const enumData = { enumerable: true, value: 'data' }; + const enumGetData = { enumerable: true, get: () => 0 }; + const enumGetFunc = { enumerable: true, get: () => () => 0 }; + const enumSet = { enumerable: true, set: () => undefined }; + const nonenumData = { enumerable: false, value: 3 }; + + const ERR_NOACCESSORS = { message: /must not be an accessor property:/ }; + const ERR_ONLYENUMERABLE = { message: /must be an enumerable property:/ }; + const ERR_REMOTABLE = { message: /cannot serialize Remotables/ }; + + // empty objects + + // @ts-ignore `isFake` purposely omitted from type + if (!harden.isFake) { + // rejected because it is not hardened + t.throws( + () => ser({}), + { message: /Cannot pass non-frozen objects/ }, + 'non-frozen data cannot be serialized', + ); + } + + // harden({}) + t.deepEqual(ser(build()), emptyData); + + const stringData = { body: '#{"enumData":"data"}', slots: [] }; + + // serialized data should roundtrip properly + t.deepEqual(unser(ser(harden({}))), {}); + t.deepEqual(unser(ser(harden({ enumData: 'data' }))), { enumData: 'data' }); + + // unserialized data can be serialized again + t.deepEqual(ser(unser(emptyData)), emptyData); + t.deepEqual(ser(unser(stringData)), stringData); + + // { key: data } + // all: pass-by-copy without warning + t.deepEqual(ser(build({ enumData })), { + body: '#{"enumData":"data"}', + slots: [], + }); + + // anything with an accessor is rejected + t.throws(() => ser(build({ enumGetData })), ERR_NOACCESSORS); + t.throws(() => ser(build({ enumGetData, enumData })), ERR_NOACCESSORS); + t.throws(() => ser(build({ enumSet })), ERR_NOACCESSORS); + t.throws(() => ser(build({ enumSet, enumData })), ERR_NOACCESSORS); + + // anything with non-enumerable properties is rejected + t.throws(() => ser(build({ nonenumData })), ERR_ONLYENUMERABLE); + t.throws(() => ser(build({ nonenumData, enumData })), ERR_ONLYENUMERABLE); + + // anything with a function-returning getter is treated as remotable + t.throws(() => ser(build({ enumGetFunc })), ERR_REMOTABLE); + t.throws(() => ser(build({ enumGetFunc, enumData })), ERR_REMOTABLE); + + const desertTopping = harden({ + '#tag': 'floor wax', + payload: 'what is it?', + '#error': 'old joke', + }); + const body = `#${JSON.stringify(desertTopping)}`; + t.throws(() => unser({ body, slots: [] }), { + message: '#tag record unexpected properties: ["#error"]', + }); +}); + +/** + * A test case to illustrate each of the encodings + * * `!` - escaped string + * * `+` - non-negative bigint + * * `-` - negative bigint + * * `#` - manifest constant + * * `%` - symbol + * * `$` - remotable + * * `&` - promise + */ +test('smallcaps encoding examples', t => { + const { toCapData, fromCapData } = makeMarshal( + val => val, + slot => slot, + { + errorTagging: 'off', + serializeBodyFormat: 'smallcaps', + }, + ); + + const assertSer = (val, body, slots, message) => + t.deepEqual(toCapData(val), { body, slots }, message); + + const assertRoundTrip = (val, body, slots, message) => { + assertSer(val, body, slots, message); + const val2 = fromCapData(harden({ body, slots })); + assertSer(val2, body, slots, message); + t.deepEqual(val, val2, message); + }; + + // Numbers + assertRoundTrip(0, '#0', [], 'zero'); + assertRoundTrip(500n, '#"+500"', [], 'bigint'); + assertRoundTrip(-400n, '#"-400"', [], '-bigint'); + + // Constants + assertRoundTrip(NaN, '#"#NaN"', [], 'NaN'); + assertRoundTrip(Infinity, '#"#Infinity"', [], 'Infinity'); + assertRoundTrip(-Infinity, '#"#-Infinity"', [], '-Infinity'); + assertRoundTrip(undefined, '#"#undefined"', [], 'undefined'); + + // Strings + assertRoundTrip('unescaped', '#"unescaped"', [], 'unescaped'); + assertRoundTrip('#escaped', `#"!#escaped"`, [], 'escaped #'); + assertRoundTrip('+escaped', `#"!+escaped"`, [], 'escaped +'); + assertRoundTrip('-escaped', `#"!-escaped"`, [], 'escaped -'); + assertRoundTrip('%escaped', `#"!%escaped"`, [], 'escaped %'); + + // Symbols + assertRoundTrip(Symbol.iterator, '#"%@@iterator"', [], 'well known symbol'); + assertRoundTrip(Symbol.for('foo'), '#"%foo"', [], 'reg symbol'); + assertRoundTrip( + Symbol.for('@@foo'), + '#"%@@@@foo"', + [], + 'reg symbol that looks well known', + ); + + // Remotables + const foo = Far('foo', {}); + const bar = Far('bar', { + [Symbol.toStringTag]: 'DebugName: Bart', + }); + assertRoundTrip(foo, '#"$0.Alleged: foo"', [foo], 'Remotable object'); + assertRoundTrip( + harden([foo, bar, foo, bar]), + '#["$0.Alleged: foo","$1.DebugName: Bart","$0","$1"]', + [foo, bar], + 'Only show iface once', + ); + + // Promises + const p = harden(Promise.resolve(null)); + assertRoundTrip(p, '#"&0"', [p], 'Promise'); + + // Arrays + assertRoundTrip(harden([1, 2n]), '#[1,"+2"]', [], 'array'); + + // Records + assertRoundTrip( + harden({ foo: 1, bar: 2n }), + '#{"bar":"+2","foo":1}', + [], + 'record', + ); + + // Tagged + const taggedFoo = makeTagged('foo', 'bar'); + assertRoundTrip(taggedFoo, '#{"#tag":"foo","payload":"bar"}', [], 'tagged'); + const taggedBangFoo = makeTagged('!foo', '!bar'); + assertRoundTrip( + taggedBangFoo, + '#{"#tag":"!!foo","payload":"!!bar"}', + [], + 'tagged', + ); + + // Error + const err1 = harden(URIError('bad uri')); + assertRoundTrip(err1, '#{"#error":"bad uri","name":"URIError"}', [], 'error'); + const err2 = harden(Error('#NaN')); + assertRoundTrip(err2, '#{"#error":"!#NaN","name":"Error"}', [], 'error'); + + // non-passable errors alone still serialize + const nonPassableErr = Error('foo'); + // @ts-expect-error this type error is what we're testing + nonPassableErr.extraProperty = 'something bad'; + harden(nonPassableErr); + t.throws(() => passStyleOf(nonPassableErr), { + message: + /Passed Error has extra unpassed properties {"extraProperty":{"value":"something bad","writable":.*,"enumerable":true,"configurable":.*}}/, + }); + assertSer( + nonPassableErr, + '#{"#error":"foo","name":"Error"}', + [], + 'non passable errors pass', + ); + // pseudo-errors + const pseudoErr1 = harden({ + __proto__: Error.prototype, + message: '$not real', + name: 'URIError', + }); + assertSer( + pseudoErr1, + '#{"#error":"!$not real","name":"URIError"}', + [], + 'pseudo error', + ); + const pseudoErr2 = harden({ + __proto__: Error.prototype, + message: '$not real', + name: '#MyError', + }); + assertSer( + pseudoErr2, + '#{"#error":"!$not real","name":"!#MyError"}', + [], + 'pseudo error', + ); + + // Hilbert record + assertRoundTrip( + harden({ + '#tag': 'what', + '#error': 'me', + '#huh': 'worry', + '': 'empty', + '%sym': 'not a symbol', + }), + '#{"":"empty","!#error":"me","!#huh":"worry","!#tag":"what","!%sym":"not a symbol"}', + [], + 'hilbert property names', + ); +}); + +test('smallcaps proto problems', t => { + const exampleAlice = Far('Alice', {}); + const { toCapData: toSmallcaps, fromCapData: fromSmallcaps } = makeMarshal( + _val => 'slot', + _slot => exampleAlice, + { + errorTagging: 'off', + serializeBodyFormat: 'smallcaps', + }, + ); + const wrongProto = harden({ ['__proto__']: exampleAlice }); + const wrongProtoSmallcaps = toSmallcaps(wrongProto); + t.deepEqual(wrongProtoSmallcaps, { + body: '#{"__proto__":"$0.Alleged: Alice"}', + slots: ['slot'], + }); + // Fails before https://github.com/endojs/endo/issues/1303 fix + t.deepEqual(fromSmallcaps(wrongProtoSmallcaps), wrongProto); +}); diff --git a/packages/marshal/test/test-marshal-smallcaps.js b/packages/marshal/test/test-marshal-smallcaps.js index 32eb882593..e150fc483a 100644 --- a/packages/marshal/test/test-marshal-smallcaps.js +++ b/packages/marshal/test/test-marshal-smallcaps.js @@ -13,7 +13,7 @@ const { freeze, isFrozen, create, prototype: objectPrototype } = Object; /** * @param {import('../src/types.js').MakeMarshalOptions} [opts] */ -export const makeSmallcapsTestMarshal = (opts = { errorTagging: 'off' }) => +const makeTestMarshal = (opts = { errorTagging: 'off' }) => makeMarshal(undefined, undefined, { serializeBodyFormat: 'smallcaps', marshalSaveError: _err => {}, @@ -21,18 +21,18 @@ export const makeSmallcapsTestMarshal = (opts = { errorTagging: 'off' }) => }); test('smallcaps serialize unserialize round trip half pairs', t => { - const { toCapData, fromCapData } = makeSmallcapsTestMarshal(); + const { serialize, unserialize } = makeTestMarshal(); for (const [plain, _] of roundTripPairs) { - const { body } = toCapData(plain); - const decoding = fromCapData({ body, slots: [] }); + const { body } = serialize(plain); + const decoding = unserialize({ body, slots: [] }); t.deepEqual(decoding, plain); t.assert(isFrozen(decoding)); } }); test('smallcaps serialize static data', t => { - const { toCapData } = makeSmallcapsTestMarshal(); - const ser = val => toCapData(val); + const { serialize } = makeTestMarshal(); + const ser = val => serialize(val); // @ts-ignore `isFake` purposely omitted from type if (!harden.isFake) { @@ -56,8 +56,8 @@ test('smallcaps serialize static data', t => { }); test('smallcaps unserialize static data', t => { - const { fromCapData } = makeSmallcapsTestMarshal(); - const uns = body => fromCapData({ body, slots: [] }); + const { unserialize } = makeTestMarshal(); + const uns = body => unserialize({ body, slots: [] }); // should be frozen const arr = uns('#[1,2]'); @@ -70,8 +70,8 @@ test('smallcaps unserialize static data', t => { }); test('smallcaps serialize errors', t => { - const { toCapData } = makeSmallcapsTestMarshal(); - const ser = val => toCapData(val); + const { serialize } = makeTestMarshal(); + const ser = val => serialize(val); t.deepEqual(ser(harden(Error())), { body: '#{"#error":"","name":"Error"}', @@ -131,8 +131,8 @@ test('smallcaps serialize errors', t => { }); test('smallcaps unserialize errors', t => { - const { fromCapData } = makeSmallcapsTestMarshal(); - const uns = body => fromCapData({ body, slots: [] }); + const { unserialize } = makeTestMarshal(); + const uns = body => unserialize({ body, slots: [] }); const em1 = uns('#{"#error":"msg","name":"ReferenceError"}'); t.truthy(em1 instanceof ReferenceError); @@ -149,8 +149,8 @@ test('smallcaps unserialize errors', t => { }); test('smallcaps mal-formed @qclass', t => { - const { fromCapData } = makeSmallcapsTestMarshal(); - const uns = body => fromCapData({ body, slots: [] }); + const { unserialize } = makeTestMarshal(); + const uns = body => unserialize({ body, slots: [] }); t.throws(() => uns('#{"#foo": 0}'), { message: 'Unrecognized record type "#foo": {"#foo":0}', }); @@ -158,7 +158,7 @@ test('smallcaps mal-formed @qclass', t => { test('smallcaps records', t => { const fauxPresence = harden({}); - const { toCapData: ser, fromCapData: unser } = makeMarshal( + const { serialize: ser, unserialize: unser } = makeMarshal( _val => 'slot', _slot => fauxPresence, { @@ -252,7 +252,7 @@ test('smallcaps records', t => { * * `&` - promise */ test('smallcaps encoding examples', t => { - const { toCapData, fromCapData } = makeMarshal( + const { serialize, unserialize } = makeMarshal( val => val, slot => slot, { @@ -262,11 +262,11 @@ test('smallcaps encoding examples', t => { ); const assertSer = (val, body, slots, message) => - t.deepEqual(toCapData(val), { body, slots }, message); + t.deepEqual(serialize(val), { body, slots }, message); const assertRoundTrip = (val, body, slots, message) => { assertSer(val, body, slots, message); - const val2 = fromCapData(harden({ body, slots })); + const val2 = unserialize(harden({ body, slots })); assertSer(val2, body, slots, message); t.deepEqual(val, val2, message); }; @@ -400,7 +400,7 @@ test('smallcaps encoding examples', t => { test('smallcaps proto problems', t => { const exampleAlice = Far('Alice', {}); - const { toCapData: toSmallcaps, fromCapData: fromSmallcaps } = makeMarshal( + const { serialize: toSmallcaps, unserialize: fromSmallcaps } = makeMarshal( _val => 'slot', _slot => exampleAlice, {