From cd2be4ec6927502aadd26d66ba3bda51bc8007d2 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 8 Jan 2025 23:41:27 +0100 Subject: [PATCH] Implement JSONB encoder in Dart --- sqlite3/lib/src/jsonb.dart | 183 +++++++++++++++++++++++++++++++++++ sqlite3/test/jsonb_test.dart | 102 +++++++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 sqlite3/lib/src/jsonb.dart create mode 100644 sqlite3/test/jsonb_test.dart diff --git a/sqlite3/lib/src/jsonb.dart b/sqlite3/lib/src/jsonb.dart new file mode 100644 index 0000000..d45a426 --- /dev/null +++ b/sqlite3/lib/src/jsonb.dart @@ -0,0 +1,183 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:typed_data/typed_buffers.dart'; + +const jsonb = _JsonbCodec(); + +final class _JsonbCodec extends Codec { + const _JsonbCodec(); + + @override + // TODO: implement decoder + Converter get decoder => throw UnimplementedError(); + + @override + Converter get encoder => const _JsonbEncoder(); +} + +enum _ElementType { + _null, + _true, + _false, + _int, + _int5, + _float, + _float5, + _text, + _textJ, + _text5, + _textraw, + _array, + _object, + _reserved13, + _reserved14, + _reserved15, +} + +final class _JsonbEncoder extends Converter { + const _JsonbEncoder(); + + @override + Uint8List convert(Object? input) { + final operation = _JsonbEncodingOperation()..write(input); + return operation._buffer.buffer + .asUint8List(operation._buffer.offsetInBytes, operation._buffer.length); + } +} + +final class _JsonbEncodingOperation { + final Uint8Buffer _buffer = Uint8Buffer(); + + void writeHeader(int payloadSize, _ElementType type) { + var firstByte = type.index; + if (payloadSize <= 11) { + _buffer.add((payloadSize << 4) | firstByte); + } else { + // We can encode the length as a 1, 2, 4 or 8 byte integer. Prefer the + // shortest. + switch (payloadSize.bitLength) { + case <= 8: + const prefix = 12 << 4; + _buffer + ..add(prefix | firstByte) + ..add(payloadSize); + case <= 16: + const prefix = 13 << 4; + _buffer + ..add(prefix | firstByte) + ..add(payloadSize >> 8) + ..add(payloadSize); + case <= 32: + const prefix = 14 << 4; + _buffer + ..add(prefix | firstByte) + ..add(payloadSize >> 24) + ..add(payloadSize >> 16) + ..add(payloadSize >> 8) + ..add(payloadSize); + default: + const prefix = 15 << 4; + _buffer + ..add(prefix | firstByte) + ..add(payloadSize >> 56) + ..add(payloadSize >> 48) + ..add(payloadSize >> 40) + ..add(payloadSize >> 32) + ..add(payloadSize >> 24) + ..add(payloadSize >> 16) + ..add(payloadSize >> 8) + ..add(payloadSize); + } + } + } + + int prepareUnknownLength(_ElementType type) { + const prefix = 15 << 4; + _buffer.add(prefix | type.index); + final index = _buffer.length; + _buffer.addAll(_eightZeroes); + return index; + } + + void fillPreviouslyUnknownLength(int index) { + final length = _buffer.length - index - 8; + for (var i = 0; i < 8; i++) { + _buffer[index + i] = length >> (8 * (7 - i)); + } + } + + void writeNull() { + writeHeader(0, _ElementType._null); + } + + void writeBool(bool value) { + writeHeader(0, value ? _ElementType._true : _ElementType._false); + } + + void writeInt(int value) { + final encoded = utf8.encode(value.toString()); + writeHeader(encoded.length, _ElementType._int); + _buffer.addAll(encoded); + } + + void writeDouble(double value) { + final encoded = utf8.encode(value.toString()); + // RFC 8259 does not support infinity or NaN. + writeHeader(encoded.length, + value.isFinite ? _ElementType._float : _ElementType._float5); + _buffer.addAll(encoded); + } + + void writeString(String value) { + final encoded = _jsonUtf8.convert(value); + // Encoding a string adds quotes at the beginning and end which we don't + // need. + const doubleQuote = 0x22; + assert(encoded[0] == doubleQuote); + assert(encoded[encoded.length - 1] == doubleQuote); + + writeHeader(encoded.length - 2, _ElementType._textJ); + _buffer.addAll(encoded, 1, encoded.length - 1); + } + + void writeArray(Iterable values) { + if (values.isEmpty) { + return writeHeader(0, _ElementType._array); + } + + final index = prepareUnknownLength(_ElementType._array); + values.forEach(write); + fillPreviouslyUnknownLength(index); + } + + void writeObject(Map values) { + if (values.isEmpty) { + return writeHeader(0, _ElementType._object); + } + + final index = prepareUnknownLength(_ElementType._object); + for (final MapEntry(:key, :value) in values.entries) { + writeString(key); + write(value); + } + fillPreviouslyUnknownLength(index); + } + + void write(Object? value) { + return switch (value) { + null => writeNull(), + bool b => writeBool(b), + int i => writeInt(i), + double d => writeDouble(d), + String s => writeString(s), + Iterable i => writeArray(i), + Map o => writeObject(o), + Map o => writeObject(o.cast()), + _ => throw ArgumentError.value(value, 'value', 'Invalid JSON value.'), + }; + } + + static final _eightZeroes = Uint8List(8); + static final _jsonUtf8 = const JsonEncoder().fuse(const Utf8Encoder()); +} diff --git a/sqlite3/test/jsonb_test.dart b/sqlite3/test/jsonb_test.dart new file mode 100644 index 0000000..3168d17 --- /dev/null +++ b/sqlite3/test/jsonb_test.dart @@ -0,0 +1,102 @@ +@Tags(['ffi']) +library; + +import 'dart:convert'; + +import 'package:sqlite3/sqlite3.dart'; +import 'package:test/test.dart'; + +import 'package:sqlite3/src/jsonb.dart'; + +void main() { + group('encode', () { + void expectEncoded(Object? object, String expectedHex) { + final encoded = jsonb.encode(object); + final hex = + encoded.map((e) => e.toRadixString(16).padLeft(2, '0')).join(); + expect(hex, expectedHex); + } + + test('null', () { + expectEncoded(null, '00'); + }); + + test('booleans', () { + expectEncoded(true, '01'); + expectEncoded(false, '02'); + }); + + test('integers', () { + expectEncoded(0, '1330'); + expectEncoded(-1, '232d31'); + }); + + test('doubles', () { + expectEncoded(0.0, '35302e30'); + expectEncoded(-0.0, '452d302e30'); + }); + + test('array', () { + expectEncoded([], '0b'); + expectEncoded([true], 'fb000000000000000101'); + }); + + test('object', () { + expectEncoded({}, '0c'); + expectEncoded({'a': true}, 'fc0000000000000003186101'); + }); + }); + + group('round trips', () { + late Database database; + late PreparedStatement jsonb2json; + + setUpAll(() { + database = sqlite3.openInMemory(); + jsonb2json = database.prepare('SELECT json(?);'); + }); + + tearDownAll(() => database.dispose()); + + void check(Object? value, {String? expectDecodesAs}) { + // Check our encoder -> sqlite3 decoder + final sqliteDecoded = jsonb2json + .select([jsonb.encode(value)]) + .single + .values + .single as String; + if (expectDecodesAs != null) { + expect(sqliteDecoded, expectDecodesAs); + } else { + expect(json.decode(sqliteDecoded), value); + } + } + + test('primitives', () { + check(null); + check(true); + check(false); + check(0); + check(-1); + check(0.0); + check(double.infinity, expectDecodesAs: 'Infinity'); + check(double.negativeInfinity, expectDecodesAs: '-Infinity'); + check(double.nan, expectDecodesAs: 'NaN'); + check('hello world'); + check('hello " world'); + check('hello \n world'); + }); + + test('arrays', () { + check([]); + check([1, 2, 3]); + check([0, 1.1, 'hello', false, null, 'world']); + }); + + test('objects', () { + check({}); + check({'foo': 'bar'}); + check({'a': null, 'b': true, 'c': 0, 'd': 0.1, 'e': 'hi'}); + }); + }); +}