diff --git a/.eslintrc.js b/.eslintrc.js index 0ab6a55..0fd415d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -13,7 +13,7 @@ module.exports = { SharedArrayBuffer: 'readonly', }, parserOptions: { - ecmaVersion: 2018, + ecmaVersion: 2019, sourceType: 'module', }, rules: { diff --git a/lib/add-group-tangle.js b/lib/add-group-tangle.js index 35c00a1..cdb36fb 100644 --- a/lib/add-group-tangle.js +++ b/lib/add-group-tangle.js @@ -5,6 +5,7 @@ const { isCloakedMsg } = require('ssb-ref') const set = require('lodash.set') const GetGroupTangle = require('./get-group-tangle') +const tanglePrune = require('./tangle-prune') module.exports = function AddGroupTangle(server) { const getGroupTangle = GetGroupTangle(server) @@ -26,39 +27,9 @@ module.exports = function AddGroupTangle(server) { if (err) return cb(null, content) set(content, 'tangles.group', tangle) - //TODO: uncomment - //tanglePrune(content) // prune the group tangle down if needed + tanglePrune(content) // prune the group tangle down if needed cb(null, content) }) } } - -/* eslint-disable camelcase */ -const MAX_SIZE_16_recps = 5320 -const MAX_SIZE_1_recps = 5800 - -function tanglePrune(content, tangle = 'group', maxSize) { - maxSize = - maxSize || (content.recps > 1 ? MAX_SIZE_16_recps : MAX_SIZE_1_recps) - if (getLength(content) <= maxSize) return content - - content.tangles[tangle].previous = content.tangles[tangle].previous.sort(() => - Math.random() < 0.5 ? -1 : +1 - ) - // we shuffle so that if multiple peers are also trying to converge, - // we hopefully tangle differently and converge faster - - while ( - content.tangles[tangle].previous.length && - getLength(content) > maxSize - ) { - content.tangles[tangle].previous.pop() - } - - return content -} - -function getLength(obj) { - return JSON.stringify(obj).length -} diff --git a/lib/tangle-prune.js b/lib/tangle-prune.js new file mode 100644 index 0000000..d183c99 --- /dev/null +++ b/lib/tangle-prune.js @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: LGPL-3.0-only + +const bipf = require('bipf') + +/* eslint-disable camelcase */ +// these variables are calculated in +// test/tangle-prune.test.js +// if these variables are out of date and +// * smaller than supposed to: we'll prune a bit much, tangles will converge a bit slower +// * bigger than supposed to: we'll prune less than we can. users might run into 'the message you want to publish is too big' more often +// but either way no catastrophe +const MAX_SIZE_16_recps = 5546 +const MAX_SIZE_1_recps = 6041 + +module.exports = function tanglePrune(content) { + const tangle = 'group' + const maxSize = + content.recps.length > 1 ? MAX_SIZE_16_recps : MAX_SIZE_1_recps + if (getLength(content) <= maxSize) return content + + content.tangles[tangle].previous = content.tangles[tangle].previous.sort(() => + Math.random() < 0.5 ? -1 : +1 + ) + // we shuffle so that if multiple peers are also trying to converge, + // we hopefully tangle differently and converge faster + + while ( + content.tangles[tangle].previous.length && + getLength(content) > maxSize + ) { + content.tangles[tangle].previous.pop() + } + + return content +} + +function getLength(obj) { + return bipf.encodingLength(obj) +} diff --git a/package.json b/package.json index bf8a061..b05a05f 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "*.js" ], "dependencies": { + "bipf": "^1.9.0", "envelope-js": "^1.3.2", "envelope-spec": "^1.1.0", "lodash.get": "^4.4.2", @@ -49,7 +50,7 @@ "pretty-quick": "^3.1.3", "secret-stack": "^6.4.1", "ssb-caps": "^1.1.0", - "ssb-db2": "^6.2.0", + "ssb-db2": "^6.2.3", "ssb-ebt": "^9.1.2", "tap-arc": "^0.3.4", "tape": "^5.5.3" diff --git a/test/tangle-prune.test.js b/test/tangle-prune.test.js new file mode 100644 index 0000000..4d51601 --- /dev/null +++ b/test/tangle-prune.test.js @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: LGPL-3.0-only + +const test = require('tape') +const bipf = require('bipf') +const Testbot = require('./helpers/testbot') +const tanglePrune = require('../lib/tangle-prune') + +const chars = 'abcABC123=+? '.split('') +const encodedLength = (obj) => bipf.encodingLength(obj) +const randomChar = () => chars.sort(() => (Math.random() < 0.5 ? -1 : +1))[0] +const randomText = (length) => { + let output = '' + while (output.length < length) output += randomChar() + return output +} + +test('tangle prune', async (t) => { + const ssb = Testbot() + const ssbId = ssb.id + + const group = await ssb.tribes2.create() + + const publishSize = async (size, recpCount = 1) => { + const content = { + type: 'post', + text: randomText(size), + recps: [group.id, ...new Array(recpCount - 1).fill(ssbId)], + } + + return new Promise((resolve, reject) => { + ssb.tribes2.publish(content, (err, msg) => { + if (err) return resolve(false) + + ssb.db.get(msg.key, (err, msgVal) => { + if (err) return reject(err) + const plainLength = encodedLength(msgVal.content) + resolve(plainLength) + }) + }) + }) + } + + async function findMaxSize(numberRecps = 1) { + // Apply bisection method to find max size of a message which can be published + + let lower = 4000 + let mid + let upper = 8000 + + const results = new Map([]) + + //let i = 0 + while (upper - lower > 1) { + mid = Math.ceil((lower + upper) / 2) + + if (!results.has(lower)) { + const res = + results.get(lower) || (await publishSize(lower, numberRecps)) + results.set(lower, res) + } + + if (!results.has(mid)) { + const res = results.get(mid) || (await publishSize(mid, numberRecps)) + results.set(mid, res) + } + if (!results.has(upper)) { + const res = + results.get(upper) || (await publishSize(upper, numberRecps)) + results.set(upper, res) + } + + //console.log(i++, { + // [lower]: results.get(lower), + // [mid]: results.get(mid), + // [upper]: results.get(upper), + //}) + + if (Boolean(results.get(lower)) !== Boolean(results.get(mid))) upper = mid + else if (Boolean(results.get(mid)) !== Boolean(results.get(upper))) + lower = mid + else throw new Error('bisection fail') + } + + const result = results.get(upper) || results.get(mid) || results.get(lower) + t.pass(`max stringied content size for ${numberRecps} recps: ${result}`) + return result + } + const max16recps = await findMaxSize(16).catch(t.error) // 5546 + const max1recp = await findMaxSize(1).catch(t.error) // 6041 + ssb.close() + + const msgId = '%RDORgMCjmL6vs51nR4bn0LWNe6wkBfbRJulSdOJsmwg=.sha256' + const content = (prevCount, numRecps) => ({ + type: 'post', + text: 'hello!', + recps: new Array(numRecps).fill(ssbId), + tangles: { + group: { + root: msgId, + previous: new Array(prevCount).fill(msgId), + }, + }, + }) + + //console.time('prune') + const result16 = tanglePrune(content(4000, 16)) + //console.timeEnd('prune') + t.true( + encodedLength(result16) <= max16recps, + `pruned ${4000 - result16.tangles.group.previous.length}` + ) + + const result1 = tanglePrune(content(4000, 1)) + t.true( + encodedLength(result1) <= max1recp, + `pruned ${4000 - result1.tangles.group.previous.length}` + ) + + t.end() +})