Enable tanglePrune

.eslintrc.js

@@ -13,7 +13,7 @@ module.exports = {
     SharedArrayBuffer: 'readonly',
   },
   parserOptions: {
-    ecmaVersion: 2018,
+    ecmaVersion: 2019,
     sourceType: 'module',
   },
   rules: {

lib/add-group-tangle.js

@@ -5,6 +5,7 @@
 const { isCloakedMsg } = require('ssb-ref')
 const set = require('lodash.set')
 const GetGroupTangle = require('./get-group-tangle')
+const tanglePrune = require('./tangle-prune')
 
 module.exports = function AddGroupTangle(server) {
   const getGroupTangle = GetGroupTangle(server)
@@ -26,39 +27,9 @@ module.exports = function AddGroupTangle(server) {
       if (err) return cb(null, content)
 
       set(content, 'tangles.group', tangle)
-      //TODO: uncomment
-      //tanglePrune(content) // prune the group tangle down if needed
+      tanglePrune(content) // prune the group tangle down if needed
 
       cb(null, content)
     })
   }
 }
-
-/* eslint-disable camelcase */
-const MAX_SIZE_16_recps = 5320
-const MAX_SIZE_1_recps = 5800
-
-function tanglePrune(content, tangle = 'group', maxSize) {
-  maxSize =
-    maxSize || (content.recps > 1 ? MAX_SIZE_16_recps : MAX_SIZE_1_recps)
-  if (getLength(content) <= maxSize) return content
-
-  content.tangles[tangle].previous = content.tangles[tangle].previous.sort(() =>
-    Math.random() < 0.5 ? -1 : +1
-  )
-  // we shuffle so that if multiple peers are also trying to converge,
-  // we hopefully tangle differently and converge faster
-
-  while (
-    content.tangles[tangle].previous.length &&
-    getLength(content) > maxSize
-  ) {
-    content.tangles[tangle].previous.pop()
-  }
-
-  return content
-}
-
-function getLength(obj) {
-  return JSON.stringify(obj).length
-}

lib/tangle-prune.js

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: 2022 Mix Irving
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+const bipf = require('bipf')
+
+/* eslint-disable camelcase */
+// these variables are calculated in
+// test/tangle-prune.test.js
+// if these variables are out of date and
+//   * smaller than supposed to: we'll prune a bit much, tangles will converge a bit slower
+//   * bigger than supposed to: we'll prune less than we can. users might run into 'the message you want to publish is too big' more often
+// but either way no catastrophe
+const MAX_SIZE_16_recps = 5546
+const MAX_SIZE_1_recps = 6041
+
+module.exports = function tanglePrune(content) {
+  const tangle = 'group'
+  const maxSize =
+    content.recps.length > 1 ? MAX_SIZE_16_recps : MAX_SIZE_1_recps
+  if (getLength(content) <= maxSize) return content
+
+  content.tangles[tangle].previous = content.tangles[tangle].previous.sort(() =>
+    Math.random() < 0.5 ? -1 : +1
+  )
+  // we shuffle so that if multiple peers are also trying to converge,
+  // we hopefully tangle differently and converge faster
+
+  while (
+    content.tangles[tangle].previous.length &&
+    getLength(content) > maxSize
+  ) {
+    content.tangles[tangle].previous.pop()
+  }
+
+  return content
+}
+
+function getLength(obj) {
+  return bipf.encodingLength(obj)
+}

package.json

@@ -28,6 +28,7 @@
     "*.js"
   ],
   "dependencies": {
+    "bipf": "^1.9.0",
     "envelope-js": "^1.3.2",
     "envelope-spec": "^1.1.0",
     "lodash.get": "^4.4.2",
@@ -49,7 +50,7 @@
     "pretty-quick": "^3.1.3",
     "secret-stack": "^6.4.1",
     "ssb-caps": "^1.1.0",
-    "ssb-db2": "^6.2.0",
+    "ssb-db2": "^6.2.3",
     "ssb-ebt": "^9.1.2",
     "tap-arc": "^0.3.4",
     "tape": "^5.5.3"

test/tangle-prune.test.js

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: 2022 Mix Irving
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+const test = require('tape')
+const bipf = require('bipf')
+const Testbot = require('./helpers/testbot')
+const tanglePrune = require('../lib/tangle-prune')
+
+const chars = 'abcABC123=+? '.split('')
+const encodedLength = (obj) => bipf.encodingLength(obj)
+const randomChar = () => chars.sort(() => (Math.random() < 0.5 ? -1 : +1))[0]
+const randomText = (length) => {
+  let output = ''
+  while (output.length < length) output += randomChar()
+  return output
+}
+
+test('tangle prune', async (t) => {
+  const ssb = Testbot()
+  const ssbId = ssb.id
+
+  const group = await ssb.tribes2.create()
+
+  const publishSize = async (size, recpCount = 1) => {
+    const content = {
+      type: 'post',
+      text: randomText(size),
+      recps: [group.id, ...new Array(recpCount - 1).fill(ssbId)],
+    }
+
+    return new Promise((resolve, reject) => {
+      ssb.tribes2.publish(content, (err, msg) => {
+        if (err) return resolve(false)
+
+        ssb.db.get(msg.key, (err, msgVal) => {
+          if (err) return reject(err)
+          const plainLength = encodedLength(msgVal.content)
+          resolve(plainLength)
+        })
+      })
+    })
+  }
+
+  async function findMaxSize(numberRecps = 1) {
+    // Apply bisection method to find max size of a message which can be published
+
+    let lower = 4000
+    let mid
+    let upper = 8000
+
+    const results = new Map([])
+
+    //let i = 0
+    while (upper - lower > 1) {
+      mid = Math.ceil((lower + upper) / 2)
+
+      if (!results.has(lower)) {
+        const res =
+          results.get(lower) || (await publishSize(lower, numberRecps))
+        results.set(lower, res)
+      }
+
+      if (!results.has(mid)) {
+        const res = results.get(mid) || (await publishSize(mid, numberRecps))
+        results.set(mid, res)
+      }
+      if (!results.has(upper)) {
+        const res =
+          results.get(upper) || (await publishSize(upper, numberRecps))
+        results.set(upper, res)
+      }
+
+      //console.log(i++, {
+      //  [lower]: results.get(lower),
+      //  [mid]: results.get(mid),
+      //  [upper]: results.get(upper),
+      //})
+
+      if (Boolean(results.get(lower)) !== Boolean(results.get(mid))) upper = mid
+      else if (Boolean(results.get(mid)) !== Boolean(results.get(upper)))
+        lower = mid
+      else throw new Error('bisection fail')
+    }
+
+    const result = results.get(upper) || results.get(mid) || results.get(lower)
+    t.pass(`max stringied content size for ${numberRecps} recps:  ${result}`)
+    return result
+  }
+  const max16recps = await findMaxSize(16).catch(t.error) // 5546
+  const max1recp = await findMaxSize(1).catch(t.error) // 6041
+  ssb.close()
+
+  const msgId = '%RDORgMCjmL6vs51nR4bn0LWNe6wkBfbRJulSdOJsmwg=.sha256'
+  const content = (prevCount, numRecps) => ({
+    type: 'post',
+    text: 'hello!',
+    recps: new Array(numRecps).fill(ssbId),
+    tangles: {
+      group: {
+        root: msgId,
+        previous: new Array(prevCount).fill(msgId),
+      },
+    },
+  })
+
+  //console.time('prune')
+  const result16 = tanglePrune(content(4000, 16))
+  //console.timeEnd('prune')
+  t.true(
+    encodedLength(result16) <= max16recps,
+    `pruned ${4000 - result16.tangles.group.previous.length}`
+  )
+
+  const result1 = tanglePrune(content(4000, 1))
+  t.true(
+    encodedLength(result1) <= max1recp,
+    `pruned ${4000 - result1.tangles.group.previous.length}`
+  )
+
+  t.end()
+})