OPFS access handle pool VFS (#130)

* WIP: OPFS VFS using SharedArrayBuffer and Atomic

WIP

WIP

wip

Tidyup and fix

Seems to work...

Experiment with not closing files

* OPFS Access Handle Pool VFS

WIP

Flush experiment

relaxed durability mode and benchmarks

Update benchmarks

* Refactor

* Format and tweaks

* Exports

* Tests

* Comments

* Tidyup

packages/benchmark/benchmarks-rtt.js

@@ -15,6 +15,23 @@ const CONFIGURATIONS = new Map(
       db: "pglite",
       dataDir: "idb://benchmark-rtt",
     },
+    {
+      label: "PGlite IDB<br> <i>relaxed durability</i>",
+      db: "pglite",
+      dataDir: "idb://benchmark-rtt-rd",
+      options: { relaxedDurability: true },
+    },
+    {
+      label: "PGlite OPFS AHP",
+      db: "pglite",
+      dataDir: "opfs-ahp://benchmark-rtt",
+    },
+    {
+      label: "PGlite OPFS AHP<br> <i>relaxed durability</i>",
+      db: "pglite",
+      dataDir: "opfs-ahp://benchmark-rtt-rd",
+      options: { relaxedDurability: true },
+    },
     {
       label: 'SQLite Memory',
       db: 'wa-sqlite',
@@ -39,6 +56,14 @@ const CONFIGURATIONS = new Map(
       vfsClass: 'OriginPrivateFileSystemVFS',
       vfsArgs: []
     },
+    {
+      label: 'SQLite OPFS AHP',
+      db: 'wa-sqlite',
+      isAsync: false,
+      vfsModule: './node_modules/wa-sqlite/src/examples/AccessHandlePoolVFS.js',
+      vfsClass: 'AccessHandlePoolVFS',
+      vfsArgs: ['/benchmark-rtt-sqlite-ahp']
+    },
   ].map((obj) => [obj.label, obj])
 );
 
@@ -107,7 +132,11 @@ document.getElementById("start").addEventListener("click", async (event) => {
   // Remove OPFS
   const root = await navigator.storage.getDirectory();
   for await (const handle of root.values()) {
-    await root.removeEntry(handle.name, { recursive: true });
+    try {
+      await root.removeEntry(handle.name, { recursive: true });
+    } catch (e) {
+      // ignore
+    }
   }
 
   const Comlink = await ComlinkReady;
@@ -170,6 +199,6 @@ document.getElementById("start").addEventListener("click", async (event) => {
 function addEntry(parent, text) {
   const tag = parent.parentElement.tagName === "TBODY" ? "td" : "th";
   const child = document.createElement(tag);
-  child.textContent = text;
+  child.innerHTML = text;
   parent.appendChild(child);
 }

packages/benchmark/benchmarks.js

@@ -18,6 +18,24 @@ const CONFIGURATIONS = new Map(
       label: "Emscripten IndexedDB FS",
       dataDir: "idb://benchmark",
     },
+    {
+      label: "Emscripten IndexedDB FS<br> <i>relaxed durability</i>",
+      dataDir: "idb://benchmark-rd",
+      options: { relaxedDurability: true },
+    },
+    {
+      label: "OPFS Access Handle Pool",
+      dataDir: "opfs-ahp://benchmark-ahp",
+    },
+    {
+      label: "OPFS Access Handle Pool<br> <i>relaxed durability</i>",
+      dataDir: "opfs-ahp://benchmark-ahp-rd",
+      options: { relaxedDurability: true },
+    },
+    // {
+    //   label: "OPFS Worker",
+    //   dataDir: "opfs-worker://benchmark-worker",
+    // },
   ].map((obj) => [obj.label, obj])
 );
 
@@ -77,6 +95,16 @@ document.getElementById("start").addEventListener("click", async (event) => {
     }
   });
 
+  // Remove OPFS
+  const root = await navigator.storage.getDirectory();
+  for await (const handle of root.values()) {
+    try {
+      await root.removeEntry(handle.name, { recursive: true });
+    } catch (e) {
+      // ignore
+    }
+  }
+
   const benchmarks = await benchmarksReady;
   const Comlink = await ComlinkReady;
   try {
@@ -101,6 +129,7 @@ document.getElementById("start").addEventListener("click", async (event) => {
         const query = await workerProxy({
           dataDir: config.dataDir,
           label: config.label,
+          options: config.options,
         });
 
         await query(preamble);
@@ -123,6 +152,7 @@ document.getElementById("start").addEventListener("click", async (event) => {
     document.getElementById("error").textContent = e.stack.includes(e.message)
       ? e.stack
       : `${e.stack}\n${e.message}`;
+    throw e;
   } finally {
     // @ts-ignore
     event.target.disabled = false;
@@ -132,6 +162,6 @@ document.getElementById("start").addEventListener("click", async (event) => {
 function addEntry(parent, text) {
   const tag = parent.parentElement.tagName === "TBODY" ? "td" : "th";
   const child = document.createElement(tag);
-  child.textContent = text;
+  child.innerHTML = text;
   parent.appendChild(child);
 }

packages/benchmark/demo-worker.js

@@ -14,7 +14,20 @@ import { PGlite } from "../pglite/dist/index.js";
    * @returns {Promise<Function>}
    */
   async function open(config) {
-    const pg = new PGlite(config.dataDir);
+    if (config.dataDir.startsWith("opfs-")) {
+      // delete the existing database
+      const root = await navigator.storage.getDirectory();
+      const dirName = config.dataDir.slice(config.dataDir.indexOf("://") + 3);
+      try {
+        const dir = await root.getDirectoryHandle(dirName, { create: false });
+        await dir.remove();
+      } catch (e) {
+        // ignore
+      }
+    }
+
+    console.log("Opening PGLite database:", JSON.stringify(config, null, 2));
+    const pg = new PGlite(config.dataDir, config.options);
     await pg.waitReady;
 
     // Create the query interface.

packages/benchmark/package.json

@@ -18,6 +18,6 @@
     "tsx": "^4.7.1"
   },
   "dependencies": {
-    "wa-sqlite": "github:rhashimoto/wa-sqlite"
+    "wa-sqlite": "github:rhashimoto/wa-sqlite#v0.9.14"
   }
 }

packages/benchmark/rtt-demo-worker.js

@@ -4,7 +4,7 @@
 
 import * as SQLite from './node_modules/wa-sqlite/src/sqlite-api.js';
 import { createTag } from "./node_modules/wa-sqlite/src/examples/tag.js";
-import { PGlite } from "../pglite/dist/index.js";
+import { PGlite } from "../pglite/dist/index.js?1";
 
 const WA_SQLITE = './node_modules/wa-sqlite/dist/wa-sqlite.mjs';
 const WA_SQLITE_ASYNC = './node_modules/wa-sqlite/dist/wa-sqlite-async.mjs';
@@ -15,18 +15,17 @@ const WA_SQLITE_ASYNC = './node_modules/wa-sqlite/dist/wa-sqlite-async.mjs';
   );
 
   async function open(config) {
-    console.log('Opening database:', config)
     if (config.db === 'wa-sqlite') {
-      console.log('Opening SQLite database:', config)
+      console.log('Opening SQLite database:', JSON.stringify(config, null, 2))
       return openSQLite(config);
     } else if (config.db === 'pglite') {
-      console.log('Opening PGLite database:', config)
+      console.log('Opening PGLite database:', JSON.stringify(config, null, 2))
       return openPGlite(config);
     }
   }
 
   async function openPGlite(config) {
-    const pg = new PGlite(config.dataDir);
+    const pg = new PGlite(config.dataDir, config.options);
     await pg.waitReady;
 
     // Create the query interface.

packages/pglite/examples/opfs-worker.js

@@ -0,0 +1,12 @@
+import { PGlite } from "../dist/index.js";
+import { worker } from "../dist/worker/index.js";
+
+worker({
+  async init() {
+    const pg = new PGlite("opfs-ahp://my-test-db2");
+    // If you want run any specific setup code for the worker process, you can do it here.
+    return pg;
+  },
+});
+
+console.log("Worker process started");

packages/pglite/examples/opfs.html

@@ -0,0 +1,49 @@
+<script type="module">
+  import { PGliteWorker } from "../dist/worker/index.js";
+
+  console.log("Starting...");
+  const start = performance.now();
+  const pg = new PGliteWorker(
+    new Worker(new URL("./opfs-worker.js", import.meta.url), {
+      type: "module",
+    })
+  );
+
+  console.log("Waiting for ready...");
+  await pg.waitReady;
+
+  console.log("Ready! Took", performance.now() - start, "ms");
+
+  console.log("Creating table...");
+  await pg.exec(`
+    CREATE TABLE IF NOT EXISTS test (
+      id SERIAL PRIMARY KEY,
+      name TEXT
+    );
+  `);
+
+  console.log("Inserting data...");
+  await pg.exec("INSERT INTO test (name) VALUES ('test');");
+
+  console.log("Selecting data...");
+  const res = await pg.exec(`
+    SELECT * FROM test;
+  `);
+
+  console.log(res);
+
+  // Transaction example:
+  console.log("Transaction example...");
+  await pg.transaction(async (tx) => {
+    await tx.exec("INSERT INTO test (name) VALUES ('test2');");
+    await tx.exec("INSERT INTO test (name) VALUES ('test3');");
+  });
+
+  console.log("Selecting data...");
+  const res2 = await pg.exec(`
+    SELECT * FROM test;
+  `);
+
+  console.log(res2);
+
+</script>

packages/pglite/package.json

@@ -21,15 +21,26 @@
     ".": "./dist/index.js",
     "./live": "./dist/live/index.js",
     "./worker": "./dist/worker/index.js",
-    "./vector": "./dist/vector/index.js"
+    "./vector": "./dist/vector/index.js",
+    "./nodefs": "./dist/fs/nodefs.js",
+    "./opfs-ahp": "./dist/fs/opfs-ahp/index.js"
   },
   "typesVersions": {
     "*": {
+      "live": [
+        "./dist/live/index.d.ts"
+      ],
       "worker": [
         "./dist/worker/index.d.ts"
       ],
       "vector": [
         "./dist/vector/index.d.ts"
+      ],
+      "nodefs": [
+        "./dist/fs/nodefs.d.ts"
+      ],
+      "opfs-ahp": [
+        "./dist/fs/opfs-ahp/index.d.ts"
       ]
     }
   },
@@ -49,6 +60,8 @@
     "test:bun": "rm -rf ./pgdata-test && npx bun test tests/basic.test.js && npx bun test tests/pgvector.test.js && npx bun test tests/targets/node-fs.test.js",
     "build:js": "tsup && tsx scripts/bundle-wasm.ts",
     "build": "npm run build:js",
+    "dev": "concurrently \"tsup --watch\" \"sleep 1 && tsx scripts/bundle-wasm.ts\" \"pnpm dev-server\"",
+    "dev-server": "pnpm http-server ../",
     "format": "prettier --write ./src"
   },
   "devDependencies": {
@@ -64,8 +77,9 @@
     "pg-protocol": "^1.6.1",
     "playwright": "^1.42.1",
     "prettier": "3.2.5",
+    "serve": "^14.2.3",
     "tinytar": "^0.1.0",
-    "tsup": "^8.0.2",
+    "tsup": "^8.1.0",
     "tsx": "^4.7.1",
     "typescript": "^5.3.3"
   }

packages/pglite/src/fs/idbfs.ts

@@ -38,7 +38,7 @@ export class IdbFs extends FilesystemBase {
     });
   }
 
-  syncToFs(fs: FS) {
+  syncToFs(fs: FS, relaxedDurability?: boolean) {
     return new Promise<void>((resolve, reject) => {
       fs.syncfs(false, (err: any) => {
         if (err) {

packages/pglite/src/fs/index.ts

@@ -20,6 +20,10 @@ export function parseDataDir(dataDir?: string) {
     // Remove the idb:// prefix, and use indexeddb filesystem
     dataDir = dataDir.slice(6);
     fsType = "idbfs";
+  } else if (dataDir?.startsWith("opfs-ahp://")) {
+    // Remove the opfsahp:// prefix, and use opfs access handle pool filesystem
+    dataDir = dataDir.slice(11);
+    fsType = "opfs-ahp";
   } else if (!dataDir || dataDir?.startsWith("memory://")) {
     // Use in-memory filesystem
     fsType = "memoryfs";
@@ -38,6 +42,10 @@ export async function loadFs(dataDir?: string, fsType?: FsType) {
     fs = new NodeFS(dataDir);
   } else if (dataDir && fsType === "idbfs") {
     fs = new IdbFs(dataDir);
+  } else if (dataDir && fsType === "opfs-ahp") {
+    // Lazy load the opfs-ahp to so that it's optional in the bundle
+    const { OpfsAhpFS } = await import("./opfs-ahp/index.js");
+    fs = new OpfsAhpFS(dataDir);
   } else {
     fs = new MemoryFS();
   }