Multi-tab worker (#120)

* Refactor PGlite Worker to use an explicit worker process created by the user * Update README * Working multi tab with live.query() * Multi tab live.changes and live.incrementalQuery * Remove withExtensions static method * Addtional options to worker * Test for multi tab worker live queries * address feedback
electric-sql · Jul 31, 2024 · de4b87a · de4b87a · github-actions · Jul 31, 2024
1 parent dcb3773
commit de4b87a
Show file tree

Hide file tree

Showing 16 changed files with 1,309 additions and 321 deletions.
diff --git a/packages/pglite/README.md b/packages/pglite/README.md
@@ -318,12 +318,33 @@ The `.query<T>()` method can take a TypeScript type describing the expected shap
 
 ### Web Workers:
 
-It's likely that you will want to run PGlite in a Web Worker so that it doesn't block the main thread. To aid in this we provide a `PGliteWorker` with the same API as the core `PGlite` but it runs Postgres in a dedicated Web Worker. To use, import from the `/worker` export:
+It's likely that you will want to run PGlite in a Web Worker so that it doesn't block the main thread. To aid in this we provide a `PGliteWorker` with the same API as the core `PGlite` but it runs Postgres in a dedicated Web Worker.
+
+First you need to create a js file for your worker instance, initiate PGlite with the worker extension, and start it:
+
+```js
+// my-pglite-worker.js
+import { PGlite } from "@electric-sql/pglite";
+import { worker } from "@electric-sql/pglite/worker";
+
+worker({
+  async init() {
+    return new PGlite();
+  },
+});
+```
+
+Then connect the `PGliteWorker` to your new worker process:
 
 ```js
 import { PGliteWorker } from "@electric-sql/pglite/worker";
 
-const pg = new PGliteWorker('idb://my-database');
+const pg = new PGliteWorker(
+  new Worker(new URL("./my-pglite-worker.js", import.meta.url), {
+    type: "module",
+  })
+);
+
 await pg.exec(`
   CREATE TABLE IF NOT EXISTS test (
     id SERIAL PRIMARY KEY,

diff --git a/packages/pglite/examples/worker-process.js b/packages/pglite/examples/worker-process.js
@@ -0,0 +1,17 @@
+import { PGlite } from "../dist/index.js";
+import { worker } from "../dist/worker/index.js";
+import { vector } from "../dist/vector/index.js";
+
+worker({
+  async init() {
+    const pg = new PGlite({
+      extensions: {
+        vector,
+      },
+    });
+    // If you want run any specific setup code for the worker process, you can do it here.
+    return pg;
+  },
+});
+
+console.log("Worker process started");
diff --git a/packages/pglite/examples/worker.html b/packages/pglite/examples/worker.html
@@ -1,47 +1,84 @@
-<script type="module">
-  import { PGliteWorker } from "../dist/worker/index.js";
+<html>
+  <head>
+    <title>PGlite Worker Example</title>
+    <script type="module">
+      import { PGliteWorker } from "../dist/worker/index.js";
+      import { live } from "../dist/live/index.js";
+
+      console.log("Creating worker...");
+      const pg = new PGliteWorker(
+        new Worker(new URL("./worker-process.js", import.meta.url), {
+          type: "module",
+        }),
+        {
+          extensions: {
+            live,
+          },
+        }
+      );
 
-  console.log("Starting...");
-  // In-memory database:
-  const pg = new PGliteWorker();
-  // Or with IndexedDB:
-  // const pg = new PGliteWorker('idb://pgdata');
+      pg.onLeaderChange(() => {
+        if (pg.isLeader) {
+          document.title = "[leader] PGlite Worker Example";
+          const leader = document.getElementById("leader");
+          leader.textContent = "true";
+          leader.style.color = "green";
+        }
+      });
+
+      await pg.exec(`
+        CREATE EXTENSION IF NOT EXISTS vector;
+        CREATE TABLE IF NOT EXISTS test (
+          id SERIAL PRIMARY KEY,
+          data vector(3)
+        );
+      `);
 
-  console.log("Waiting for ready...");
-  await pg.waitReady;
+      async function insertData() {
+        const data = [
+          Math.random(),
+          Math.random(),
+          Math.random(),
+        ];
+        await pg.query(
+          `INSERT INTO test (data) VALUES ($1)`,
+          [JSON.stringify(data)]
+        );
+      }
 
-  console.log("Ready!");
+      async function clearData() {
+        await pg.query(`DELETE FROM test`);
+      }
+
+      const btnInsert = document.querySelector("#insert");
+      btnInsert.addEventListener("click", insertData);
 
-  console.log("Creating table...");
-  await pg.exec(`
-    CREATE TABLE IF NOT EXISTS test (
-      id SERIAL PRIMARY KEY,
-      name TEXT
-    );
-  `);
+      const btnClear = document.querySelector("#clear");
+      btnClear.addEventListener("click", clearData);
 
-  console.log("Inserting data...");
-  await pg.exec("INSERT INTO test (name) VALUES ('test');");
-
-  console.log("Selecting data...");
-  const res = await pg.exec(`
-    SELECT * FROM test;
-  `);
-
-  console.log(res);
-
-  // Transaction example:
-  console.log("Transaction example...");
-  await pg.transaction(async (tx) => {
-    await tx.exec("INSERT INTO test (name) VALUES ('test2');");
-    await tx.exec("INSERT INTO test (name) VALUES ('test3');");
-  });
-
-  console.log("Selecting data...");
-  const res2 = await pg.exec(`
-    SELECT * FROM test;
-  `);
-
-  console.log(res2);
-
-</script>
+      // pg.live.query(
+      pg.live.incrementalQuery(
+        `SELECT * FROM test`,
+        [],
+        'id',
+        (data) => {
+          const output = document.getElementById("output");
+          output.textContent = JSON.stringify(data.rows, null, 2);
+        }
+      );
+    </script>
+  </head>
+  <body>
+    <h1>PGlite Worker Example</h1>
+    <p>Leader: <span id="leader" style="color: red;">false</span></p>
+    <p>
+      <button id="insert">
+        Insert Data
+      </button>
+      <button id="clear">
+        Clear Data
+      </button>
+    </p>
+    <pre id="output"></pre>
+  </body>
+</html>
diff --git a/packages/pglite/package.json b/packages/pglite/package.json
@@ -59,7 +59,6 @@
     "ava": "^6.1.2",
     "buffer": "^6.0.3",
     "bun": "^1.1.18",
-    "comlink": "^4.4.1",
     "concurrently": "^8.2.2",
     "http-server": "^14.1.1",
     "pg-protocol": "^1.6.1",

diff --git a/packages/pglite/src/extensionUtils.ts b/packages/pglite/src/extensionUtils.ts
@@ -35,7 +35,7 @@ export async function loadExtensionBundle(
     const response = await fetch(bundlePath.toString());
     if (!response.ok || !response.body) {
       return null;
-    } else if (response.headers.get('Content-Encoding') === 'gzip') {
+    } else if (response.headers.get("Content-Encoding") === "gzip") {
       // Although the bundle is manually compressed, some servers will recognize
       // that and add a content-encoding header. Fetch will then automatically
       // decompress the response.

diff --git a/packages/pglite/src/interface.ts b/packages/pglite/src/interface.ts
@@ -32,6 +32,7 @@ export interface ExtensionSetupResult {
 export type ExtensionSetup = (
   pg: PGliteInterface,
   emscriptenOpts: any,
+  clientOnly?: boolean,
 ) => Promise<ExtensionSetupResult>;
 
 export interface Extension {