Dump and load tarball of a database (#116)

* WIP dump of data dir to a tar.gz

* Dump and loading of a datadir to a tarball + tests

* Swap to using File object for dump/load

packages/pglite/examples/dumpDataDir.html

@@ -0,0 +1 @@
+<script type="module" src="./dumpDataDir.js"></script>

packages/pglite/examples/dumpDataDir.js

@@ -0,0 +1,32 @@
+import { PGlite } from "../dist/index.js";
+
+const pg = new PGlite();
+await pg.exec(`
+  CREATE TABLE IF NOT EXISTS test (
+    id SERIAL PRIMARY KEY,
+    name TEXT
+  );
+`);
+await pg.exec("INSERT INTO test (name) VALUES ('test');");
+
+const file = await pg.dumpDataDir();
+
+if (typeof window !== "undefined") {
+  // Download the dump
+  const url = URL.createObjectURL(file);
+  const a = document.createElement("a");
+  a.href = url;
+  a.download = file.name;
+  a.click();
+} else {
+  // Save the dump to a file using node fs
+  const fs = await import("fs");
+  fs.writeFileSync(file.name, await file.arrayBuffer());
+}
+
+const pg2 = new PGlite({
+  loadDataDir: file,
+});
+
+const rows = await pg2.query("SELECT * FROM test;");
+console.log(rows);

packages/pglite/src/definitions/tinytar.d.ts

@@ -34,6 +34,8 @@ declare module "tinytar" {
   const NULL_CHAR: string;
   const TMAGIC: string;
   const OLDGNU_MAGIC: string;
+
+  // Values used in typeflag field
   const REGTYPE: number;
   const LNKTYPE: number;
   const SYMTYPE: number;
@@ -42,6 +44,8 @@ declare module "tinytar" {
   const DIRTYPE: number;
   const FIFOTYPE: number;
   const CONTTYPE: number;
+
+  // Bits used in the mode field, values in octal
   const TSUID: number;
   const TSGID: number;
   const TSVTX: number;
@@ -54,6 +58,7 @@ declare module "tinytar" {
   const TOREAD: number;
   const TOWRITE: number;
   const TOEXEC: number;
+
   const TPERMALL: number;
   const TPERMMASK: number;
 }

packages/pglite/src/fs/idbfs.ts

@@ -1,6 +1,7 @@
 import { FilesystemBase } from "./types.js";
 import type { FS, PostgresMod } from "../postgres.js";
 import { PGDATA } from "./index.js";
+import { dumpTar } from "./tarUtils.js";
 
 export class IdbFs extends FilesystemBase {
   async emscriptenOpts(opts: Partial<PostgresMod>) {
@@ -48,4 +49,8 @@ export class IdbFs extends FilesystemBase {
       });
     });
   }
+
+  async dumpTar(mod: FS, dbname: string) {
+    return dumpTar(mod, dbname);
+  }
 }

packages/pglite/src/fs/memoryfs.ts

@@ -1,9 +1,14 @@
 import { FilesystemBase } from "./types.js";
-import type { PostgresMod } from "../postgres.js";
+import type { PostgresMod, FS } from "../postgres.js";
+import { dumpTar } from "./tarUtils.js";
 
 export class MemoryFS extends FilesystemBase {
   async emscriptenOpts(opts: Partial<PostgresMod>) {
     // Nothing to do for memoryfs
     return opts;
   }
+
+  async dumpTar(mod: FS, dbname: string) {
+    return dumpTar(mod, dbname);
+  }
 }

packages/pglite/src/fs/nodefs.ts

@@ -2,7 +2,8 @@ import * as fs from "fs";
 import * as path from "path";
 import { FilesystemBase } from "./types.js";
 import { PGDATA } from "./index.js";
-import type { PostgresMod } from "../postgres.js";
+import type { PostgresMod, FS } from "../postgres.js";
+import { dumpTar } from "./tarUtils.js";
 
 export class NodeFS extends FilesystemBase {
   protected rootDir: string;
@@ -29,4 +30,8 @@ export class NodeFS extends FilesystemBase {
     };
     return options;
   }
+
+  async dumpTar(mod: FS, dbname: string) {
+    return dumpTar(mod, dbname);
+  }
 }

packages/pglite/src/fs/tarUtils.ts

@@ -0,0 +1,203 @@
+import { tar, untar, type TarFile, REGTYPE, DIRTYPE } from "tinytar";
+import { FS } from "../postgres.js";
+import { PGDATA } from "./index.js";
+
+export async function dumpTar(FS: FS, dbname?: string): Promise<File> {
+  const tarball = createTarball(FS, PGDATA);
+  const [compressed, zipped] = await maybeZip(tarball);
+  const filename = (dbname || "pgdata") + (zipped ? ".tar.gz" : ".tar");
+  return new File([compressed], filename, {
+    type: zipped ? "application/x-gtar" : "application/x-tar",
+  });
+}
+
+const compressedMimeTypes = [
+  "application/x-gtar",
+  "application/x-tar+gzip",
+  "application/x-gzip",
+  "application/gzip",
+];
+
+export async function loadTar(FS: FS, file: File | Blob): Promise<void> {
+  let tarball = new Uint8Array(await file.arrayBuffer());
+  const filename = file instanceof File ? file.name : undefined;
+  const compressed =
+    compressedMimeTypes.includes(file.type) ||
+    filename?.endsWith(".tgz") ||
+    filename?.endsWith(".tar.gz");
+  if (compressed) {
+    tarball = await unzip(tarball);
+  }
+
+  const files = untar(tarball);
+  for (const file of files) {
+    const filePath = PGDATA + file.name;
+
+    // Ensure the directory structure exists
+    const dirPath = filePath.split("/").slice(0, -1);
+    for (let i = 1; i <= dirPath.length; i++) {
+      const dir = dirPath.slice(0, i).join("/");
+      if (!FS.analyzePath(dir).exists) {
+        FS.mkdir(dir);
+      }
+    }
+
+    // Write the file or directory
+    if (file.type == REGTYPE) {
+      FS.writeFile(filePath, file.data);
+      FS.utime(
+        filePath,
+        dateToUnixTimestamp(file.modifyTime),
+        dateToUnixTimestamp(file.modifyTime),
+      );
+    } else if (file.type == DIRTYPE) {
+      FS.mkdir(filePath);
+    }
+  }
+}
+
+function readDirectory(FS: FS, path: string) {
+  let files: TarFile[] = [];
+
+  const traverseDirectory = (currentPath: string) => {
+    const entries = FS.readdir(currentPath);
+    entries.forEach((entry) => {
+      if (entry === "." || entry === "..") {
+        return;
+      }
+      const fullPath = currentPath + "/" + entry;
+      const stats = FS.stat(fullPath);
+      const data = FS.isFile(stats.mode)
+        ? FS.readFile(fullPath, { encoding: "binary" })
+        : new Uint8Array(0);
+      files.push({
+        name: fullPath.substring(path.length), // remove the root path
+        mode: stats.mode,
+        size: stats.size,
+        type: FS.isFile(stats.mode) ? REGTYPE : DIRTYPE,
+        modifyTime: stats.mtime,
+        data,
+      });
+      if (FS.isDir(stats.mode)) {
+        traverseDirectory(fullPath);
+      }
+    });
+  };
+
+  traverseDirectory(path);
+  return files;
+}
+
+export function createTarball(FS: FS, directoryPath: string) {
+  const files = readDirectory(FS, directoryPath);
+  const tarball = tar(files);
+  return tarball;
+}
+
+export async function maybeZip(
+  file: Uint8Array,
+): Promise<[Uint8Array, boolean]> {
+  if (typeof window !== "undefined" && "CompressionStream" in window) {
+    return [await zipBrowser(file), true];
+  } else if (
+    typeof process !== "undefined" &&
+    process.versions &&
+    process.versions.node
+  ) {
+    return [await zipNode(file), true];
+  } else {
+    return [file, false];
+  }
+}
+
+export async function zipBrowser(file: Uint8Array): Promise<Uint8Array> {
+  const cs = new CompressionStream("gzip");
+  const writer = cs.writable.getWriter();
+  const reader = cs.readable.getReader();
+
+  writer.write(file);
+  writer.close();
+
+  const chunks: Uint8Array[] = [];
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    if (value) chunks.push(value);
+  }
+
+  const compressed = new Uint8Array(
+    chunks.reduce((acc, chunk) => acc + chunk.length, 0),
+  );
+  let offset = 0;
+  chunks.forEach((chunk) => {
+    compressed.set(chunk, offset);
+    offset += chunk.length;
+  });
+
+  return compressed;
+}
+
+export async function zipNode(file: Uint8Array): Promise<Uint8Array> {
+  const { promisify } = await import("util");
+  const { gzip } = await import("zlib");
+  const gzipPromise = promisify(gzip);
+  return await gzipPromise(file);
+}
+
+export async function unzip(file: Uint8Array): Promise<Uint8Array> {
+  if (typeof window !== "undefined" && "DecompressionStream" in window) {
+    return await unzipBrowser(file);
+  } else if (
+    typeof process !== "undefined" &&
+    process.versions &&
+    process.versions.node
+  ) {
+    return await unzipNode(file);
+  } else {
+    throw new Error("Unsupported environment for decompression");
+  }
+}
+
+export async function unzipBrowser(file: Uint8Array): Promise<Uint8Array> {
+  const ds = new DecompressionStream("gzip");
+  const writer = ds.writable.getWriter();
+  const reader = ds.readable.getReader();
+
+  writer.write(file);
+  writer.close();
+
+  const chunks: Uint8Array[] = [];
+
+  while (true) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    if (value) chunks.push(value);
+  }
+
+  const decompressed = new Uint8Array(
+    chunks.reduce((acc, chunk) => acc + chunk.length, 0),
+  );
+  let offset = 0;
+  chunks.forEach((chunk) => {
+    decompressed.set(chunk, offset);
+    offset += chunk.length;
+  });
+
+  return decompressed;
+}
+
+export async function unzipNode(file: Uint8Array): Promise<Uint8Array> {
+  const { promisify } = await import("util");
+  const { gunzip } = await import("zlib");
+  const gunzipPromise = promisify(gunzip);
+  return await gunzipPromise(file);
+}
+
+function dateToUnixTimestamp(date: Date | number | undefined): number {
+  if (!date) {
+    return Math.floor(Date.now() / 1000);
+  } else {
+    return typeof date === "number" ? date : Math.floor(date.getTime() / 1000);
+  }
+}

packages/pglite/src/fs/types.ts

@@ -15,15 +15,17 @@ export interface Filesystem {
   /**
    * Sync the filesystem to the emscripten filesystem.
    */
-  syncToFs(mod: FS): Promise<void>;
+  syncToFs(FS: FS): Promise<void>;
 
   /**
    * Sync the emscripten filesystem to the filesystem.
    */
-  initialSyncFs(mod: FS): Promise<void>;
+  initialSyncFs(FS: FS): Promise<void>;
 
-  //  on_mount(): Function<void>;
-  // load_extension(ext: string): Promise<void>;
+  /**
+   * Dump the PGDATA dir from the filesystem to a gziped tarball.
+   */
+  dumpTar(FS: FS, dbname: string): Promise<File>;
 }
 
 export abstract class FilesystemBase implements Filesystem {
@@ -34,6 +36,7 @@ export abstract class FilesystemBase implements Filesystem {
   abstract emscriptenOpts(
     opts: Partial<PostgresMod>,
   ): Promise<Partial<PostgresMod>>;
-  async syncToFs(mod: FS) {}
+  async syncToFs(FS: FS) {}
   async initialSyncFs(mod: FS) {}
+  abstract dumpTar(mod: FS, dbname: string): Promise<File>;
 }

packages/pglite/src/interface.ts

@@ -43,12 +43,19 @@ export type Extensions = {
   [namespace: string]: Extension | URL;
 };
 
+export interface DumpDataDirResult {
+  tarball: Uint8Array;
+  extension: ".tar" | ".tgz";
+  filename: string;
+}
+
 export interface PGliteOptions {
   dataDir?: string;
   fs?: Filesystem;
   debug?: DebugLevel;
   relaxedDurability?: boolean;
   extensions?: Extensions;
+  loadDataDir?: Blob | File;
 }
 
 export type PGliteInterface = {
@@ -83,6 +90,7 @@ export type PGliteInterface = {
     callback: (channel: string, payload: string) => void,
   ): () => void;
   offNotification(callback: (channel: string, payload: string) => void): void;
+  dumpDataDir(): Promise<File>;
 };
 
 export type PGliteInterfaceExtensions<E> = E extends Extensions