diff --git a/package-lock.json b/package-lock.json
index fd774558..dd1eb651 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1044,6 +1044,19 @@
         "node": ">= 10"
       }
     },
+    "node_modules/@obsidize/tar-browserify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@obsidize/tar-browserify/-/tar-browserify-3.0.0.tgz",
+      "integrity": "sha512-Gc5M9Sf/2lg34GGaI8eu891WNOS6yUwxQxSQxQYEOUf6AboQ00JHR874Us7ca7XssVcGaR+6w+K1PEQ2W1H7Jw==",
+      "dependencies": {
+        "tslib": "2.4.1"
+      }
+    },
+    "node_modules/@obsidize/tar-browserify/node_modules/tslib": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
+      "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA=="
+    },
     "node_modules/@octokit/auth-token": {
       "version": "3.0.4",
       "dev": true,
@@ -1225,12 +1238,13 @@
       }
     },
     "node_modules/@playwright/test": {
-      "version": "1.35.0",
+      "version": "1.37.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.37.1.tgz",
+      "integrity": "sha512-bq9zTli3vWJo8S3LwB91U0qDNQDpEXnw7knhxLM0nwDvexQAwx9tO8iKDZSqqneVq+URd/WIoz+BALMqUTgdSg==",
       "dev": true,
-      "license": "Apache-2.0",
       "dependencies": {
         "@types/node": "*",
-        "playwright-core": "1.35.0"
+        "playwright-core": "1.37.1"
       },
       "bin": {
         "playwright": "cli.js"
@@ -4459,6 +4473,7 @@
     },
     "node_modules/js-untar": {
       "version": "2.0.0",
+      "dev": true,
       "license": "MIT"
     },
     "node_modules/js-yaml": {
@@ -6899,9 +6914,10 @@
       }
     },
     "node_modules/playwright-core": {
-      "version": "1.35.0",
+      "version": "1.37.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.1.tgz",
+      "integrity": "sha512-17EuQxlSIYCmEMwzMqusJ2ztDgJePjrbttaefgdsiqeLWidjYz9BxXaTaZWxH1J95SHGk6tjE+dwgWILJoUZfA==",
       "dev": true,
-      "license": "Apache-2.0",
       "bin": {
         "playwright-core": "cli.js"
       },
@@ -9406,7 +9422,7 @@
         "url-safe-base64": "^1.1.1"
       },
       "devDependencies": {
-        "@playwright/test": "^1.35.0",
+        "@playwright/test": "^1.37.1",
         "@types/url-safe-base64": "^1.1.0",
         "eslint": "^8.2.0",
         "typescript": "^5.1.6",
@@ -9572,9 +9588,9 @@
         "@codemirror/language": "^0.19.0",
         "@codemirror/rectangular-selection": "^0.19.2",
         "@codemirror/view": "^0.19.31",
+        "@obsidize/tar-browserify": "^3.0.0",
         "@runno/host": "^0.5.1",
         "@runno/wasi": "^0.5.1",
-        "js-untar": "^2.0.0",
         "lit": "^2.7.6",
         "pako": "^1.0.11",
         "runno-codemirror-lang-ruby": "^0.0.2",
@@ -9583,6 +9599,7 @@
         "xterm-addon-web-links": "^0.6.0"
       },
       "devDependencies": {
+        "@playwright/test": "^1.37.1",
         "@rollup/plugin-typescript": "^8.2.5",
         "eslint": "^7.28.0",
         "typedoc": "^0.24.8",
@@ -9924,7 +9941,7 @@
       "version": "0.5.1",
       "license": "MIT",
       "devDependencies": {
-        "@playwright/test": "^1.26.1",
+        "@playwright/test": "^1.37.1",
         "typedoc": "^0.24.8",
         "typescript": "^5.1.6",
         "vite": "^4.4.9"
@@ -10991,6 +11008,21 @@
       "dev": true,
       "optional": true
     },
+    "@obsidize/tar-browserify": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@obsidize/tar-browserify/-/tar-browserify-3.0.0.tgz",
+      "integrity": "sha512-Gc5M9Sf/2lg34GGaI8eu891WNOS6yUwxQxSQxQYEOUf6AboQ00JHR874Us7ca7XssVcGaR+6w+K1PEQ2W1H7Jw==",
+      "requires": {
+        "tslib": "2.4.1"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz",
+          "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA=="
+        }
+      }
+    },
     "@octokit/auth-token": {
       "version": "3.0.4",
       "dev": true
@@ -11110,12 +11142,14 @@
       "optional": true
     },
     "@playwright/test": {
-      "version": "1.35.0",
+      "version": "1.37.1",
+      "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.37.1.tgz",
+      "integrity": "sha512-bq9zTli3vWJo8S3LwB91U0qDNQDpEXnw7knhxLM0nwDvexQAwx9tO8iKDZSqqneVq+URd/WIoz+BALMqUTgdSg==",
       "dev": true,
       "requires": {
         "@types/node": "*",
         "fsevents": "2.3.2",
-        "playwright-core": "1.35.0"
+        "playwright-core": "1.37.1"
       }
     },
     "@rollup/plugin-typescript": {
@@ -11138,7 +11172,7 @@
     "@runno/client": {
       "version": "file:packages/client",
       "requires": {
-        "@playwright/test": "^1.35.0",
+        "@playwright/test": "^1.37.1",
         "@runno/host": "^0.5.1",
         "@runno/runtime": "^0.5.1",
         "@types/url-safe-base64": "^1.1.0",
@@ -11234,11 +11268,12 @@
         "@codemirror/language": "^0.19.0",
         "@codemirror/rectangular-selection": "^0.19.2",
         "@codemirror/view": "^0.19.31",
+        "@obsidize/tar-browserify": "^3.0.0",
+        "@playwright/test": "^1.37.1",
         "@rollup/plugin-typescript": "^8.2.5",
         "@runno/host": "^0.5.1",
         "@runno/wasi": "^0.5.1",
         "eslint": "^7.28.0",
-        "js-untar": "^2.0.0",
         "lit": "^2.7.6",
         "pako": "^1.0.11",
         "runno-codemirror-lang-ruby": "^0.0.2",
@@ -11452,7 +11487,7 @@
     "@runno/wasi": {
       "version": "file:packages/wasi",
       "requires": {
-        "@playwright/test": "^1.26.1",
+        "@playwright/test": "^1.37.1",
         "typedoc": "^0.24.8",
         "typescript": "^5.1.6",
         "vite": "^4.4.9"
@@ -13596,7 +13631,8 @@
       "dev": true
     },
     "js-untar": {
-      "version": "2.0.0"
+      "version": "2.0.0",
+      "dev": true
     },
     "js-yaml": {
       "version": "4.1.0",
@@ -15204,7 +15240,9 @@
       }
     },
     "playwright-core": {
-      "version": "1.35.0",
+      "version": "1.37.1",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.37.1.tgz",
+      "integrity": "sha512-17EuQxlSIYCmEMwzMqusJ2ztDgJePjrbttaefgdsiqeLWidjYz9BxXaTaZWxH1J95SHGk6tjE+dwgWILJoUZfA==",
       "dev": true
     },
     "please-upgrade-node": {
diff --git a/package.json b/package.json
index 89f7f43d..79ffb204 100644
--- a/package.json
+++ b/package.json
@@ -27,7 +27,8 @@
     "build:deploy": "npm run bootstrap && npm run build",
     "test:client": "cd packages/client && npm run test",
     "test:wasi": "cd packages/wasi && npm run test:prepare && npm run test",
-    "test": "npm run test:client && npm run test:wasi"
+    "test:runtime": "cd packages/runtime && npm run test",
+    "test": "npm run test:client && npm run test:wasi && npm run test:runtime"
   },
   "eslintConfig": {
     "extends": "preact",
diff --git a/packages/client/package.json b/packages/client/package.json
index 79929188..33f69f45 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -26,7 +26,7 @@
     "test": "npm run test:playwright"
   },
   "devDependencies": {
-    "@playwright/test": "^1.35.0",
+    "@playwright/test": "^1.37.1",
     "@types/url-safe-base64": "^1.1.0",
     "eslint": "^8.2.0",
     "typescript": "^5.1.6",
diff --git a/packages/runtime/.gitignore b/packages/runtime/.gitignore
index 53f7466a..7407bdba 100644
--- a/packages/runtime/.gitignore
+++ b/packages/runtime/.gitignore
@@ -2,4 +2,7 @@ node_modules
 .DS_Store
 dist
 dist-ssr
-*.local
\ No newline at end of file
+*.local
+/test-results/
+/playwright-report/
+/playwright/.cache/
diff --git a/packages/runtime/lib/headless.ts b/packages/runtime/lib/headless.ts
index 325b7a60..1dadfa8b 100644
--- a/packages/runtime/lib/headless.ts
+++ b/packages/runtime/lib/headless.ts
@@ -5,7 +5,7 @@ import {
   commandsForRuntime,
   getBinaryPathFromCommand,
 } from "./commands";
-import { fetchWASIFS } from "./helpers";
+import { fetchWASIFS, makeRunnoError } from "./helpers";
 
 export async function headlessRunCode(
   runtime: Runtime,
@@ -35,12 +35,32 @@ export async function headlessRunFS(
 ): Promise<RunResult> {
   const commands = commandsForRuntime(runtime, entryPath);
 
-  const prepare = await headlessPrepareFS(commands.prepare ?? [], fs);
-  fs = prepare.fs;
+  let prepare: CompleteResult;
+  try {
+    prepare = await headlessPrepareFS(commands.prepare ?? [], fs);
+    fs = prepare.fs;
+  } catch (e) {
+    return {
+      resultType: "crash",
+      error: makeRunnoError(e),
+    };
+  }
 
   const { run } = commands;
   const binaryPath = getBinaryPathFromCommand(run, fs);
 
+  if (run.baseFSURL) {
+    try {
+      const baseFS = await fetchWASIFS(run.baseFSURL);
+      fs = { ...fs, ...baseFS };
+    } catch (e) {
+      return {
+        resultType: "crash",
+        error: makeRunnoError(e),
+      };
+    }
+  }
+
   const workerHost = new WASIWorkerHost(binaryPath, {
     args: [run.binaryName, ...(run.args ?? [])],
     env: run.env,
@@ -61,7 +81,7 @@ export async function headlessRunFS(
 
   const result = await workerHost.start();
 
-  prepare.fs = { ...prepare.fs, ...result.fs };
+  prepare.fs = { ...fs, ...result.fs };
   prepare.exitCode = result.exitCode;
 
   return prepare;
@@ -129,9 +149,6 @@ export async function headlessPrepareFS(
     prepare.exitCode = result.exitCode;
 
     if (result.exitCode !== 0) {
-      // TODO: Remove this
-      console.error("Prepare failed", prepare);
-
       // If a prepare step fails then we stop.
       throw new PrepareError(
         "Prepare step returned a non-zero exit code",
diff --git a/packages/runtime/lib/helpers.ts b/packages/runtime/lib/helpers.ts
index bf13490a..8a329298 100644
--- a/packages/runtime/lib/helpers.ts
+++ b/packages/runtime/lib/helpers.ts
@@ -1,4 +1,3 @@
-import { WASIFS } from "@runno/wasi";
 import { extractTarGz } from "./tar";
 
 export function stripWhitespace(text: string): string {
@@ -62,23 +61,7 @@ export function elementCodeContent(element: HTMLElement): string {
 export async function fetchWASIFS(fsURL: string) {
   const response = await fetch(fsURL);
   const buffer = await response.arrayBuffer();
-  const files = await extractTarGz(new Uint8Array(buffer));
-
-  const fs: WASIFS = {};
-  for (const file of files) {
-    fs[file.name] = {
-      path: file.name,
-      timestamps: {
-        change: new Date(file.lastModified),
-        access: new Date(file.lastModified),
-        modification: new Date(file.lastModified),
-      },
-      mode: "binary",
-      content: new Uint8Array(await file.arrayBuffer()),
-    };
-  }
-
-  return fs;
+  return await extractTarGz(new Uint8Array(buffer));
 }
 
 export function isErrorObject(
diff --git a/packages/runtime/lib/provider.ts b/packages/runtime/lib/provider.ts
index 5c2c20bb..1c4c7013 100644
--- a/packages/runtime/lib/provider.ts
+++ b/packages/runtime/lib/provider.ts
@@ -102,8 +102,18 @@ export class RunnoProvider implements RuntimeMethods {
     this.terminal.terminal.clear();
 
     if (run.baseFSURL) {
-      const baseFS = await fetchWASIFS(run.baseFSURL);
-      fs = { ...fs, ...baseFS };
+      try {
+        const baseFS = await fetchWASIFS(run.baseFSURL);
+        fs = { ...fs, ...baseFS };
+      } catch (e) {
+        console.error(e);
+        this.terminal.terminal.write(`\nRunno crashed: ${e}\n`);
+
+        return {
+          resultType: "crash",
+          error: makeRunnoError(e),
+        };
+      }
     }
 
     return this.terminal.run(
diff --git a/packages/runtime/lib/tar.ts b/packages/runtime/lib/tar.ts
index 62806672..a3955955 100644
--- a/packages/runtime/lib/tar.ts
+++ b/packages/runtime/lib/tar.ts
@@ -1,15 +1,9 @@
-// @ts-ignore - No type definitions
-import untar from "js-untar";
+import type { WASIFS } from "@runno/wasi";
+import { Tarball } from "@obsidize/tar-browserify";
+
 // @ts-ignore - No type definitions
 import { inflate } from "pako/dist/pako_inflate.min.js";
 
-type TarFile = {
-  name: string;
-  type: string | number;
-  buffer: ArrayBuffer;
-  blob: Blob;
-};
-
 /**
  * Extract a .tar.gz file.
  *
@@ -18,9 +12,7 @@ type TarFile = {
  * @param binary .tar.gz file
  * @returns
  */
-export const extractTarGz = async (binary: Uint8Array): Promise<File[]> => {
-  let files: File[] = [];
-
+export const extractTarGz = async (binary: Uint8Array): Promise<WASIFS> => {
   // If we receive a tar.gz, we first need to uncompress it.
   let inflatedBinary: Uint8Array;
   try {
@@ -29,21 +21,27 @@ export const extractTarGz = async (binary: Uint8Array): Promise<File[]> => {
     inflatedBinary = binary;
   }
 
-  try {
-    files = (await untar(inflatedBinary.buffer))
-      .filter((file: TarFile) => {
-        return file.type === "file" || file.type === "0" || file.type == 0;
-      })
-      .map((file: TarFile) => {
-        // HACK: Make all files start with / to solve compatibility issues
-        const name = file.name.replace(/^([^/])/, "/$1");
-        return new File([file.blob], name, {
-          lastModified: Date.now(),
-        });
-      });
-  } catch (e) {
-    console.log("failed untar", e);
+  const entries = Tarball.extract(inflatedBinary);
+
+  const fs: WASIFS = {};
+  for (const entry of entries) {
+    if (!entry.isFile()) {
+      continue;
+    }
+
+    // HACK: Make sure each file name starts with /
+    const name = entry.fileName.replace(/^([^/])/, "/$1");
+    fs[name] = {
+      path: name,
+      timestamps: {
+        change: new Date(entry.lastModified),
+        access: new Date(entry.lastModified),
+        modification: new Date(entry.lastModified),
+      },
+      mode: "binary",
+      content: entry.content!,
+    };
   }
 
-  return files;
+  return fs;
 };
diff --git a/packages/runtime/package.json b/packages/runtime/package.json
index 1e5e3b39..f6ca667e 100644
--- a/packages/runtime/package.json
+++ b/packages/runtime/package.json
@@ -31,6 +31,9 @@
   "scripts": {
     "dev": "vite",
     "watch": "vite build --watch",
+    "test:serve": "vite --config tests/vite.config.js",
+    "test:playwright": "playwright test",
+    "test": "npm run test:playwright",
     "build:docs": "typedoc --options typedoc.config.cjs",
     "build:package": "tsc --noEmit && vite build",
     "build": "npm run build:docs && npm run build:package",
@@ -38,6 +41,7 @@
     "lint": "npx eslint src"
   },
   "devDependencies": {
+    "@playwright/test": "^1.37.1",
     "@rollup/plugin-typescript": "^8.2.5",
     "eslint": "^7.28.0",
     "typedoc": "^0.24.8",
@@ -57,9 +61,9 @@
     "@codemirror/language": "^0.19.0",
     "@codemirror/rectangular-selection": "^0.19.2",
     "@codemirror/view": "^0.19.31",
+    "@obsidize/tar-browserify": "^3.0.0",
     "@runno/host": "^0.5.1",
     "@runno/wasi": "^0.5.1",
-    "js-untar": "^2.0.0",
     "lit": "^2.7.6",
     "pako": "^1.0.11",
     "runno-codemirror-lang-ruby": "^0.0.2",
diff --git a/packages/runtime/playwright.config.ts b/packages/runtime/playwright.config.ts
new file mode 100644
index 00000000..107fa5be
--- /dev/null
+++ b/packages/runtime/playwright.config.ts
@@ -0,0 +1,86 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: "./tests",
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:5679",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+
+    {
+      name: "firefox",
+      use: { ...devices["Desktop Firefox"] },
+    },
+
+    // TODO: Playwright doesn't support Shared Array Buffer in Safari
+    // https://github.com/microsoft/playwright/issues/14043
+    //
+    // {
+    //   name: "webkit",
+    //   use: { ...devices["Desktop Safari"] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: [
+    {
+      command: "npm run test:serve",
+      port: 5679,
+    },
+    {
+      command: "cd ../website && npm run dev",
+      port: 4321,
+      reuseExistingServer: !process.env.CI,
+    },
+  ],
+});
diff --git a/packages/runtime/src/main.ts b/packages/runtime/src/main.ts
index 121587a1..09ebc189 100644
--- a/packages/runtime/src/main.ts
+++ b/packages/runtime/src/main.ts
@@ -1 +1,6 @@
 import "../lib/main";
+import { headlessRunCode } from "../lib/main";
+
+globalThis.Runno = {
+  headlessRunCode,
+};
diff --git a/packages/runtime/tests/headless.spec.ts b/packages/runtime/tests/headless.spec.ts
new file mode 100644
index 00000000..ad06ff3a
--- /dev/null
+++ b/packages/runtime/tests/headless.spec.ts
@@ -0,0 +1,39 @@
+import { test, expect } from "@playwright/test";
+import { RunResult } from "@runno/host";
+
+// TODO: These are dependent on `@runno/website` being run on localhost:4321
+// See: https://github.com/taybenlor/runno/issues/258
+
+// TODO: This python test times out, I think because of the massive base tar file.
+// When I comment out the base FS it works fine.
+test.skip("a simple python example", async ({ page }) => {
+  await page.goto("/");
+
+  const result: RunResult = await page.evaluate(async () => {
+    return await globalThis.Runno.headlessRunCode(
+      "python",
+      `print("Hello, World!")`
+    );
+  });
+
+  expect(result.resultType).toBe("complete");
+  if (result.resultType !== "complete") throw new Error("wtf");
+  expect(result.stderr).toBe("");
+  expect(result.stdout).toBe("Hello, World!\n");
+});
+
+test("a simple ruby example", async ({ page }) => {
+  await page.goto("/");
+
+  const result: RunResult = await page.evaluate(async () => {
+    return await globalThis.Runno.headlessRunCode(
+      "ruby",
+      `puts "Hello, World!"`
+    );
+  });
+
+  expect(result.resultType).toBe("complete");
+  if (result.resultType !== "complete") throw new Error("wtf");
+  expect(result.stderr).toBe("");
+  expect(result.stdout).toBe("Hello, World!\n");
+});
diff --git a/packages/runtime/tests/vite.config.js b/packages/runtime/tests/vite.config.js
new file mode 100644
index 00000000..e71217ee
--- /dev/null
+++ b/packages/runtime/tests/vite.config.js
@@ -0,0 +1,20 @@
+import { defineConfig } from "vite";
+
+const crossOriginPolicy = {
+  name: "configure-server",
+
+  configureServer(server) {
+    server.middlewares.use((_req, res, next) => {
+      res.setHeader("Cross-Origin-Opener-Policy", "same-origin");
+      res.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
+      next();
+    });
+  },
+};
+
+export default defineConfig({
+  plugins: [crossOriginPolicy],
+  server: {
+    port: 5679,
+  },
+});
diff --git a/packages/wasi/package.json b/packages/wasi/package.json
index 51b9417a..faaefd5f 100644
--- a/packages/wasi/package.json
+++ b/packages/wasi/package.json
@@ -40,7 +40,7 @@
     "preview": "vite preview"
   },
   "devDependencies": {
-    "@playwright/test": "^1.26.1",
+    "@playwright/test": "^1.37.1",
     "typescript": "^5.1.6",
     "typedoc": "^0.24.8",
     "vite": "^4.4.9"
diff --git a/packages/website/public/langs/python-3.11.3.tar.gz b/packages/website/public/langs/python-3.11.3.tar.gz
index 30fd050d..ce8b6d5c 100644
Binary files a/packages/website/public/langs/python-3.11.3.tar.gz and b/packages/website/public/langs/python-3.11.3.tar.gz differ