update local testing workflow to allow for parallelism, improved dx

package.json

@@ -15,7 +15,7 @@
     "dev-web": "npm run dev -w web-local",
     "dev-runtime": "go run cli/main.go start dev-project --no-ui",
     "clean": "rm -rf dev-project",
-    "test": "npm run test -w web-common && npm run test -w web-local && npm run test -w web-auth"
+    "test": "npm run test -w web-common & npm run test -w web-auth & make cli && npm run test -w web-local"
   },
   "overrides": {
     "@rgossiaux/svelte-headlessui": {

web-local/playwright.config.ts

@@ -4,13 +4,13 @@ import { defineConfig, devices } from "@playwright/test";
  */
 export default defineConfig({
   testDir: "./tests",
-  /* Don't run tests in files in parallel */
-  fullyParallel: false,
+  /* Don't run tests in files in parallel in CI*/
+  fullyParallel: !process.env.CI,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   retries: 0,
-  /* Opt out of parallel testing for now */
-  workers: 1,
+  /* Opt out of parallel testing in CI */
+  workers: process.env.CI ? 1 : 8,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: "html",
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

web-local/tests/dashboards/dashboard-flow-test-setup.ts

@@ -1,11 +1,10 @@
-import { test } from "@playwright/test";
+import { test } from "../utils/test";
 import { createDashboardFromModel } from "web-local/tests/utils/dashboardHelpers";
 import { createAdBidsModel } from "web-local/tests/utils/dataSpecifcHelpers";
 
 export function useDashboardFlowTestSetup() {
   test.beforeEach(async ({ page }) => {
     test.setTimeout(60000);
-    await page.goto("/");
     // disable animations
     await page.addStyleTag({
       content: `

web-local/tests/dashboards/dashboards.spec.ts

@@ -1,4 +1,4 @@
-import { expect, Page, test } from "@playwright/test";
+import { expect, Page } from "@playwright/test";
 import {
   TestEntityType,
   updateCodeEditor,
@@ -22,15 +22,11 @@ import {
   createAdBidsModel,
 } from "../utils/dataSpecifcHelpers";
 import { createOrReplaceSource } from "../utils/sourceHelpers";
-import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
 import { waitForEntity } from "../utils/waitHelpers";
+import { test } from "../utils/test";
 
 test.describe("dashboard", () => {
-  startRuntimeForEachTest();
-
   test("Autogenerate dashboard from source", async ({ page }) => {
-    await page.goto("/");
-
     await createOrReplaceSource(page, "AdBids.csv", "AdBids");
     await createDashboardFromSource(page, "AdBids");
     await waitForEntity(
@@ -43,8 +39,6 @@ test.describe("dashboard", () => {
   });
 
   test("Autogenerate dashboard from model", async ({ page }) => {
-    await page.goto("/");
-
     await createAdBidsModel(page);
     await Promise.all([
       waitForEntity(
@@ -103,7 +97,7 @@ test.describe("dashboard", () => {
     // });
 
     test.setTimeout(60000);
-    await page.goto("/");
+
     // disable animations
     await page.addStyleTag({
       content: `

web-local/tests/dashboards/dimension-and-measure-selection.spec.ts

@@ -1,9 +1,8 @@
-import { expect, test } from "@playwright/test";
+import { expect } from "@playwright/test";
 import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
-import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
+import { test } from "../utils/test";
 
 test.describe("dimension and measure selectors", () => {
-  startRuntimeForEachTest();
   // dashboard test setup
   useDashboardFlowTestSetup();
 

web-local/tests/dashboards/leaderboard-and-dim-table-sort.spec.ts

@@ -1,6 +1,6 @@
 import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
-import { test, expect, Locator } from "@playwright/test";
-import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
+import { expect, Locator } from "@playwright/test";
+import { test } from "../utils/test";
 
 async function assertAAboveB(locA: Locator, locB: Locator) {
   const topA = await locA.boundingBox().then((box) => box?.y);
@@ -14,7 +14,6 @@ async function assertAAboveB(locA: Locator, locB: Locator) {
 }
 
 test.describe("leaderboard and dimension table sorting", () => {
-  startRuntimeForEachTest();
   useDashboardFlowTestSetup();
 
   test("leaderboard and dimension table sorting", async ({ page }) => {

web-local/tests/dashboards/number-formatting.spec.ts

@@ -4,12 +4,12 @@ import {
   interactWithTimeRangeMenu,
   waitForDashboard,
 } from "../utils/dashboardHelpers";
-import { test, expect } from "@playwright/test";
-import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
+import { expect } from "@playwright/test";
+
 import { updateCodeEditor } from "../utils/commonHelpers";
+import { test } from "../utils/test";
 
 test.describe("smoke tests for number formatting", () => {
-  startRuntimeForEachTest();
   useDashboardFlowTestSetup();
 
   test("smoke tests for number formatting", async ({ page }) => {

web-local/tests/dashboards/time-controls-from-config.spec.ts

@@ -1,14 +1,13 @@
-import { expect, test } from "@playwright/test";
+import { expect } from "@playwright/test";
 import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
 import { updateCodeEditor } from "web-local/tests/utils/commonHelpers";
 import {
   interactWithTimeRangeMenu,
   waitForDashboard,
 } from "web-local/tests/utils/dashboardHelpers";
-import { startRuntimeForEachTest } from "web-local/tests/utils/startRuntimeForEachTest";
+import { test } from "../utils/test";
 
 test.describe("time controls settings from dashboard config", () => {
-  startRuntimeForEachTest();
   // dashboard test setup
   useDashboardFlowTestSetup();
 

web-local/tests/models.spec.ts

@@ -1,4 +1,3 @@
-import { test } from "@playwright/test";
 import {
   TestEntityType,
   deleteEntity,
@@ -14,15 +13,11 @@ import {
   modelHasError,
 } from "./utils/modelHelpers";
 import { createOrReplaceSource } from "./utils/sourceHelpers";
-import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest";
 import { entityNotPresent, waitForEntity } from "./utils/waitHelpers";
+import { test } from "./utils/test";
 
 test.describe("models", () => {
-  startRuntimeForEachTest();
-
   test("Create and edit model", async ({ page }) => {
-    await page.goto("/");
-
     await createOrReplaceSource(page, "AdBids.csv", "AdBids");
     await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions");
 
@@ -48,8 +43,6 @@ test.describe("models", () => {
   });
 
   test("Rename and delete model", async ({ page }) => {
-    await page.goto("/");
-
     // make sure AdBids_rename_delete is present
     await createModel(page, "AdBids_rename_delete");
 
@@ -74,8 +67,6 @@ test.describe("models", () => {
   });
 
   test("Create model from source", async ({ page }) => {
-    await page.goto("/");
-
     await createOrReplaceSource(page, "AdBids.csv", "AdBids");
 
     await Promise.all([

web-local/tests/sources.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "@playwright/test";
+import { expect } from "@playwright/test";
 import {
   TestEntityType,
   deleteEntity,
@@ -14,15 +14,11 @@ import {
   createOrReplaceSource,
   uploadFile,
 } from "./utils/sourceHelpers";
-import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest";
 import { entityNotPresent, waitForEntity } from "./utils/waitHelpers";
+import { test } from "./utils/test";
 
 test.describe("sources", () => {
-  startRuntimeForEachTest();
-
   test("Import sources", async ({ page }) => {
-    await page.goto("/");
-
     await Promise.all([
       waitForAdBids(page, "AdBids"),
       uploadFile(page, "AdBids.csv"),
@@ -46,8 +42,6 @@ test.describe("sources", () => {
   });
 
   test("Rename and delete sources", async ({ page }) => {
-    await page.goto("/");
-
     await createOrReplaceSource(page, "AdBids.csv", "AdBids");
 
     // rename
@@ -62,8 +56,6 @@ test.describe("sources", () => {
   });
 
   test("Edit source", async ({ page }) => {
-    await page.goto("/");
-
     // Upload data & create two sources
     await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions");
     await createOrReplaceSource(page, "AdBids.csv", "AdBids");

web-local/tests/utils/getOpenPort.ts

@@ -0,0 +1,14 @@
+import { createServer } from "net";
+
+export async function getOpenPort(): Promise<number> {
+  return new Promise((res) => {
+    const srv = createServer();
+    srv.listen(0, () => {
+      const address = srv?.address();
+      if (!address || typeof address === "string") {
+        throw new Error("Invalid address");
+      }
+      srv.close(() => res(address.port));
+    });
+  });
+}

web-local/tests/utils/test.ts

@@ -0,0 +1,70 @@
+import { test as base } from "@playwright/test";
+import { rmSync, writeFileSync, existsSync, mkdirSync } from "fs";
+import { spawn } from "node:child_process";
+import treeKill from "tree-kill";
+import { getOpenPort } from "./getOpenPort";
+import { asyncWaitUntil } from "@rilldata/web-common/lib/waitUtils";
+import axios from "axios";
+
+const BASE_PROJECT_DIRECTORY = "temp/test-project";
+
+export const test = base.extend({
+  page: async ({ page }, use) => {
+    const TEST_PORT = await getOpenPort();
+    const TEST_PORT_GRPC = await getOpenPort();
+    const TEST_PROJECT_DIRECTORY = `${BASE_PROJECT_DIRECTORY}-${TEST_PORT}`;
+
+    rmSync(TEST_PROJECT_DIRECTORY, {
+      force: true,
+      recursive: true,
+    });
+
+    if (!existsSync(TEST_PROJECT_DIRECTORY)) {
+      mkdirSync(TEST_PROJECT_DIRECTORY, { recursive: true });
+    }
+
+    // Add `rill.yaml` file to the project repo
+    writeFileSync(
+      `${TEST_PROJECT_DIRECTORY}/rill.yaml`,
+      'compiler: rill-beta\ntitle: "Test Project"',
+    );
+
+    const cmd = `start --no-open --port ${TEST_PORT} --port-grpc ${TEST_PORT_GRPC} --db ${TEST_PROJECT_DIRECTORY}/stage.db?rill_pool_size=4 ${TEST_PROJECT_DIRECTORY}`;
+
+    const childProcess = spawn("../rill", cmd.split(" "), {
+      stdio: "inherit",
+      shell: true,
+    });
+
+    childProcess.on("error", console.log);
+
+    // Ping runtime until it's ready
+    await asyncWaitUntil(async () => {
+      try {
+        const response = await axios.get(
+          `http://localhost:${TEST_PORT}/v1/ping`,
+        );
+        return response.status === 200;
+      } catch (err) {
+        return false;
+      }
+    });
+
+    await page.goto(`http://localhost:${TEST_PORT}`);
+
+    await use(page);
+
+    rmSync(TEST_PROJECT_DIRECTORY, {
+      force: true,
+      recursive: true,
+    });
+
+    const processExit = new Promise((resolve) => {
+      childProcess.on("exit", resolve);
+    });
+
+    if (childProcess.pid) treeKill(childProcess.pid);
+
+    await processExit;
+  },
+});