From c588b51d2bd5fcca483240e2e8407902d1142ca5 Mon Sep 17 00:00:00 2001 From: Brian Holmes <120223836+briangregoryholmes@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:28:59 -0500 Subject: [PATCH] update local testing workflow to allow for parallelism, improved dx (#3707) * update local testing workflow to allow for parallelism, improved dx * update comment * remove extraneous comment * move file/directory removal to before process exit * prettier --- package.json | 2 +- web-local/playwright.config.ts | 8 +-- .../dashboards/dashboard-flow-test-setup.ts | 3 +- web-local/tests/dashboards/dashboards.spec.ts | 12 +--- .../dimension-and-measure-selection.spec.ts | 5 +- .../leaderboard-and-dim-table-sort.spec.ts | 5 +- .../dashboards/number-formatting.spec.ts | 6 +- .../time-controls-from-config.spec.ts | 5 +- web-local/tests/models.spec.ts | 11 +-- web-local/tests/sources.spec.ts | 12 +--- web-local/tests/utils/getOpenPort.ts | 14 ++++ web-local/tests/utils/test.ts | 70 +++++++++++++++++++ 12 files changed, 105 insertions(+), 48 deletions(-) create mode 100644 web-local/tests/utils/getOpenPort.ts create mode 100644 web-local/tests/utils/test.ts diff --git a/package.json b/package.json index d41bf3d86a6..8a406154d51 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "dev-web": "npm run dev -w web-local", "dev-runtime": "go run cli/main.go start dev-project --no-ui", "clean": "rm -rf dev-project", - "test": "npm run test -w web-common && npm run test -w web-local && npm run test -w web-auth" + "test": "npm run test -w web-common & npm run test -w web-auth & make cli && npm run test -w web-local" }, "overrides": { "@rgossiaux/svelte-headlessui": { diff --git a/web-local/playwright.config.ts b/web-local/playwright.config.ts index 5259b648eda..c9e6708b003 100644 --- a/web-local/playwright.config.ts +++ b/web-local/playwright.config.ts @@ -4,13 +4,13 @@ import { defineConfig, devices } from "@playwright/test"; */ export default defineConfig({ testDir: "./tests", - /* Don't run tests in files in parallel */ - fullyParallel: false, + /* Don't run tests in files in parallel in CI*/ + fullyParallel: !process.env.CI, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, retries: 0, - /* Opt out of parallel testing for now */ - workers: 1, + /* Opt out of parallel testing in CI */ + workers: process.env.CI ? 1 : 8, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ diff --git a/web-local/tests/dashboards/dashboard-flow-test-setup.ts b/web-local/tests/dashboards/dashboard-flow-test-setup.ts index 353ec8b5407..26b24b61ad1 100644 --- a/web-local/tests/dashboards/dashboard-flow-test-setup.ts +++ b/web-local/tests/dashboards/dashboard-flow-test-setup.ts @@ -1,11 +1,10 @@ -import { test } from "@playwright/test"; +import { test } from "../utils/test"; import { createDashboardFromModel } from "web-local/tests/utils/dashboardHelpers"; import { createAdBidsModel } from "web-local/tests/utils/dataSpecifcHelpers"; export function useDashboardFlowTestSetup() { test.beforeEach(async ({ page }) => { test.setTimeout(60000); - await page.goto("/"); // disable animations await page.addStyleTag({ content: ` diff --git a/web-local/tests/dashboards/dashboards.spec.ts b/web-local/tests/dashboards/dashboards.spec.ts index 3642cb6f351..7e5095da9eb 100644 --- a/web-local/tests/dashboards/dashboards.spec.ts +++ b/web-local/tests/dashboards/dashboards.spec.ts @@ -1,4 +1,4 @@ -import { expect, Page, test } from "@playwright/test"; +import { expect, Page } from "@playwright/test"; import { TestEntityType, updateCodeEditor, @@ -22,15 +22,11 @@ import { createAdBidsModel, } from "../utils/dataSpecifcHelpers"; import { createOrReplaceSource } from "../utils/sourceHelpers"; -import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest"; import { waitForEntity } from "../utils/waitHelpers"; +import { test } from "../utils/test"; test.describe("dashboard", () => { - startRuntimeForEachTest(); - test("Autogenerate dashboard from source", async ({ page }) => { - await page.goto("/"); - await createOrReplaceSource(page, "AdBids.csv", "AdBids"); await createDashboardFromSource(page, "AdBids"); await waitForEntity( @@ -43,8 +39,6 @@ test.describe("dashboard", () => { }); test("Autogenerate dashboard from model", async ({ page }) => { - await page.goto("/"); - await createAdBidsModel(page); await Promise.all([ waitForEntity( @@ -103,7 +97,7 @@ test.describe("dashboard", () => { // }); test.setTimeout(60000); - await page.goto("/"); + // disable animations await page.addStyleTag({ content: ` diff --git a/web-local/tests/dashboards/dimension-and-measure-selection.spec.ts b/web-local/tests/dashboards/dimension-and-measure-selection.spec.ts index 6bc5208018a..0ed3a75d83d 100644 --- a/web-local/tests/dashboards/dimension-and-measure-selection.spec.ts +++ b/web-local/tests/dashboards/dimension-and-measure-selection.spec.ts @@ -1,9 +1,8 @@ -import { expect, test } from "@playwright/test"; +import { expect } from "@playwright/test"; import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup"; -import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest"; +import { test } from "../utils/test"; test.describe("dimension and measure selectors", () => { - startRuntimeForEachTest(); // dashboard test setup useDashboardFlowTestSetup(); diff --git a/web-local/tests/dashboards/leaderboard-and-dim-table-sort.spec.ts b/web-local/tests/dashboards/leaderboard-and-dim-table-sort.spec.ts index 2e60e2549f5..dc5c56dfb4c 100644 --- a/web-local/tests/dashboards/leaderboard-and-dim-table-sort.spec.ts +++ b/web-local/tests/dashboards/leaderboard-and-dim-table-sort.spec.ts @@ -1,6 +1,6 @@ import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup"; -import { test, expect, Locator } from "@playwright/test"; -import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest"; +import { expect, Locator } from "@playwright/test"; +import { test } from "../utils/test"; async function assertAAboveB(locA: Locator, locB: Locator) { const topA = await locA.boundingBox().then((box) => box?.y); @@ -14,7 +14,6 @@ async function assertAAboveB(locA: Locator, locB: Locator) { } test.describe("leaderboard and dimension table sorting", () => { - startRuntimeForEachTest(); useDashboardFlowTestSetup(); test("leaderboard and dimension table sorting", async ({ page }) => { diff --git a/web-local/tests/dashboards/number-formatting.spec.ts b/web-local/tests/dashboards/number-formatting.spec.ts index 226afba4b43..01a7300b693 100644 --- a/web-local/tests/dashboards/number-formatting.spec.ts +++ b/web-local/tests/dashboards/number-formatting.spec.ts @@ -4,12 +4,12 @@ import { interactWithTimeRangeMenu, waitForDashboard, } from "../utils/dashboardHelpers"; -import { test, expect } from "@playwright/test"; -import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest"; +import { expect } from "@playwright/test"; + import { updateCodeEditor } from "../utils/commonHelpers"; +import { test } from "../utils/test"; test.describe("smoke tests for number formatting", () => { - startRuntimeForEachTest(); useDashboardFlowTestSetup(); test("smoke tests for number formatting", async ({ page }) => { diff --git a/web-local/tests/dashboards/time-controls-from-config.spec.ts b/web-local/tests/dashboards/time-controls-from-config.spec.ts index 3429cb1bb12..1785547e744 100644 --- a/web-local/tests/dashboards/time-controls-from-config.spec.ts +++ b/web-local/tests/dashboards/time-controls-from-config.spec.ts @@ -1,14 +1,13 @@ -import { expect, test } from "@playwright/test"; +import { expect } from "@playwright/test"; import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup"; import { updateCodeEditor } from "web-local/tests/utils/commonHelpers"; import { interactWithTimeRangeMenu, waitForDashboard, } from "web-local/tests/utils/dashboardHelpers"; -import { startRuntimeForEachTest } from "web-local/tests/utils/startRuntimeForEachTest"; +import { test } from "../utils/test"; test.describe("time controls settings from dashboard config", () => { - startRuntimeForEachTest(); // dashboard test setup useDashboardFlowTestSetup(); diff --git a/web-local/tests/models.spec.ts b/web-local/tests/models.spec.ts index d3db47b5a0f..26132555b10 100644 --- a/web-local/tests/models.spec.ts +++ b/web-local/tests/models.spec.ts @@ -1,4 +1,3 @@ -import { test } from "@playwright/test"; import { TestEntityType, deleteEntity, @@ -14,15 +13,11 @@ import { modelHasError, } from "./utils/modelHelpers"; import { createOrReplaceSource } from "./utils/sourceHelpers"; -import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest"; import { entityNotPresent, waitForEntity } from "./utils/waitHelpers"; +import { test } from "./utils/test"; test.describe("models", () => { - startRuntimeForEachTest(); - test("Create and edit model", async ({ page }) => { - await page.goto("/"); - await createOrReplaceSource(page, "AdBids.csv", "AdBids"); await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions"); @@ -48,8 +43,6 @@ test.describe("models", () => { }); test("Rename and delete model", async ({ page }) => { - await page.goto("/"); - // make sure AdBids_rename_delete is present await createModel(page, "AdBids_rename_delete"); @@ -74,8 +67,6 @@ test.describe("models", () => { }); test("Create model from source", async ({ page }) => { - await page.goto("/"); - await createOrReplaceSource(page, "AdBids.csv", "AdBids"); await Promise.all([ diff --git a/web-local/tests/sources.spec.ts b/web-local/tests/sources.spec.ts index d5f237cd98d..5124b128c1b 100644 --- a/web-local/tests/sources.spec.ts +++ b/web-local/tests/sources.spec.ts @@ -1,4 +1,4 @@ -import { expect, test } from "@playwright/test"; +import { expect } from "@playwright/test"; import { TestEntityType, deleteEntity, @@ -14,15 +14,11 @@ import { createOrReplaceSource, uploadFile, } from "./utils/sourceHelpers"; -import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest"; import { entityNotPresent, waitForEntity } from "./utils/waitHelpers"; +import { test } from "./utils/test"; test.describe("sources", () => { - startRuntimeForEachTest(); - test("Import sources", async ({ page }) => { - await page.goto("/"); - await Promise.all([ waitForAdBids(page, "AdBids"), uploadFile(page, "AdBids.csv"), @@ -46,8 +42,6 @@ test.describe("sources", () => { }); test("Rename and delete sources", async ({ page }) => { - await page.goto("/"); - await createOrReplaceSource(page, "AdBids.csv", "AdBids"); // rename @@ -62,8 +56,6 @@ test.describe("sources", () => { }); test("Edit source", async ({ page }) => { - await page.goto("/"); - // Upload data & create two sources await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions"); await createOrReplaceSource(page, "AdBids.csv", "AdBids"); diff --git a/web-local/tests/utils/getOpenPort.ts b/web-local/tests/utils/getOpenPort.ts new file mode 100644 index 00000000000..3bdc7ec391e --- /dev/null +++ b/web-local/tests/utils/getOpenPort.ts @@ -0,0 +1,14 @@ +import { createServer } from "net"; + +export async function getOpenPort(): Promise { + return new Promise((res) => { + const srv = createServer(); + srv.listen(0, () => { + const address = srv?.address(); + if (!address || typeof address === "string") { + throw new Error("Invalid address"); + } + srv.close(() => res(address.port)); + }); + }); +} diff --git a/web-local/tests/utils/test.ts b/web-local/tests/utils/test.ts new file mode 100644 index 00000000000..eeb77374379 --- /dev/null +++ b/web-local/tests/utils/test.ts @@ -0,0 +1,70 @@ +import { test as base } from "@playwright/test"; +import { rmSync, writeFileSync, existsSync, mkdirSync } from "fs"; +import { spawn } from "node:child_process"; +import treeKill from "tree-kill"; +import { getOpenPort } from "./getOpenPort"; +import { asyncWaitUntil } from "@rilldata/web-common/lib/waitUtils"; +import axios from "axios"; + +const BASE_PROJECT_DIRECTORY = "temp/test-project"; + +export const test = base.extend({ + page: async ({ page }, use) => { + const TEST_PORT = await getOpenPort(); + const TEST_PORT_GRPC = await getOpenPort(); + const TEST_PROJECT_DIRECTORY = `${BASE_PROJECT_DIRECTORY}-${TEST_PORT}`; + + rmSync(TEST_PROJECT_DIRECTORY, { + force: true, + recursive: true, + }); + + if (!existsSync(TEST_PROJECT_DIRECTORY)) { + mkdirSync(TEST_PROJECT_DIRECTORY, { recursive: true }); + } + + // Add `rill.yaml` file to the project repo + writeFileSync( + `${TEST_PROJECT_DIRECTORY}/rill.yaml`, + 'compiler: rill-beta\ntitle: "Test Project"', + ); + + const cmd = `start --no-open --port ${TEST_PORT} --port-grpc ${TEST_PORT_GRPC} --db ${TEST_PROJECT_DIRECTORY}/stage.db?rill_pool_size=4 ${TEST_PROJECT_DIRECTORY}`; + + const childProcess = spawn("../rill", cmd.split(" "), { + stdio: "inherit", + shell: true, + }); + + childProcess.on("error", console.log); + + // Ping runtime until it's ready + await asyncWaitUntil(async () => { + try { + const response = await axios.get( + `http://localhost:${TEST_PORT}/v1/ping`, + ); + return response.status === 200; + } catch (err) { + return false; + } + }); + + await page.goto(`http://localhost:${TEST_PORT}`); + + await use(page); + + rmSync(TEST_PROJECT_DIRECTORY, { + force: true, + recursive: true, + }); + + const processExit = new Promise((resolve) => { + childProcess.on("exit", resolve); + }); + + if (childProcess.pid) treeKill(childProcess.pid); + + await processExit; + }, +});