Skip to content

Commit

Permalink
update local testing workflow to allow for parallelism, improved dx (#…
Browse files Browse the repository at this point in the history
…3707)

* update local testing workflow to allow for parallelism, improved dx

* update comment

* remove extraneous comment

* move file/directory removal to before process exit

* prettier
  • Loading branch information
briangregoryholmes authored Feb 15, 2024
1 parent 5cc2f23 commit c588b51
Show file tree
Hide file tree
Showing 12 changed files with 105 additions and 48 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"dev-web": "npm run dev -w web-local",
"dev-runtime": "go run cli/main.go start dev-project --no-ui",
"clean": "rm -rf dev-project",
"test": "npm run test -w web-common && npm run test -w web-local && npm run test -w web-auth"
"test": "npm run test -w web-common & npm run test -w web-auth & make cli && npm run test -w web-local"
},
"overrides": {
"@rgossiaux/svelte-headlessui": {
Expand Down
8 changes: 4 additions & 4 deletions web-local/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import { defineConfig, devices } from "@playwright/test";
*/
export default defineConfig({
testDir: "./tests",
/* Don't run tests in files in parallel */
fullyParallel: false,
/* Don't run tests in files in parallel in CI*/
fullyParallel: !process.env.CI,
/* Fail the build on CI if you accidentally left test.only in the source code. */
forbidOnly: !!process.env.CI,
retries: 0,
/* Opt out of parallel testing for now */
workers: 1,
/* Opt out of parallel testing in CI */
workers: process.env.CI ? 1 : 8,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: "html",
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
Expand Down
3 changes: 1 addition & 2 deletions web-local/tests/dashboards/dashboard-flow-test-setup.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import { test } from "@playwright/test";
import { test } from "../utils/test";
import { createDashboardFromModel } from "web-local/tests/utils/dashboardHelpers";
import { createAdBidsModel } from "web-local/tests/utils/dataSpecifcHelpers";

export function useDashboardFlowTestSetup() {
test.beforeEach(async ({ page }) => {
test.setTimeout(60000);
await page.goto("/");
// disable animations
await page.addStyleTag({
content: `
Expand Down
12 changes: 3 additions & 9 deletions web-local/tests/dashboards/dashboards.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { expect, Page, test } from "@playwright/test";
import { expect, Page } from "@playwright/test";
import {
TestEntityType,
updateCodeEditor,
Expand All @@ -22,15 +22,11 @@ import {
createAdBidsModel,
} from "../utils/dataSpecifcHelpers";
import { createOrReplaceSource } from "../utils/sourceHelpers";
import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
import { waitForEntity } from "../utils/waitHelpers";
import { test } from "../utils/test";

test.describe("dashboard", () => {
startRuntimeForEachTest();

test("Autogenerate dashboard from source", async ({ page }) => {
await page.goto("/");

await createOrReplaceSource(page, "AdBids.csv", "AdBids");
await createDashboardFromSource(page, "AdBids");
await waitForEntity(
Expand All @@ -43,8 +39,6 @@ test.describe("dashboard", () => {
});

test("Autogenerate dashboard from model", async ({ page }) => {
await page.goto("/");

await createAdBidsModel(page);
await Promise.all([
waitForEntity(
Expand Down Expand Up @@ -103,7 +97,7 @@ test.describe("dashboard", () => {
// });

test.setTimeout(60000);
await page.goto("/");

// disable animations
await page.addStyleTag({
content: `
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import { expect, test } from "@playwright/test";
import { expect } from "@playwright/test";
import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
import { test } from "../utils/test";

test.describe("dimension and measure selectors", () => {
startRuntimeForEachTest();
// dashboard test setup
useDashboardFlowTestSetup();

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
import { test, expect, Locator } from "@playwright/test";
import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
import { expect, Locator } from "@playwright/test";
import { test } from "../utils/test";

async function assertAAboveB(locA: Locator, locB: Locator) {
const topA = await locA.boundingBox().then((box) => box?.y);
Expand All @@ -14,7 +14,6 @@ async function assertAAboveB(locA: Locator, locB: Locator) {
}

test.describe("leaderboard and dimension table sorting", () => {
startRuntimeForEachTest();
useDashboardFlowTestSetup();

test("leaderboard and dimension table sorting", async ({ page }) => {
Expand Down
6 changes: 3 additions & 3 deletions web-local/tests/dashboards/number-formatting.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ import {
interactWithTimeRangeMenu,
waitForDashboard,
} from "../utils/dashboardHelpers";
import { test, expect } from "@playwright/test";
import { startRuntimeForEachTest } from "../utils/startRuntimeForEachTest";
import { expect } from "@playwright/test";

import { updateCodeEditor } from "../utils/commonHelpers";
import { test } from "../utils/test";

test.describe("smoke tests for number formatting", () => {
startRuntimeForEachTest();
useDashboardFlowTestSetup();

test("smoke tests for number formatting", async ({ page }) => {
Expand Down
5 changes: 2 additions & 3 deletions web-local/tests/dashboards/time-controls-from-config.spec.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import { expect, test } from "@playwright/test";
import { expect } from "@playwright/test";
import { useDashboardFlowTestSetup } from "web-local/tests/dashboards/dashboard-flow-test-setup";
import { updateCodeEditor } from "web-local/tests/utils/commonHelpers";
import {
interactWithTimeRangeMenu,
waitForDashboard,
} from "web-local/tests/utils/dashboardHelpers";
import { startRuntimeForEachTest } from "web-local/tests/utils/startRuntimeForEachTest";
import { test } from "../utils/test";

test.describe("time controls settings from dashboard config", () => {
startRuntimeForEachTest();
// dashboard test setup
useDashboardFlowTestSetup();

Expand Down
11 changes: 1 addition & 10 deletions web-local/tests/models.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { test } from "@playwright/test";
import {
TestEntityType,
deleteEntity,
Expand All @@ -14,15 +13,11 @@ import {
modelHasError,
} from "./utils/modelHelpers";
import { createOrReplaceSource } from "./utils/sourceHelpers";
import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest";
import { entityNotPresent, waitForEntity } from "./utils/waitHelpers";
import { test } from "./utils/test";

test.describe("models", () => {
startRuntimeForEachTest();

test("Create and edit model", async ({ page }) => {
await page.goto("/");

await createOrReplaceSource(page, "AdBids.csv", "AdBids");
await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions");

Expand All @@ -48,8 +43,6 @@ test.describe("models", () => {
});

test("Rename and delete model", async ({ page }) => {
await page.goto("/");

// make sure AdBids_rename_delete is present
await createModel(page, "AdBids_rename_delete");

Expand All @@ -74,8 +67,6 @@ test.describe("models", () => {
});

test("Create model from source", async ({ page }) => {
await page.goto("/");

await createOrReplaceSource(page, "AdBids.csv", "AdBids");

await Promise.all([
Expand Down
12 changes: 2 additions & 10 deletions web-local/tests/sources.spec.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { expect, test } from "@playwright/test";
import { expect } from "@playwright/test";
import {
TestEntityType,
deleteEntity,
Expand All @@ -14,15 +14,11 @@ import {
createOrReplaceSource,
uploadFile,
} from "./utils/sourceHelpers";
import { startRuntimeForEachTest } from "./utils/startRuntimeForEachTest";
import { entityNotPresent, waitForEntity } from "./utils/waitHelpers";
import { test } from "./utils/test";

test.describe("sources", () => {
startRuntimeForEachTest();

test("Import sources", async ({ page }) => {
await page.goto("/");

await Promise.all([
waitForAdBids(page, "AdBids"),
uploadFile(page, "AdBids.csv"),
Expand All @@ -46,8 +42,6 @@ test.describe("sources", () => {
});

test("Rename and delete sources", async ({ page }) => {
await page.goto("/");

await createOrReplaceSource(page, "AdBids.csv", "AdBids");

// rename
Expand All @@ -62,8 +56,6 @@ test.describe("sources", () => {
});

test("Edit source", async ({ page }) => {
await page.goto("/");

// Upload data & create two sources
await createOrReplaceSource(page, "AdImpressions.tsv", "AdImpressions");
await createOrReplaceSource(page, "AdBids.csv", "AdBids");
Expand Down
14 changes: 14 additions & 0 deletions web-local/tests/utils/getOpenPort.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { createServer } from "net";

export async function getOpenPort(): Promise<number> {
return new Promise((res) => {
const srv = createServer();
srv.listen(0, () => {
const address = srv?.address();
if (!address || typeof address === "string") {
throw new Error("Invalid address");
}
srv.close(() => res(address.port));
});
});
}
70 changes: 70 additions & 0 deletions web-local/tests/utils/test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { test as base } from "@playwright/test";
import { rmSync, writeFileSync, existsSync, mkdirSync } from "fs";
import { spawn } from "node:child_process";
import treeKill from "tree-kill";
import { getOpenPort } from "./getOpenPort";
import { asyncWaitUntil } from "@rilldata/web-common/lib/waitUtils";
import axios from "axios";

const BASE_PROJECT_DIRECTORY = "temp/test-project";

export const test = base.extend({
page: async ({ page }, use) => {
const TEST_PORT = await getOpenPort();
const TEST_PORT_GRPC = await getOpenPort();
const TEST_PROJECT_DIRECTORY = `${BASE_PROJECT_DIRECTORY}-${TEST_PORT}`;

rmSync(TEST_PROJECT_DIRECTORY, {
force: true,
recursive: true,
});

if (!existsSync(TEST_PROJECT_DIRECTORY)) {
mkdirSync(TEST_PROJECT_DIRECTORY, { recursive: true });
}

// Add `rill.yaml` file to the project repo
writeFileSync(
`${TEST_PROJECT_DIRECTORY}/rill.yaml`,
'compiler: rill-beta\ntitle: "Test Project"',
);

const cmd = `start --no-open --port ${TEST_PORT} --port-grpc ${TEST_PORT_GRPC} --db ${TEST_PROJECT_DIRECTORY}/stage.db?rill_pool_size=4 ${TEST_PROJECT_DIRECTORY}`;

const childProcess = spawn("../rill", cmd.split(" "), {
stdio: "inherit",
shell: true,
});

childProcess.on("error", console.log);

// Ping runtime until it's ready
await asyncWaitUntil(async () => {
try {
const response = await axios.get(
`http://localhost:${TEST_PORT}/v1/ping`,
);
return response.status === 200;
} catch (err) {
return false;
}
});

await page.goto(`http://localhost:${TEST_PORT}`);

await use(page);

rmSync(TEST_PROJECT_DIRECTORY, {
force: true,
recursive: true,
});

const processExit = new Promise((resolve) => {
childProcess.on("exit", resolve);
});

if (childProcess.pid) treeKill(childProcess.pid);

await processExit;
},
});

0 comments on commit c588b51

Please sign in to comment.