Setup Cloud e2e test framework and write first tests

.github/workflows/web-test-e2e.yml

@@ -70,10 +70,13 @@ jobs:
       - name: Build and embed static UI
         run: PLAYWRIGHT_TEST=true make cli
 
+      - name: Add CLI binary to PATH
+        run: echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
+
       - name: Install browser for UI tests
         run: npx playwright install
 
-      - name: Test web local
+      - name: Test `web-local`
         if: ${{ steps.filter.outputs.local == 'true' || steps.filter.outputs.common == 'true' }}
         run: npm run test -w web-local
 
@@ -84,7 +87,8 @@ jobs:
           path: web-local/playwright-report/
           retention-days: 30
 
-      - name: Build web admin
+      - name: Build and test `web-admin`
         if: ${{ steps.filter.outputs.admin == 'true' || steps.filter.outputs.common == 'true' }}
         run: |-
           npm run build -w web-admin
+          npm run test -w web-admin

web-admin/.gitignore

@@ -1,2 +1,3 @@
 /.svelte-kit
-build
+build
+playwright

web-admin/playwright.config.ts

@@ -1,10 +1,28 @@
-import type { PlaywrightTestConfig } from "@playwright/test";
+import { devices, type PlaywrightTestConfig } from "@playwright/test";
 
 const config: PlaywrightTestConfig = {
+  globalSetup: "./tests/setup/globalSetup.ts",
+  globalTeardown: "./tests/setup/globalTeardown.ts",
   webServer: {
     command: "npm run build && npm run preview",
-    port: 4173,
+    port: 3000,
+    reuseExistingServer: !process.env.CI,
+    timeout: 120_000,
   },
+  use: {
+    baseURL: "http://localhost:3000",
+    ...devices["Desktop Chrome"],
+  },
+  projects: [
+    { name: "auth", testMatch: "authenticate.spec.ts" },
+    {
+      name: "e2e",
+      use: {
+        storageState: "playwright/.auth/user.json",
+      },
+      dependencies: ["auth"],
+    },
+  ],
 };
 
 export default config;

web-admin/tests/authenticate.spec.ts

@@ -0,0 +1,37 @@
+import dotenv from "dotenv";
+import { test } from "./setup/base";
+
+dotenv.config();
+
+const authFile = "playwright/.auth/user.json";
+
+test("authenticate", async ({ page }) => {
+  if (
+    !process.env.RILL_STAGE_QA_ACCOUNT_EMAIL ||
+    !process.env.RILL_STAGE_QA_ACCOUNT_PASSWORD
+  ) {
+    throw new Error(
+      "Missing required environment variables for authentication",
+    );
+  }
+
+  // Log in with the QA account
+  await page.goto("/");
+  await page.getByRole("button", { name: "Continue with Email" }).click();
+  await page.getByPlaceholder("Enter your email address").click();
+  await page
+    .getByPlaceholder("Enter your email address")
+    .fill(process.env.RILL_STAGE_QA_ACCOUNT_EMAIL);
+  await page.getByPlaceholder("Enter your email address").press("Tab");
+  await page
+    .getByPlaceholder("Enter your password")
+    .fill(process.env.RILL_STAGE_QA_ACCOUNT_PASSWORD);
+  await page.getByRole("button", { name: "Continue with Email" }).click();
+
+  // The login flow sets cookies in the process of several redirects.
+  // Wait for the final URL to ensure that the cookies are actually set.
+  await page.waitForURL("/");
+
+  // End of authentication steps.
+  await page.context().storageState({ path: authFile });
+});

web-admin/tests/explores.spec.ts

@@ -0,0 +1,21 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Explores", () => {
+  test("should have data", async ({ project: _, page }) => {
+    // Wait for reconciliation to complete
+    // (But, really, the dashboard link should be disabled until it's been reconciled)
+    await page.waitForTimeout(5000);
+
+    // Navigate to the explore
+    await page
+      .getByRole("link", { name: "Programmatic Ads Auction" })
+      .first()
+      .click();
+
+    // Check the Big Number
+    await expect(
+      page.getByRole("button", { name: "Requests 635M" }),
+    ).toBeVisible();
+  });
+});

web-admin/tests/homepage.spec.ts

@@ -0,0 +1,16 @@
+import { expect } from "@playwright/test";
+import { test } from "./setup/base";
+
+test.describe("Homepage", () => {
+  test("Authenticated user should see the homepage", async ({ page }) => {
+    await page.goto("/");
+    await expect(page.getByText("Hi [email protected]!")).toBeVisible();
+  });
+
+  test("Unauthenticated user should be redirected to login", async ({
+    anonPage,
+  }) => {
+    await anonPage.goto("/");
+    await expect(anonPage.getByText("Log in to Rill")).toBeVisible();
+  });
+});

web-admin/tests/organizations.spec.ts

@@ -0,0 +1,21 @@
+import { expect } from "@playwright/test";
+import { exec } from "child_process";
+import { promisify } from "util";
+import { test } from "./setup/base";
+
+const execAsync = promisify(exec);
+
+test.describe("Organizations", () => {
+  test("should create an organization", async ({ cli: _, page }) => {
+    // Create an organization
+    const { stdout: orgCreateStdout } = await execAsync("rill org create e2e");
+    expect(orgCreateStdout).toContain("Created organization");
+
+    // Go to the organization's page
+    await page.goto("/e2e");
+    await expect(page.getByRole("heading", { name: "e2e" })).toBeVisible();
+
+    // Clean up quickly
+    await execAsync("rill org delete e2e --force");
+  });
+});

web-admin/tests/projects.spec.ts

@@ -0,0 +1,41 @@
+import { expect } from "@playwright/test";
+import { exec } from "child_process";
+import { promisify } from "util";
+import { test } from "./setup/base";
+
+const execAsync = promisify(exec);
+
+test.describe("Projects", () => {
+  test("should deploy a project", async ({ page, organization: _ }) => {
+    // Execute the deploy command
+    const { stdout: deployStdout } = await execAsync(
+      "rill deploy --path tests/setup/git/repos/rill-examples --subpath rill-openrtb-prog-ads --project openrtb --github true",
+    );
+
+    // Confirm the CLI output
+    expect(deployStdout).toContain(`Created project "e2e/openrtb"`);
+    expect(deployStdout).toContain(`Opening project in browser...`);
+
+    // Expect to see the successful deployment
+    await page.goto("/e2e/openrtb");
+    await expect(page.getByText("Your trial expires in 30 days")).toBeVisible(); // Billing banner
+    await expect(page.getByText("e2e")).toBeVisible(); // Organization breadcrumb
+    await expect(page.getByText("Free trial")).toBeVisible(); // Billing status
+    await expect(page.getByText("openrtb")).toBeVisible(); // Project breadcrumb
+    await expect(
+      page.getByRole("link", { name: "Programmatic Ads Auction" }).first(),
+    ).toBeVisible(); // Link to dashboard
+    await expect(
+      page.getByRole("link", { name: "Programmatic Ads Bids" }),
+    ).toBeVisible(); // Link to dashboard
+
+    // Clean up quickly
+    await execAsync("rill project delete openrtb --force");
+  });
+});
+
+test.afterAll(async () => {
+  await execAsync(
+    "rm -rf tests/setup/git/repos/rill-examples/rill-openrtb-prog-ads/.rillcloud",
+  );
+});

web-admin/tests/setup/base.ts

@@ -0,0 +1,40 @@
+import { test as base, type Page } from "@playwright/test";
+import { cliLogin, cliLogout } from "./fixtures/cli";
+import { orgCreate, orgDelete } from "./fixtures/org";
+import { projectDelete, projectDeploy } from "./fixtures/project";
+
+type MyFixtures = {
+  anonPage: Page;
+  cli: void;
+  organization: void;
+  project: void;
+};
+
+export const test = base.extend<MyFixtures>({
+  anonPage: async ({ browser }, use) => {
+    const context = await browser.newContext({
+      storageState: { cookies: [], origins: [] },
+    });
+    const anonPage = await context.newPage();
+    await use(anonPage);
+    await context.close();
+  },
+
+  cli: async ({ page }, use) => {
+    await cliLogin(page);
+    await use();
+    await cliLogout();
+  },
+
+  organization: async ({ cli: _ }, use) => {
+    await orgCreate();
+    await use();
+    await orgDelete();
+  },
+
+  project: async ({ organization: _, page }, use) => {
+    await projectDeploy(page);
+    await use();
+    await projectDelete();
+  },
+});

web-admin/tests/setup/fixtures/cli.ts

@@ -0,0 +1,45 @@
+import { type Page } from "@playwright/test";
+import { exec, spawn } from "child_process";
+import { promisify } from "util";
+
+const execAsync = promisify(exec);
+
+export async function cliLogin(page: Page) {
+  // Run the login command
+  const loginProcess = spawn("rill", ["login"], {
+    stdio: ["inherit", "pipe", "inherit"],
+  });
+
+  // Capture the verification URL from the CLI output
+  let verificationUrl = "";
+  loginProcess.stdout.on("data", (data) => {
+    const output = data.toString();
+    const match = output.match(
+      /Open this URL in your browser to confirm the login: (.*)\n/,
+    );
+    if (match) {
+      verificationUrl = match[1];
+    }
+  });
+  while (!verificationUrl) {
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+
+  // Manually navigate to the verification URL
+  await page.goto(verificationUrl);
+
+  // Click the confirm button
+  await page.getByRole("button", { name: /confirm/i }).click();
+
+  // Wait for the process to complete
+  await new Promise((resolve, reject) => {
+    loginProcess.on("close", (code) => {
+      if (code === 0) resolve(null);
+      else reject(new Error(`Process exited with code ${code}`));
+    });
+  });
+}
+
+export async function cliLogout() {
+  await execAsync("rill logout");
+}

web-admin/tests/setup/fixtures/org.ts

@@ -0,0 +1,17 @@
+import { expect } from "@playwright/test";
+import { exec } from "child_process";
+import { promisify } from "util";
+
+const execAsync = promisify(exec);
+
+export async function orgCreate() {
+  const { stdout: orgCreateStdout } = await execAsync("rill org create e2e");
+  expect(orgCreateStdout).toContain("Created organization");
+}
+
+export async function orgDelete() {
+  const { stdout: orgDeleteStdout } = await execAsync(
+    "rill org delete e2e --force",
+  );
+  expect(orgDeleteStdout).toContain("Deleted organization");
+}

web-admin/tests/setup/fixtures/project.ts

@@ -0,0 +1,19 @@
+import type { Page } from "@playwright/test";
+import { exec } from "child_process";
+import { promisify } from "util";
+
+const execAsync = promisify(exec);
+
+export async function projectDeploy(page: Page) {
+  await execAsync(
+    "rill deploy --path tests/setup/git/repos/rill-examples --subpath rill-openrtb-prog-ads --project openrtb --github true",
+  );
+  await page.goto("/e2e/openrtb");
+}
+
+export async function projectDelete() {
+  await execAsync("rill project delete openrtb --force");
+  await execAsync(
+    "rm -rf tests/setup/git/repos/rill-examples/rill-openrtb-prog-ads/.rillcloud",
+  );
+}

web-admin/tests/setup/globalSetup.ts

@@ -0,0 +1,54 @@
+import { waitUntil } from "@rilldata/web-common/lib/waitUtils";
+import { exec } from "child_process";
+import { spawn } from "node:child_process";
+import path from "path";
+import { fileURLToPath } from "url";
+import { promisify } from "util";
+
+const execAsync = promisify(exec);
+
+const skipGlobalSetup = process.env.E2E_SKIP_GLOBAL_SETUP === "true";
+const timeout = 120_000;
+
+export default async function globalSetup() {
+  if (skipGlobalSetup) return;
+
+  // Get the repository root directory, the only place from which `rill devtool` is allowed to be run
+  const currentDir = path.dirname(fileURLToPath(import.meta.url));
+  const repoRoot = path.resolve(currentDir, "../../../");
+
+  // Start the cloud services (except for the UI, which is run by Playwright)
+  const cloudProcess = spawn(
+    "rill",
+    ["devtool", "start", "e2e", "--reset", "--except", "ui"],
+    {
+      stdio: "pipe",
+      cwd: repoRoot,
+    },
+  );
+
+  // Capture output
+  let logBuffer = "";
+  cloudProcess.stdout?.on("data", (data) => {
+    logBuffer += data.toString();
+    console.log(data.toString());
+  });
+
+  cloudProcess.stderr?.on("data", (data) => {
+    logBuffer += data.toString();
+    console.error(data.toString());
+  });
+
+  // Wait for services to be ready
+  const ready = await waitUntil(() => {
+    return logBuffer.includes("All services ready");
+  }, timeout);
+  if (!ready) {
+    throw new Error("Cloud services did not start in time");
+  }
+
+  // Pull the repositories to be used for testing
+  await execAsync(
+    "git clone https://github.com/rilldata/rill-examples.git tests/setup/git/repos/rill-examples",
+  );
+}