StorybookのVRTをPlaywrightでやる (#2291)

Co-authored-by: Hiroshiba <[email protected]>
Co-authored-by: Hiroshiba <[email protected]>
Co-authored-by: hiroshiba <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.eslintrc.js

@@ -114,7 +114,7 @@ module.exports = {
       },
     },
     {
-      files: ["*.ts"],
+      files: ["*.ts", "*.mts"],
       parser: "@typescript-eslint/parser",
       extends: ["plugin:@typescript-eslint/recommended-type-checked"],
       parserOptions: tsEslintOptions,

.github/workflows/test.yml

@@ -135,6 +135,17 @@ jobs:
             npm run test:electron-e2e
           fi
 
+      - name: Run npm run test:storybook-vrt
+        run: |
+          if [ -n "${{ runner.debug }}" ]; then
+            export DEBUG="pw:browser*"
+          fi
+          ARGS=""
+          if [[ ${{ needs.config.outputs.shouldUpdateSnapshots }} == 'true' ]]; then
+            ARGS="--update-snapshots"
+          fi
+          npm run test:storybook-vrt -- $ARGS
+
       - name: Upload playwright report to artifact
         if: failure()
         uses: actions/upload-artifact@v4

.gitignore

@@ -41,3 +41,6 @@ electron-builder.yml
 
 # generated licenses.json
 /*licenses.json
+
+# Storybook
+storybook-static/

README.md

@@ -156,9 +156,20 @@ npx playwright codegen http://localhost:5173/  --viewport-size=1024,630
 
 詳細は [Playwright ドキュメントの Test generator](https://playwright.dev/docs/codegen-intro) を参照してください。
 
+### Storybook の Visual Regression Testing
+
+Storybook のコンポーネントのスクリーンショットを比較して、変更がある場合は差分を表示します。
+
+> [!NOTE]
+> このテストは Windows でのみ実行できます。
+
+```bash
+npm run test:storybook-vrt
+```
+
 #### スクリーンショットの更新
 
-ブラウザ End to End テストでは Visual Regression Testing を行っています。
+ブラウザ End to End テストと Storybook では Visual Regression Testing を行っています。
 現在 VRT テストは Windows のみで行っています。
 以下の手順でスクリーンショットを更新できます：
 

package.json

@@ -20,8 +20,9 @@
     "test-watch:electron-e2e": "cross-env PWTEST_WATCH=1 VITE_TARGET=electron playwright test",
     "test:browser-e2e": "cross-env VITE_TARGET=browser playwright test",
     "test-watch:browser-e2e": "cross-env PWTEST_WATCH=1 VITE_TARGET=browser playwright test",
-    "lint": "eslint --ext .js,.vue,.ts *.config.* src tests build .storybook",
-    "fmt": "eslint --ext .js,.vue,.ts *.config.* src tests build .storybook --fix",
+    "test:storybook-vrt": "cross-env TARGET=storybook playwright test",
+    "lint": "eslint --ext .js,.vue,.ts,.mts *.config.* src tests build .storybook",
+    "fmt": "eslint --ext .js,.vue,.ts,.mts *.config.* src tests build .storybook --fix",
     "markdownlint": "markdownlint --ignore node_modules/ --ignore dist/ --ignore dist_electron/ ./",
     "typecheck": "vue-tsc --noEmit",
     "typos": "cross-env ./build/vendored/typos/typos",

playwright.config.ts

@@ -5,39 +5,51 @@ import dotenv from "dotenv";
 dotenv.config({ override: true });
 
 let project: Project;
-const additionalWebServer: PlaywrightTestConfig["webServer"] = [];
+let webServers: PlaywrightTestConfig["webServer"];
 const isElectron = process.env.VITE_TARGET === "electron";
 const isBrowser = process.env.VITE_TARGET === "browser";
+const isStorybook = process.env.TARGET === "storybook";
+
+// エンジンの起動が必要
+const defaultEngineInfosEnv = process.env.VITE_DEFAULT_ENGINE_INFOS ?? "[]";
+const envSchema = z // FIXME: electron起動時のものと共通化したい
+  .object({
+    host: z.string(),
+    executionFilePath: z.string(),
+    executionArgs: z.array(z.string()),
+    executionEnabled: z.boolean(),
+  })
+  .passthrough()
+  .array();
+const engineInfos = envSchema.parse(JSON.parse(defaultEngineInfosEnv));
+
+const engineServers = engineInfos
+  .filter((info) => info.executionEnabled)
+  .map((info) => ({
+    command: `${info.executionFilePath} ${info.executionArgs.join(" ")}`,
+    url: `${info.host}/version`,
+    reuseExistingServer: !process.env.CI,
+  }));
+const viteServer = {
+  command: "vite --mode test --port 7357",
+  port: 7357,
+  reuseExistingServer: !process.env.CI,
+};
+const storybookServer = {
+  command: "storybook dev --ci --port 7357",
+  port: 7357,
+  reuseExistingServer: !process.env.CI,
+};
 
 if (isElectron) {
   project = { name: "electron", testDir: "./tests/e2e/electron" };
+  webServers = [viteServer];
 } else if (isBrowser) {
   project = { name: "browser", testDir: "./tests/e2e/browser" };
-
-  // エンジンの起動が必要
-  const defaultEngineInfosEnv = process.env.VITE_DEFAULT_ENGINE_INFOS ?? "[]";
-  const envSchema = z // FIXME: electron起動時のものと共通化したい
-    .object({
-      host: z.string(),
-      executionFilePath: z.string(),
-      executionArgs: z.array(z.string()),
-      executionEnabled: z.boolean(),
-    })
-    .passthrough()
-    .array();
-  const engineInfos = envSchema.parse(JSON.parse(defaultEngineInfosEnv));
-
-  for (const info of engineInfos) {
-    if (!info.executionEnabled) {
-      continue;
-    }
-
-    additionalWebServer.push({
-      command: `${info.executionFilePath} ${info.executionArgs.join(" ")}`,
-      url: `${info.host}/version`,
-      reuseExistingServer: !process.env.CI,
-    });
-  }
+  webServers = [viteServer, ...engineServers];
+} else if (isStorybook) {
+  project = { name: "storybook", testDir: "./tests/e2e/storybook" };
+  webServers = [storybookServer];
 } else {
   throw new Error(`VITE_TARGETの指定が不正です。${process.env.VITE_TARGET}`);
 }
@@ -90,14 +102,7 @@ const config: PlaywrightTestConfig = {
   /* Folder for test artifacts such as screenshots, videos, traces, etc. */
   // outputDir: 'test-results/',
 
-  webServer: [
-    {
-      command: "vite --mode test --port 7357",
-      port: 7357,
-      reuseExistingServer: !process.env.CI,
-    },
-    ...additionalWebServer,
-  ],
+  webServer: webServers,
 };
 
 export default config;

tests/e2e/storybook/スクリーンショット.spec.mts

@@ -0,0 +1,70 @@
+// 起動中のStorybookで様々なStoryを表示し、スクリーンショットを撮って比較するVRT。
+// テスト自体はend-to-endではないが、Playwrightを使う関係でe2eディレクトリ内でテストしている。
+import { test, expect } from "@playwright/test";
+import z from "zod";
+
+// Storybook 8.3.5時点でのindex.jsonのスキーマ。
+// もしスキーマが変わってテストが通らなくなった場合は、このスキーマを修正する。
+const storybookIndexSchema = z.object({
+  v: z.literal(5),
+  entries: z.record(
+    z.object({
+      type: z.string(),
+      id: z.string(),
+      name: z.string(),
+      title: z.string(),
+      tags: z.array(z.string()),
+    }),
+  ),
+});
+type StorybookIndex = z.infer<typeof storybookIndexSchema>;
+type Story = StorybookIndex["entries"][string];
+
+// テスト対象のStory一覧を取得する。
+// play-fnが付いているStoryはUnit Test用Storyとみなしてスクリーンショットを撮らない
+const getStoriesToTest = (index: StorybookIndex) =>
+  Object.values(index.entries).filter(
+    (entry) => entry.type === "story" && !entry.tags.includes("play-fn"),
+  );
+
+let index: StorybookIndex;
+
+try {
+  index = storybookIndexSchema.parse(
+    await fetch("http://localhost:7357/index.json").then((res) => res.json()),
+  );
+} catch (e) {
+  throw new Error(`Storybookのindex.jsonの取得に失敗しました`, {
+    cause: e,
+  });
+}
+
+const currentStories = getStoriesToTest(index);
+
+const allStories: Record<string, Story[]> = {};
+for (const story of currentStories) {
+  if (!allStories[story.title]) {
+    allStories[story.title] = [];
+  }
+  allStories[story.title].push(story);
+}
+
+for (const [story, stories] of Object.entries(allStories)) {
+  test.describe(story, () => {
+    for (const story of stories) {
+      test(story.name, async ({ page }) => {
+        test.skip(
+          process.platform !== "win32",
+          "Windows以外のためスキップします",
+        );
+
+        await page.goto(`http://localhost:7357/iframe.html?id=${story.id}`);
+        const body = page.locator("body.sb-show-main");
+        await body.waitFor({ state: "visible" });
+        await expect(page).toHaveScreenshot(`${story.id}.png`, {
+          fullPage: true,
+        });
+      });
+    }
+  });
+}

tsconfig.json

@@ -27,6 +27,7 @@
     "src/**/*.tsx",
     "src/**/*.vue",
     "tests/**/*.ts",
+    "tests/**/*.mts",
     "tests/**/*.tsx",
     ".storybook/**/*.ts"
   ],

vite.config.mts

@@ -88,7 +88,7 @@ export default defineConfig((options) => {
             // ref: https://github.com/electron-vite/vite-plugin-electron/pull/122
             onstart: ({ startup }) => {
               if (options.mode !== "test") {
-                startup([".", "--no-sandbox"]);
+                void startup([".", "--no-sandbox"]);
               }
             },
             vite: {