feat: ability to generate multiple sets of autotests (#28)

* feat: ability to generate multiple sets of autotests * fix: apply edits --------- Co-authored-by: Voytenok Artur <[email protected]>
gemini-testing · Nov 7, 2024 · c1359eb · c1359eb
1 parent 0a01148
commit c1359eb
Show file tree

Hide file tree

Showing 14 changed files with 198 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -52,25 +52,45 @@ With this minimal config, you will be able to run `npx testplane --storybook` to
 
 Full plugin config:
 
-| **Parameter**      | **Type**                | **Default&nbsp;value** | **Description**                                                             |
-| ------------------ | ----------------------- | ---------------------- | --------------------------------------------------------------------------- |
-| enabled	         | Boolean                 | true                   | Enable / disable the plugin                                                                      |
-| storybookConfigDir | String                  | ".storybook"           | Path to the storybook configuration directory                                                                   |
-| autoScreenshots	 | Boolean                 | true                   | Enable / disable auto-screenshot tests                                                                       |
-| localport	         | Number                  | 6006                   | Port to launch storybook dev server on                                                                          |
-| remoteStorybookUrl | String                  | ""                     | URL of the remote Storybook. If specified, local storybook dev sever would not be launched                             |
-| browserIds         | Array<String \| RegExp> | []                     | Array of `browserId` to run storybook tests on. By default, all of browsers, specified in Testplane config would be used |
+| **Parameter**                      | **Type**                                | **Default&nbsp;value** | **Description**                                                             |
+| ---------------------------------- | --------------------------------------- | ---------------------- | --------------------------------------------------------------------------- |
+| enabled	                         | Boolean                                 | true                   | Enable / disable the plugin                                                                      |
+| storybookConfigDir                 | String                                  | ".storybook"           | Path to the storybook configuration directory                                                                   |
+| autoScreenshots	                 | Boolean                                 | true                   | Enable / disable auto-screenshot tests                                                                       |
+| autoScreenshotStorybookGlobals	 | Record<string, Record<string, unknown>> | {}                     | Run multiple auto-screenshot tests with different [storybook globals](https://storybook.js.org/docs/7/essentials/toolbars-and-globals#globals)                                                                       |
+| localport	                         | Number                                  | 6006                   | Port to launch storybook dev server on                                                                          |
+| remoteStorybookUrl                 | String                                  | ""                     | URL of the remote Storybook. If specified, local storybook dev sever would not be launched                             |
+| browserIds                         | Array<String \| RegExp>                 | []                     | Array of `browserId` to run storybook tests on. By default, all of browsers, specified in Testplane config would be used |
 
 > ⚠️ *Storybook tests performance greatly depends on [Testplane testsPerSession](https://github.com/gemini-testing/testplane#testspersession) parameter, as these tests speeds up on reusing existing sessions, so setting values around 20+ is preferred*
 
 > ⚠️ *These tests ignore [Testplane isolation](https://github.com/gemini-testing/testplane#isolation). It would be turned off unconditionally*
 
+#### autoScreenshotStorybookGlobals
+
+For example, with `autoScreenshotStorybookGlobals` set to:
+
+```json
+{
+  "default": {},
+  "light theme": {
+    "theme": "light"
+  },
+  "dark theme": {
+    "theme": "dark"
+  }
+}
+```
+
+3 autoscreenshot tests will be generated for each story, each test having its corresponding storybook globals value:
+- `... Autoscreenshot default`
+- `... Autoscreenshot light theme`
+- `... Autoscreenshot dark theme`
+
 ## Advanced usage
 
 If you have `ts-node` in your project, you can write your Testplane tests right inside of storybook story files:
 
-> ⚠️ *Storybook story files must have `.js` or `.ts` extension for this to work*
-
 ```ts
 import type { StoryObj } from "@storybook/react";
 import type { WithTestplane } from "@testplane/storybook"
@@ -103,6 +123,12 @@ const meta: WithTestplane<Meta<typeof Button>> = {
         skip: false, // if true, skips all Testplane tests from this story file
         autoscreenshotSelector: ".my-selector", // Custom selector to auto-screenshot elements
         browserIds: ["chrome"], // Testplane browsers to run tests from this story file
+        autoScreenshotStorybookGlobals: {
+            // override default autoScreenshotStorybookGlobals options from plugin config
+            // tests for default autoScreenshotStorybookGlobals from plugin config won't be generated
+            "es locale": { locale: "es" },
+            "fr locale": { locale: "fr" }
+        },
         assertViewOpts: { // override default assertView options for tests from this file
             ignoreDiffPixelCount: 5
         }

diff --git a/src/config.ts b/src/config.ts
@@ -1,4 +1,4 @@
-import { isBoolean, isString, isNumber, isArray, isRegExp } from "lodash";
+import { isBoolean, isString, isNumber, isArray, isRegExp, isNull, isPlainObject } from "lodash";
 import { option, root, section } from "gemini-configparser";
 import type { Parser } from "gemini-configparser";
 
@@ -10,6 +10,20 @@ const assertType = <T>(name: string, validationFn: (v: unknown) => boolean, type
     };
 };
 
+const assertRecordOfRecords = (value: unknown, name: string): void => {
+    if (!isNull(value) && !isPlainObject(value)) {
+        throw new Error(`"${name}" must be an object`);
+    }
+
+    const record = value as Record<string, unknown>;
+
+    for (const key of Object.keys(record)) {
+        if (!isPlainObject(record[key])) {
+            throw new Error(`"${name}.${key}" must be an object`);
+        }
+    }
+};
+
 const booleanOption = (name: string, defaultValue = false): Parser<boolean> =>
     option({
         parseEnv: (val: string) => Boolean(JSON.parse(val)),
@@ -48,10 +62,22 @@ const stringAndRegExpArrayOption = (name: string, defaultValue: string[]): Parse
         },
     });
 
+const optionalRecordOfRecordsOption = (
+    name: string,
+    defaultValue: Record<string, unknown> = {},
+): Parser<Record<string, Record<string, unknown>>> =>
+    option({
+        parseEnv: JSON.parse,
+        parseCli: JSON.parse,
+        defaultValue,
+        validate: value => assertRecordOfRecords(value, name),
+    });
+
 export interface PluginConfig {
     enabled: boolean;
     storybookConfigDir: string;
     autoScreenshots: boolean;
+    autoScreenshotStorybookGlobals: Record<string, Record<string, unknown>>;
     localport: number;
     remoteStorybookUrl: string;
     browserIds: Array<string | RegExp>;
@@ -68,6 +94,7 @@ export function parseConfig(options: PluginPartialConfig): PluginConfig {
             enabled: booleanOption("enabled", true),
             storybookConfigDir: stringOption("storybookConfigDir", ".storybook"),
             autoScreenshots: booleanOption("autoScreenshots", true),
+            autoScreenshotStorybookGlobals: optionalRecordOfRecordsOption("autoScreenshotStorybookGlobals", {}),
             localport: numberOption("localport", 6006),
             remoteStorybookUrl: stringOption("remoteStorybookUrl", ""),
             browserIds: stringAndRegExpArrayOption("browserIds", []),

diff --git a/src/index.ts b/src/index.ts
@@ -78,7 +78,10 @@ function onTestplaneMaster(testplane: Testplane, config: PluginConfig): void {
 
         const stories = await getStories(storybookUrl);
 
-        const storyTestFiles = await buildStoryTestFiles(stories, { autoScreenshots: config.autoScreenshots });
+        const storyTestFiles = await buildStoryTestFiles(stories, {
+            autoScreenshots: config.autoScreenshots,
+            autoScreenshotStorybookGlobals: config.autoScreenshotStorybookGlobals,
+        });
 
         patchTestplaneBaseUrl(testplane.config, iframeUrl);
         disableTestplaneIsolation(testplane.config, config.browserIds);

diff --git a/src/storybook/story-test-runner/extend-stories.test.ts b/src/storybook/story-test-runner/extend-stories.test.ts
@@ -10,7 +10,9 @@ describe("storybook/story-test-runner/extend-stories", () => {
 
         const expectedMsg = [
             '"testplane" section is ignored in storyfile "not/existing.ts", because the file could not be read:',
-            "Error: Cannot find module 'not/existing.ts' from 'src/storybook/story-test-runner/extend-stories.ts'",
+            "Error: Cannot find module 'not/existing.ts' from 'src/storybook/story-test-runner/extend-stories.ts' ",
+            "There could be other story files. ",
+            "Set 'TESTPLANE_STORYBOOK_DISABLE_STORY_REQUIRE_WARNING' environment variable to hide this warning",
         ].join("\n");
         expect(console.warn).toBeCalledWith(expectedMsg);
     });

diff --git a/src/storybook/story-test-runner/extend-stories.ts b/src/storybook/story-test-runner/extend-stories.ts
@@ -23,6 +23,7 @@ export function extendStoriesFromStoryFile(stories: StorybookStory[]): Storybook
             story.skip = false;
             story.assertViewOpts = {};
             story.browserIds = null;
+            story.autoScreenshotStorybookGlobals = {};
 
             return story;
         });
@@ -37,6 +38,7 @@ export function extendStoriesFromStoryFile(stories: StorybookStory[]): Storybook
                 story.assertViewOpts = testplaneStoryOpts.assertViewOpts || {};
                 story.browserIds = testplaneStoryOpts.browserIds || null;
                 story.autoscreenshotSelector = testplaneStoryOpts.autoscreenshotSelector || null;
+                story.autoScreenshotStorybookGlobals = testplaneStoryOpts.autoScreenshotStorybookGlobals || {};
             });
 
             continue;
@@ -86,6 +88,8 @@ function getStoryFile(storyPath: string): StoryFile | null {
             const warningMessage = [
                 `"testplane" section is ignored in storyfile "${storyPath}",`,
                 `because the file could not be read:\n${error}`,
+                "\nThere could be other story files.",
+                "\nSet 'TESTPLANE_STORYBOOK_DISABLE_STORY_REQUIRE_WARNING' environment variable to hide this warning",
             ].join(" ");
 
             console.warn(warningMessage);

diff --git a/src/storybook/story-test-runner/index.ts b/src/storybook/story-test-runner/index.ts
@@ -16,16 +16,36 @@ export function run(stories: StorybookStory[], opts: TestplaneOpts): void {
     withStoryFileDataStories.forEach(story => createTestplaneTests(story, opts));
 }
 
-function createTestplaneTests(story: StorybookStoryExtended, { autoScreenshots }: TestplaneOpts): void {
+function createTestplaneTests(
+    story: StorybookStoryExtended,
+    { autoScreenshots, autoScreenshotStorybookGlobals }: TestplaneOpts,
+): void {
     nestedDescribe(story, () => {
+        const rawAutoScreenshotGlobalSets = {
+            ...autoScreenshotStorybookGlobals,
+            ...story.autoScreenshotStorybookGlobals,
+        };
+
+        const screenshotGlobalSetNames = Object.keys(rawAutoScreenshotGlobalSets);
+
+        const autoScreenshotGlobalSets = screenshotGlobalSetNames.length
+            ? screenshotGlobalSetNames.map(name => ({ name, globals: rawAutoScreenshotGlobalSets[name] }))
+            : [{ name: "", globals: {} }];
+
         if (autoScreenshots) {
-            extendedIt(story, "Autoscreenshot", async function (ctx: TestFunctionExtendedCtx) {
-                ctx.expect = globalThis.expect;
+            for (const { name, globals } of autoScreenshotGlobalSets) {
+                extendedIt(
+                    story,
+                    `Autoscreenshot${name ? ` ${name}` : ""}`,
+                    async function (ctx: TestFunctionExtendedCtx) {
+                        ctx.expect = globalThis.expect;
 
-                const result = await openStoryStep(ctx.browser, story);
+                        const result = await openStoryStep(ctx.browser, story, globals);
 
-                await autoScreenshotStep(ctx.browser, story.autoscreenshotSelector || result.rootSelector);
-            });
+                        await autoScreenshotStep(ctx.browser, story.autoscreenshotSelector || result.rootSelector);
+                    },
+                );
+            }
         }
 
         if (story.extraTests) {
@@ -45,8 +65,12 @@ function createTestplaneTests(story: StorybookStoryExtended, { autoScreenshots }
     });
 }
 
-async function openStoryStep(browser: WebdriverIO.Browser, story: StorybookStoryExtended): Promise<StoryLoadResult> {
-    return browser.runStep("@testplane/storybook: open story", () => openStory(browser, story));
+async function openStoryStep(
+    browser: WebdriverIO.Browser,
+    story: StorybookStoryExtended,
+    storybookGlobals: Record<string, unknown> = {},
+): Promise<StoryLoadResult> {
+    return browser.runStep("@testplane/storybook: open story", () => openStory(browser, story, storybookGlobals));
 }
 
 async function autoScreenshotStep(browser: WebdriverIO.Browser, rootSelector: string): Promise<void> {

diff --git a/src/storybook/story-test-runner/open-story/index.ts b/src/storybook/story-test-runner/open-story/index.ts
@@ -1,9 +1,14 @@
 import { PlayFunctionError } from "../play-function-error";
 import testplaneOpenStory from "./testplane-open-story";
+export type { StorybookWindow } from "./testplane-open-story";
 import type { ExecutionContextExtended, StorybookStoryExtended } from "../types";
 import type { StoryLoadResult } from "./testplane-open-story";
 
-export async function openStory(browser: WebdriverIO.Browser, story: StorybookStoryExtended): Promise<StoryLoadResult> {
+export async function openStory(
+    browser: WebdriverIO.Browser,
+    story: StorybookStoryExtended,
+    storybookGlobals: Record<string, unknown>,
+): Promise<StoryLoadResult> {
     const browserConfig = await browser.getConfig();
     const currentBrowserUrl = await browser.getUrl();
 
@@ -24,7 +29,7 @@ export async function openStory(browser: WebdriverIO.Browser, story: StorybookSt
     await extendBrowserMeta(browser, story);
 
     return browser.runStep("wait story load", async (): Promise<StoryLoadResult> => {
-        const storyLoadResult = await testplaneOpenStory.execute(browser, story.id, shouldRemount);
+        const storyLoadResult = await testplaneOpenStory.execute(browser, story.id, storybookGlobals, shouldRemount);
 
         if (storyLoadResult.loadError) {
             throw new Error(storyLoadResult.loadError);

diff --git a/src/storybook/story-test-runner/open-story/testplane-open-story.ts b/src/storybook/story-test-runner/open-story/testplane-open-story.ts
@@ -11,10 +11,19 @@ interface HTMLElement {
     innerText: string;
 }
 
-type StorybookWindow = Window &
+export type StorybookWindow = Window &
     typeof globalThis & {
-        __HERMIONE_OPEN_STORY__: (storyId: string, remountOnly: boolean, done: (result: string) => void) => void;
-        __STORYBOOK_ADDONS_CHANNEL__: EventEmitter;
+        __HERMIONE_OPEN_STORY__: (
+            storyId: string,
+            storybookGlobals: Record<string, unknown>,
+            remountOnly: boolean,
+            done: (result: string) => void,
+        ) => void;
+        __TESTPLANE_STORYBOOK_INITIAL_GLOBALS__?: Record<string, unknown>;
+        __STORYBOOK_PREVIEW__?: { storeInitializationPromise?: Promise<void> };
+        __STORYBOOK_ADDONS_CHANNEL__: EventEmitter & {
+            data?: { setGlobals?: Array<{ globals: Record<string, unknown> }> };
+        };
         __STORYBOOK_ADDON_INTERACTIONS_INSTRUMENTER_STATE__: Record<
             string,
             {
@@ -34,10 +43,11 @@ export async function inject(browser: WebdriverIO.Browser): Promise<void> {
 export async function execute(
     browser: WebdriverIO.Browser,
     storyId: string,
+    storybookGlobals: Record<string, unknown>,
     shouldRemount: boolean,
 ): Promise<StoryLoadResult> {
     const getResult = (): Promise<StoryLoadResult> =>
-        browser.executeAsync(executeOpenStoryScript, storyId, shouldRemount).then(JSON.parse);
+        browser.executeAsync(executeOpenStoryScript, storyId, storybookGlobals, shouldRemount).then(JSON.parse);
 
     const result: StoryLoadResult = await getResult();
 
@@ -58,7 +68,12 @@ export async function execute(
 
 export default { inject, execute };
 
-function openStoryScript(storyId: string, shouldRemount: boolean, done: (result: string) => void): void {
+function openStoryScript(
+    storyId: string,
+    storybookGlobals: Record<string, unknown>,
+    shouldRemount: boolean,
+    done: (result: string) => void,
+): void {
     function onPageLoad(fn: () => void): void {
         if (document.readyState === "complete") {
             fn();
@@ -81,6 +96,7 @@ function openStoryScript(storyId: string, shouldRemount: boolean, done: (result:
             channel.off("storyMissing", onStoryMissing);
             channel.off("storyThrewException", onStoryThrewException);
             channel.off("storyErrored", onStoryErrored);
+            channel.off("globalsUpdated", onGlobalsUpdated);
 
             done(JSON.stringify(value));
         }
@@ -140,23 +156,59 @@ function openStoryScript(storyId: string, shouldRemount: boolean, done: (result:
             doneJson(result);
         }
 
-        channel.once("playFunctionThrewException", onPlayFunctionThrewException);
-        channel.once("storyRendered", onStoryRendered);
-        channel.once("storyMissing", onStoryMissing);
-        channel.once("storyThrewException", onStoryThrewException);
-        channel.once("storyErrored", onStoryErrored);
+        function onGlobalsUpdated(): void {
+            channel.once("playFunctionThrewException", onPlayFunctionThrewException);
+            channel.once("storyRendered", onStoryRendered);
+            channel.once("storyMissing", onStoryMissing);
+            channel.once("storyThrewException", onStoryThrewException);
+            channel.once("storyErrored", onStoryErrored);
 
-        if (shouldRemount) {
-            channel.emit("setCurrentStory", { storyId: "" });
+            if (shouldRemount) {
+                channel.emit("setCurrentStory", { storyId: "" });
+            }
+
+            channel.emit("setCurrentStory", { storyId });
         }
 
-        channel.emit("setCurrentStory", { storyId });
+        if (!channel) {
+            result.loadError = "Couldn't find storybook channel. Looks like the opened page is not storybook preview";
+
+            doneJson(result);
+        }
+
+        const storybookPreview = (window as StorybookWindow).__STORYBOOK_PREVIEW__;
+        const isStorybookPreviewAvailable = storybookPreview && storybookPreview.storeInitializationPromise;
+        const shouldUpdateStorybookGlobals = storybookGlobals && isStorybookPreviewAvailable;
+
+        if (shouldUpdateStorybookGlobals) {
+            (storybookPreview.storeInitializationPromise as Promise<void>).then(function () {
+                let defaultGlobals = (window as StorybookWindow).__TESTPLANE_STORYBOOK_INITIAL_GLOBALS__;
+
+                if (!defaultGlobals) {
+                    const setGlobalCalls = (window as StorybookWindow).__STORYBOOK_ADDONS_CHANNEL__.data?.setGlobals;
+                    const initValue = (setGlobalCalls && setGlobalCalls[0].globals) || {};
+
+                    defaultGlobals = (window as StorybookWindow).__TESTPLANE_STORYBOOK_INITIAL_GLOBALS__ = initValue;
+                }
+
+                channel.once("globalsUpdated", onGlobalsUpdated);
+
+                channel.emit("updateGlobals", { globals: Object.assign({}, defaultGlobals, storybookGlobals) });
+            });
+        } else {
+            onGlobalsUpdated();
+        }
     });
 }
 
-function executeOpenStoryScript(storyId: string, remountOnly: boolean, done: (result: string) => void): void {
+function executeOpenStoryScript(
+    storyId: string,
+    storybookGlobals: Record<string, unknown>,
+    remountOnly: boolean,
+    done: (result: string) => void,
+): void {
     if ((window as StorybookWindow).__HERMIONE_OPEN_STORY__) {
-        (window as StorybookWindow).__HERMIONE_OPEN_STORY__(storyId, remountOnly, done);
+        (window as StorybookWindow).__HERMIONE_OPEN_STORY__(storyId, storybookGlobals, remountOnly, done);
     } else {
         done(JSON.stringify({ notInjected: true }));
     }