Automate snapshot test in CircleCI

.circleci/config.yml

@@ -22,6 +22,10 @@ executors:
       - image: cimg/node:20.13.1
     resource_class: medium
     working_directory: ~/project
+  ci-macos:
+    macos:
+      xcode: 15.0.0  # Use the closest available version of macOS (Sonoma equivalent may not be available yet)
+    working_directory: ~/project
 
 commands:
   attach_project:
@@ -75,10 +79,31 @@ jobs:
           paths:
             - node_modules
             - packages/self-service/node_modules
-            # - packages/uikit/node_modules --> submodules are not cached
       - persist_to_workspace:
           root: .
           paths: .
+
+  # CI - install dependencies for macOS
+  prepare-macos:
+    executor: ci-macos
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /Users/distiller/project
+      - run:
+          name: Install dependencies
+          command: |
+            git submodule update --init --recursive
+            yarn install --immutable
+      - save_cache:
+          key: v2-dependencies-{{ checksum "yarn.lock" }}
+          paths:
+            - node_modules
+            - packages/self-service/node_modules
+      - persist_to_workspace:
+          root: /Users/distiller/project
+          paths: .
+
   # CI - check format
   run-format:
     executor: ci-node
@@ -105,6 +130,31 @@ jobs:
           no_output_timeout: 15m
           name: Run test
           command: yarn test
+  # CI - Run snapshot tests
+  run-snapshot-test:
+    executor: ci-macos
+    steps:
+      - attach_project
+      - run:
+          name: Clean and reinstall dependencies
+          command: |
+            rm -rf node_modules package-lock.json
+            yarn install
+      - run:
+          name: Install Playwright dependencies
+          command: sudo yarn playwright install-deps
+      - run:
+          name: Install Playwright browsers
+          command: yarn playwright install
+      - run:
+          name: List all __visual_tests__ snapshots
+          command: find /Users/distiller/project/__visual_tests__ -type f
+      - run:
+          name: Run Playwright snapshot tests
+          command: SNAPSHOT_TEST_APP_ID=$SNAPSHOT_TEST_APP_ID SNAPSHOT_TEST_BOT_ID=$SNAPSHOT_TEST_BOT_ID yarn playwright test
+      - store_artifacts:
+          path: /Users/distiller/project/test-results
+          destination: playwright-test-results
 
   # Publish - build self-service
   build:
@@ -201,6 +251,12 @@ workflows:
       - run-test:
           requires:
             - prepare
+  ci-snapshot-test:
+    jobs:
+      - prepare-macos
+      - run-snapshot-test:
+          requires:
+            - prepare-macos
 
   deploy_prod:
     when: << pipeline.parameters.run_deploy_prod >>

__visual_tests__/const.ts

@@ -1,4 +1,8 @@
-export const TEST_URL = 'http://localhost:5173/chat-ai-widget/?app_id=833E2DC4-DFA2-4508-A283-6E5C7BFCF18A&bot_id=onboarding_bot&disable_timestamps=true';
+const appId = process.env.SNAPSHOT_TEST_APP_ID;
+const botId = process.env.SNAPSHOT_TEST_BOT_ID;
+
+export const TEST_URL = `http://localhost:5173/chat-ai-widget/?app_id=${appId}&bot_id=${botId}&disable_timestamps=true`;
+// export const TEST_URL = `http://localhost:5173/chat-ai-widget/?app_id=833E2DC4-DFA2-4508-A283-6E5C7BFCF18A&bot_id=onboarding_bot&disable_timestamps=true`;
 
 export const WidgetComponentIds = {
   WIDGET: '#aichatbot-widget-window',

__visual_tests__/utils.ts

@@ -2,13 +2,15 @@ import {expect, Page} from "@playwright/test";
 
 import {WidgetComponentIds} from "./const";
 
-export async function assertScreenshot(page: Page, screenshotName: string) {
+export async function assertScreenshot(page: Page, screenshotName: string, browserName: string) {
+  const name = `${screenshotName}.${browserName}.png`; // Include the browser name in the filename
   await expect(page.locator(WidgetComponentIds.WIDGET)).toHaveScreenshot(
-    `${screenshotName}.png`,
+    name,
     {
       omitBackground: false,
-      // threshold: 0.1, // Keep this in case you need it. It is for letting tests with little difference in pixels to pass.
-    });
+      maxDiffPixelRatio: 0.01, // Uncomment if you need a pixel threshold
+    }
+  );
 }
 
 export async function loadWidget(page: Page) {

__visual_tests__/workflow-tests.spec.ts

@@ -6,67 +6,7 @@ import {assertScreenshot, clickNthChip, loadWidget, sendTextMessage} from "./uti
 
 test.beforeEach(async ({ page }) => {
   await page.goto(TEST_URL);
-});
-
-/**
- * 001
- * Test workflow1
- * Scenario:
- * 1. Trigger workflow1
- * 2. Get form message response
- * 3. Submit without filling the form
- * 4. Fill form message and then submit
- * 5. Go back // If possible to remove go back, remove it.
- * 6. Get text message response
- * Verify after each step
- */
-test.skip('001', async ({ page, browserName }) => {
-  await page.click(WidgetComponentIds.WIDGET_BUTTON);
   await page.waitForTimeout(2500);
-  const input = page.locator(WidgetComponentIds.MESSAGE_INPUT);
-
-  // 1
-  await input.fill('trigger workflow1');
-  await assertScreenshot(page, `100-1.${browserName}`);
-  await input.press('Enter');
-  await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-2.${browserName}`);
-
-  // 2
-  let options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
-  await options.first().click();
-  await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-3.${browserName}`);
-
-  // 3
-  let submitButton = page.locator(WidgetComponentIds.BUTTON);
-  await submitButton.click();
-  await assertScreenshot(page, `100-4.${browserName}`);
-
-  // 4
-  const inputs = page.locator(WidgetComponentIds.INPUT);
-  await inputs.nth(0).fill('guy ordering food');
-  await inputs.nth(2).fill('2');
-  await inputs.nth(3).fill('[email protected]');
-  await inputs.nth(4).fill('123-456-7890');
-  const chipContainer = page.locator(WidgetComponentIds.CHIPS_CONTAINER);
-  await chipContainer.locator(':scope > *').nth(5).click();
-  submitButton = page.locator(WidgetComponentIds.BUTTON);
-  await submitButton.click();
-  await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-5.${browserName}`);
-
-  // 5
-  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
-  await options.first().click();
-  await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-6.${browserName}`);
-
-  // 6
-  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
-  await options.nth(1).click();
-  await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-7.${browserName}`);
 });
 
 /**
@@ -80,14 +20,15 @@ test.skip('001', async ({ page, browserName }) => {
  */
 test('100', async ({ page, browserName }) => {
   await loadWidget(page);
+
   // 1
   await sendTextMessage(page, 'Give me a food order form');
-  await assertScreenshot(page, `100-1.${browserName}`);
+  await assertScreenshot(page, '100-1', browserName);
 
   // 2
   let submitButton = page.locator(WidgetComponentIds.BUTTON);
   await submitButton.click();
-  await assertScreenshot(page, `100-2.${browserName}`);
+  await assertScreenshot(page, '100-2', browserName);
 
   // 3
   const inputs = page.locator(WidgetComponentIds.INPUT);
@@ -98,15 +39,15 @@ test('100', async ({ page, browserName }) => {
   await clickNthChip(page, 4);
   submitButton = page.locator(WidgetComponentIds.BUTTON);
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-3.${browserName}`);
+  await assertScreenshot(page, '100-3', browserName);
 
   // 4
   await inputs.nth(2).fill('2');
   await inputs.nth(3).fill('[email protected]');
   await inputs.nth(4).fill('123-456-7890');
   await submitButton.click();
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `100-4.${browserName}`);
+  await assertScreenshot(page, '100-4', browserName);
 });
 
 /**
@@ -118,8 +59,8 @@ test('100', async ({ page, browserName }) => {
 test('101', async ({ page, browserName }) => {
   await loadWidget(page);
   // 1
-  await sendTextMessage(page, 'Tell me about one cat breed');
-  await assertScreenshot(page, `101-1.${browserName}`);
+  await sendTextMessage(page, 'Tell me about one cat breed', 2000);
+  await assertScreenshot(page, '101-1', browserName);
 });
 
 /**
@@ -132,7 +73,7 @@ test('102', async ({ page, browserName }) => {
   await loadWidget(page);
   // 1
   await sendTextMessage(page, 'Give me a travel agency poster', 4000);
-  await assertScreenshot(page, `102-1.${browserName}`);
+  await assertScreenshot(page, '102-1', browserName);
 });
 
 /**
@@ -150,35 +91,35 @@ test('103', async ({ page, browserName }) => {
   await loadWidget(page);
   // 1
   await sendTextMessage(page, 'Suggested replies', 2000);
-  await assertScreenshot(page, `103-1.${browserName}`);
+  await assertScreenshot(page, '103-1', browserName);
 
   // 2
   let options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
   await options.nth(0).click();
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `103-2.${browserName}`);
+  await assertScreenshot(page, '103-2', browserName);
 
   // 3
   options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
   await options.nth(0).click();
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `103-3.${browserName}`);
+  await assertScreenshot(page, '103-3', browserName);
 
   // 4
   options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
   await options.nth(1).click();
-  await page.waitForTimeout(2000);
-  await assertScreenshot(page, `103-4.${browserName}`);
+  await page.waitForTimeout(3000);
+  await assertScreenshot(page, '103-4', browserName);
 
   // 5
   options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
   await options.nth(0).click();
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `103-5.${browserName}`);
+  await assertScreenshot(page, '103-5', browserName);
 
   // 6
   options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
   await options.nth(2).click();
   await page.waitForTimeout(1000);
-  await assertScreenshot(page, `103-6.${browserName}`);
+  await assertScreenshot(page, '103-6', browserName);
 });

package.json

@@ -1,6 +1,7 @@
 {
   "name": "@sendbird/chat-ai-widget",
   "version": "1.8.5",
+  "private": true,
   "description": "Sendbird Chat AI Widget,\n Detailed documentation can be found at https://github.com/sendbird/chat-ai-widget#readme",
   "main": "./dist/index.umd.js",
   "module": "./dist/index.es.js",

playwright.config.ts

@@ -1,4 +1,4 @@
-import { defineConfig, devices } from '@playwright/test';
+import {defineConfig, devices} from '@playwright/test';
 
 /**
  * Read environment variables from file.
@@ -13,14 +13,15 @@ import { defineConfig, devices } from '@playwright/test';
  */
 export default defineConfig({
   testDir: './__visual_tests__',
+  snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}', // Refer to: https://playwright.dev/docs/next/api/class-testproject#test-project-snapshot-path-template
   /* Run tests in files in parallel */
   fullyParallel: true,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: process.env.CI ? 2 : 0,
+  retries: 0, // process.env.CI ? 2 : 0,
   /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  workers: undefined, // process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -36,12 +37,12 @@ export default defineConfig({
   projects: [
     {
       name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
+      use: { ...devices['Desktop Chrome'] }, // Note we cannot use ...devices['Desktop Chrome'] because the name varies between devices and in CircleCI environment. CI test will fail because of this.
     },
 
     {
       name: 'firefox',
-      use: { ...devices['Desktop Firefox'] },
+      use: { ...devices['Desktop Firefox'] }, // Note we cannot use ...devices['Desktop Firefox'] because the name varies between devices and in CircleCI environment. CI test will fail because of this.
     },
 
     // {

tsconfig.json

@@ -29,6 +29,7 @@
     "src",
     "./src/custom.d.ts",
     "./src/styled-components.d.ts",
+    "__visual_tests__",
   ],
   "exclude": ["node_modules"],
   "references": [{ "path": "./tsconfig.node.json" }]