test: write workflow snapshot tests

.circleci/config.yml

@@ -22,6 +22,10 @@ executors:
       - image: cimg/node:20.13.1
     resource_class: medium
     working_directory: ~/project
+  ci-macos:
+    macos:
+      xcode: 15.0.0  # Use the closest available version of macOS (Sonoma equivalent may not be available yet)
+    working_directory: ~/project
 
 commands:
   attach_project:
@@ -105,6 +109,30 @@ jobs:
           no_output_timeout: 15m
           name: Run test
           command: yarn test
+  # CI - Run snapshot tests
+  run-snapshot-test:
+    executor: ci-macos
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /Users/distiller/project
+      - run:
+          name: Enable Corepack
+          command: corepack enable
+      - run:
+          name: Install dependencies
+          command: |
+            git submodule update --init --recursive
+            yarn install --immutable
+      - run:
+          name: Install Playwright browsers
+          command: npx playwright install --with-deps
+      - run:
+          name: Run Playwright snapshot tests
+          command: yarn playwright test
+      - store_artifacts:
+          path: /Users/distiller/project/test-results
+          destination: playwright-test-results
 
   # Publish - build self-service
   build:
@@ -201,6 +229,9 @@ workflows:
       - run-test:
           requires:
             - prepare
+  ci-snapshot-test:
+    jobs:
+      - run-snapshot-test
 
   deploy_prod:
     when: << pipeline.parameters.run_deploy_prod >>

.gitignore

@@ -35,3 +35,7 @@ dist-ssr
 *.njsproj
 *.sln
 *.sw?
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

__visual_tests__/const.ts

@@ -0,0 +1,15 @@
+const appId = process.env.SNAPSHOT_TEST_APP_ID;
+const botId = process.env.SNAPSHOT_TEST_BOT_ID;
+
+export const TEST_URL = `http://localhost:5173/chat-ai-widget/?app_id=${appId}&bot_id=${botId}&snapshot=true`;
+
+export const WidgetComponentIds = {
+  WIDGET: '#aichatbot-widget-window',
+  WIDGET_BUTTON: '#aichatbot-widget-button',
+  MESSAGE_INPUT: '#sendbird-message-input-text-field',
+  SUGGESTED_REPLIES_OPTIONS: '.sendbird-suggested-replies__option',
+  BUTTON: 'button.sendbird-button--primary',
+  INPUT: '.sendbird-input__input',
+  CHIPS_CONTAINER: '.sendbird-form-chip__container',
+  FORM: '#aichatbot-widget-form',
+};

__visual_tests__/utils.ts

@@ -0,0 +1,34 @@
+import {expect, Page} from "@playwright/test";
+
+import {WidgetComponentIds} from "./const";
+
+export async function assertScreenshot(page: Page, screenshotName: string, browserName: string) {
+  const name = `${screenshotName}.${browserName}.${process.platform}.png`; // Include the browser and OS architecture info in the filename
+  await expect(page.locator(WidgetComponentIds.WIDGET)).toHaveScreenshot(
+    name,
+    {
+      omitBackground: false,
+      maxDiffPixelRatio: 0.01, // Need this because Sendbird logo is slightly differently rendered in CI.
+    }
+  );
+}
+
+export async function loadWidget(page: Page) {
+  await page.click(WidgetComponentIds.WIDGET_BUTTON);
+  // NOTE: below fails sometimes in CI.
+  const widgetWindow = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await widgetWindow.waitFor({ state: 'visible' });
+  // await page.waitForTimeout(3000);
+}
+
+export async function sendTextMessage(page: Page, text: string, waitTime = 1000) {
+  const input = page.locator(WidgetComponentIds.MESSAGE_INPUT);
+  await input.fill(text);
+  await input.press('Enter');
+  await page.waitForTimeout(waitTime);
+}
+
+export async function clickNthChip(page: Page, nth: number) {
+  const chipContainer = page.locator(WidgetComponentIds.CHIPS_CONTAINER);
+  await chipContainer.locator(':scope > *').nth(nth).click();
+}

__visual_tests__/workflow-tests.spec.ts

@@ -0,0 +1,130 @@
+import {test} from '@playwright/test';
+
+import {TEST_URL, WidgetComponentIds} from "./const";
+import {assertScreenshot, clickNthChip, loadWidget, sendTextMessage} from "./utils";
+
+
+test.beforeEach(async ({ page }) => {
+  await page.goto(TEST_URL);
+  const widgetWindow = page.locator(WidgetComponentIds.WIDGET_BUTTON);
+  await widgetWindow.waitFor({ state: 'visible' });
+});
+
+/**
+ * 100
+ * Workflow - Form message
+ * Steps:
+ * 1. Send the trigger message: "Give me a food order form"
+ * 2. Submit form without filling the required fields.
+ * 3. Submit form with at least one invalid value.
+ * 4. Submit form with valid values.
+ */
+test('100', async ({ page, browserName }) => {
+  await loadWidget(page);
+
+  // 1
+  await sendTextMessage(page, 'Give me a food order form', 0);
+  const widgetWindow = page.locator(WidgetComponentIds.FORM);
+  await widgetWindow.waitFor({ state: 'visible' });
+  await assertScreenshot(page, '100-1', browserName);
+
+  // 2
+  let submitButton = page.locator(WidgetComponentIds.BUTTON);
+  await submitButton.click();
+  await assertScreenshot(page, '100-2', browserName);
+
+  // 3
+  const inputs = page.locator(WidgetComponentIds.INPUT);
+  await inputs.nth(0).fill('guy ordering food');
+  await inputs.nth(2).fill('not a number');
+  await inputs.nth(3).fill('not.a.valid.email.com');
+  await inputs.nth(4).fill('123_456_7890');
+  await clickNthChip(page, 4);
+  submitButton = page.locator(WidgetComponentIds.BUTTON);
+  await page.waitForTimeout(1000);
+  await assertScreenshot(page, '100-3', browserName);
+
+  // 4
+  await inputs.nth(2).fill('2');
+  await inputs.nth(3).fill('[email protected]');
+  await inputs.nth(4).fill('123-456-7890');
+  await submitButton.click();
+  await page.waitForTimeout(1000);
+  await assertScreenshot(page, '100-4', browserName);
+});
+
+/**
+ * 101
+ * Workflow - Function calls: user message
+ * Steps:
+ * 1. Send the trigger message: "Tell me about one cat breed"
+ */
+test('101', async ({ page, browserName }) => {
+  await loadWidget(page);
+  // 1
+  await sendTextMessage(page, 'Tell me about one cat breed', 2000);
+  await assertScreenshot(page, '101-1', browserName);
+});
+
+/**
+ * 102
+ * Workflow - File message
+ * Steps:
+ * 1. Send the trigger message: "Give me a travel agency poster"
+ */
+test('102', async ({ page, browserName }) => {
+  await loadWidget(page);
+  // 1
+  await sendTextMessage(page, 'Give me a travel agency poster', 5000);
+  await assertScreenshot(page, '102-1', browserName);
+});
+
+/**
+ * 103
+ * Workflow - Suggested replies with 'Back' enabled
+ * Steps:
+ * 1. Send the trigger message: "Suggested replies"
+ * 2. Click "Text"
+ * 3. Click "Back"
+ * 4. Click "File"
+ * 5. Click "Back"
+ * 6. Click "Link to workflow: form message"
+ */
+test('103', async ({ page, browserName }) => {
+  await loadWidget(page);
+  // 1
+  await sendTextMessage(page, 'Suggested replies', 2000);
+  await assertScreenshot(page, '103-1', browserName);
+
+  // 2
+  let options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await options.nth(0).click();
+  await page.waitForTimeout(1000);
+  await assertScreenshot(page, '103-2', browserName);
+
+  // 3
+  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await options.nth(0).click();
+  await page.waitForTimeout(1000);
+  await assertScreenshot(page, '103-3', browserName);
+
+  // 4
+  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await options.nth(1).click();
+  await page.waitForTimeout(4000); // Time takes long for file message to be rendered and then scrolled to bottom in CI browsers.
+  await assertScreenshot(page, '103-4', browserName);
+
+  // 5
+  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await options.nth(0).click();
+  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  // Expecting three options.
+  await options.nth(2).waitFor({ state: 'visible' });
+  await assertScreenshot(page, '103-5', browserName);
+
+  // 6
+  options = page.locator(WidgetComponentIds.SUGGESTED_REPLIES_OPTIONS);
+  await options.nth(2).click();
+  await page.waitForTimeout(1000);
+  await assertScreenshot(page, '103-6', browserName);
+});

package.json

@@ -37,7 +37,9 @@
     "@linaria/atomic": "^6.2.0",
     "@linaria/core": "^6.2.0",
     "@linaria/react": "^6.2.1",
+    "@playwright/test": "^1.48.1",
     "@types/dompurify": "^3.0.5",
+    "@types/node": "^22.7.9",
     "@types/react": "^18.0.37",
     "@types/react-dom": "^18.0.11",
     "@types/styled-components": "^5.1.26",

playwright.config.ts

@@ -0,0 +1,80 @@
+import {defineConfig, devices} from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './__visual_tests__',
+  snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}{ext}', // Refer to: https://playwright.dev/docs/next/api/class-testproject#test-project-snapshot-path-template
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: 0, // process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: undefined, // process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] }, // Note we cannot use ...devices['Desktop Chrome'] because the name varies between devices and in CircleCI environment. CI test will fail because of this.
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] }, // Note we cannot use ...devices['Desktop Firefox'] because the name varies between devices and in CircleCI environment. CI test will fail because of this.
+    },
+
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: 'yarn dev',
+    url: 'http://localhost:5173/chat-ai-widget/',
+    reuseExistingServer: !process.env.CI,
+  },
+});

src/components/BotMessageWithBodyInput.tsx

@@ -54,7 +54,7 @@ const HEIGHTS = {
 
 export default function BotMessageWithBodyInput(props: Props) {
   const { botUser } = useChatContext();
-  const { botStudioEditProps, dateLocale } = useConstantState();
+  const { botStudioEditProps, dateLocale, stringSet } = useConstantState();
 
   const { createdAt, bodyComponent, chainTop, chainBottom, messageFeedback, wideContainer = false } = props;
 
@@ -86,10 +86,16 @@ export default function BotMessageWithBodyInput(props: Props) {
         <Content>
           {bodyComponent}
           {!wideContainer && !!createdAt && (
-            <DefaultSentTime>{formatCreatedAtToAMPM(createdAt, dateLocale)}</DefaultSentTime>
+            <DefaultSentTime>
+              {formatCreatedAtToAMPM(createdAt, stringSet.DATE_FORMAT__MESSAGE_TIMESTAMP, dateLocale)}
+            </DefaultSentTime>
           )}
         </Content>
-        {wideContainer && !!createdAt && <WideSentTime>{formatCreatedAtToAMPM(createdAt, dateLocale)}</WideSentTime>}
+        {wideContainer && !!createdAt && (
+          <WideSentTime>
+            {formatCreatedAtToAMPM(createdAt, stringSet.DATE_FORMAT__MESSAGE_TIMESTAMP, dateLocale)}
+          </WideSentTime>
+        )}
         {displayProfileImage && messageFeedback}
       </FullBodyContainer>
     </Root>

src/components/MyMessageStatus.tsx

@@ -5,6 +5,7 @@ import { Locale } from 'date-fns';
 import { useTheme } from 'styled-components';
 
 import { DefaultSentTime } from './MessageComponent';
+import { useConstantState } from '../context/ConstantContext';
 import { Icon } from '../foundation/components/Icon';
 import { Loader } from '../foundation/components/Loader';
 import { formatCreatedAtToAMPM } from '../utils/messageTimestamp';
@@ -16,6 +17,7 @@ interface MyMessageStatusProps {
 
 export default function MyMessageStatus(props: MyMessageStatusProps) {
   const { message, dateLocale } = props;
+  const { stringSet } = useConstantState();
   const theme = useTheme();
 
   switch (message.sendingStatus) {
@@ -32,7 +34,11 @@ export default function MyMessageStatus(props: MyMessageStatusProps) {
         </div>
       );
     default:
-      return <DefaultSentTime>{formatCreatedAtToAMPM(message.createdAt, dateLocale)}</DefaultSentTime>;
+      return (
+        <DefaultSentTime>
+          {formatCreatedAtToAMPM(message.createdAt, stringSet.DATE_FORMAT__MESSAGE_TIMESTAMP, dateLocale)}
+        </DefaultSentTime>
+      );
   }
 }