Add ui-tests

.github/workflows/build.yml

@@ -12,33 +12,78 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.7", "3.10"]
+        os: [ubuntu-latest]
+        python-version: ["3.8", "3.12"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
 
-      - name: Install dependencies
-        run: |
-          python -m pip install -U codecov
-          npm install -g codecov
-      - name: Test the extension
+      - name: Install build dependencies
+        run: python -m pip install build jupyterlab==4
+
+      - name: Install yarn dependencies
         run: |
-          python -m pip install --upgrade -v -e ".[test, examples, docs]"
-          python -m pytest
-          yarn run test
+          jlpm install
+          jlpm run build
 
       - name: Linting
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: |
-          yarn run lint:check
+        run: jlpm run lint:check
 
-      - name: Check docs can be build + links
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        working-directory: docs
-        run: |
-          sudo apt install -y pandoc
-          make html
-          python -m pytest --check-links
+      - name: Build Python package
+        run: python -m build
+
+      - name: Upload builds
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist ${{ github.run_number }}
+          path: ./dist
+
+  visual-regression-tests:
+    runs-on: ubuntu-latest
+    needs: [build]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: dist ${{ github.run_number }}
+          path: ./dist
+
+      - name: Install dependencies
+        run: python -m pip install jupyterlab==4
+
+      - name: Install the package
+        run: pip install -vv ipyopenlayers*.whl
+        working-directory: dist
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        working-directory: ui-tests
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: jlpm install
+
+      - name: Install browser
+        shell: bash -l {0}
+        run: npx playwright install chromium
+        working-directory: ui-tests
+
+      - name: Execute integration tests
+        shell: bash -l {0}
+        working-directory: ui-tests
+        run: jlpm run test
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ipyopenlayers-playwright-tests
+          path: |
+            ui-tests/test-results
+            ui-tests/playwright-report

.github/workflows/update_galata_references.yaml

@@ -0,0 +1,59 @@
+name: Update Galata References
+
+on:
+  issue_comment:
+    types: [created, edited]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  update-reference-screenshots:
+    name: Update Galata References
+    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'update galata references') }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure git to use https
+        run: git config --global hub.protocol https
+
+      - name: Install hub
+        run: sudo apt-get update && sudo apt-get install -y hub
+
+      - name: Checkout the branch from the PR that triggered the job
+        run: hub pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/setup-micromamba@main
+        with:
+          environment-file: test-environment.yml
+          channels: conda-forge
+
+      - name: Install
+        run: pip install .
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        working-directory: ui-tests
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: jlpm install
+
+      - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@main
+        with:
+          npm_client: jlpm
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          start_server_script: 'null'
+          test_folder: ui-tests

ui-tests/.yarnrc.yml

@@ -0,0 +1,2 @@
+nodeLinker: node-modules
+enableImmutableInstalls: false

ui-tests/README.md

@@ -0,0 +1,36 @@
+# Visual regression tests using Galata
+
+This directory contains visual regression tests for ipyopenlayers, using Galata.
+
+In order to run them, you need to install dependencies:
+
+```bash
+conda install -c conda-forge jupyterlab=4
+
+jlpm install
+```
+
+Then start JupyterLab in one terminal (you need to check that it properly starts on port 8888):
+```bash
+jlpm run start-jlab
+```
+
+Finally, run the galata tests:
+```bash
+jlpm run test
+```
+
+If ipyopenlayers visuals change, you can re-generate reference images by running:
+```bash
+jlpm run update-references
+```
+
+## Notebooks directory
+
+The `tests/notebooks` directory contains the test notebooks. For most notebooks (*e.g.* `bars.ipynb`, `scatter.ipynb`) Galata will run them cell by cell and take a screenshot of each output, comparing with the reference images.
+
+When running notebooks named `*_update.ipynb`, Galata will always take the first cell output as reference which must contain the plot, later cells will only be used to update the plot, those notebooks are checking that ipyopenlayers is properly taking updates into account on already-created plots.
+
+## Add a new test
+
+You can add a new test by simply adding a new notebook to the `tests/notebooks` directory and updating the references. If you want to test updating plots, create notebook named `*_update.ipynb`, create a plot in your first cell then update the plot in later cells.

ui-tests/jupyter_server_config.py

@@ -0,0 +1,5 @@
+from jupyterlab.galata import configure_jupyter_server
+
+configure_jupyter_server(c)  # noqa F821
+# Uncomment to set server log level to debug level
+# c.ServerApp.log_level = "DEBUG"

ui-tests/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "ipyopenlayers-ui-tests",
+  "version": "1.0.0",
+  "description": "ipyopenlayers UI Tests",
+  "private": true,
+  "scripts": {
+    "start": "jupyter lab --config jupyter_server_config.py",
+    "test": "playwright test",
+    "test:update": "playwright test --update-snapshots",
+    "test:debug": "PWDEBUG=1 playwright test"
+  },
+  "author": "ipyopenlayers",
+  "license": "Apache-2.0",
+  "devDependencies": {
+    "@jupyterlab/galata": "^5.0.1",
+    "@playwright/test": "^1.32.0"
+  },
+  "dependencies": {
+    "@types/klaw-sync": "^6.0.1",
+    "klaw-sync": "^6.0.0"
+  }
+}

ui-tests/playwright.config.js

@@ -0,0 +1,15 @@
+/**
+ * Configuration for Playwright using default from @jupyterlab/galata
+ */
+const baseConfig = require('@jupyterlab/galata/lib/playwright-config');
+
+module.exports = {
+  ...baseConfig,
+  webServer: {
+    command: 'jlpm start',
+    url: 'http://localhost:8888/lab',
+    reuseExistingServer: !process.env.CI
+  },
+  timeout: 600000,
+  retries: 3
+};

ui-tests/tests/ipyopenlayers.test.ts

@@ -0,0 +1,136 @@
+// Copyright (c) Jupyter Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+import { expect, IJupyterLabPageFixture, test } from '@jupyterlab/galata';
+import * as path from 'path';
+const klaw = require('klaw-sync');
+
+
+const filterUpdateNotebooks = item => {
+  const basename = path.basename(item.path);
+  return basename.includes('_update');
+}
+
+const testCellOutputs = async (page: IJupyterLabPageFixture, tmpPath: string, theme: 'JupyterLab Light' | 'JupyterLab Dark') => {
+  const paths = klaw(path.resolve(__dirname, './notebooks'), {filter: item => !filterUpdateNotebooks(item), nodir: true});
+  const notebooks = paths.map(item => path.basename(item.path));
+
+  const contextPrefix = theme == 'JupyterLab Dark' ? 'light' : 'dark';
+  if (theme == 'JupyterLab Dark') {
+    page.theme.setTheme(theme);
+  }
+
+  for (const notebook of notebooks) {
+    let results = [];
+
+    await page.notebook.openByPath(`${tmpPath}/${notebook}`);
+    await page.notebook.activate(notebook);
+
+    let numCellImages = 0;
+
+    const getCaptureImageName = (contextPrefix: string, notebook: string, id: number): string => {
+      return `${contextPrefix}-${notebook}-cell-${id}.png`;
+    };
+
+    await page.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        const cell = await page.notebook.getCellOutput(cellIndex);
+        if (cell) {
+          results.push(await cell.screenshot());
+          numCellImages++;
+        }
+      }
+    });
+
+    await page.notebook.save();
+
+    for (let c = 0; c < numCellImages; ++c) {
+      expect(results[c]).toMatchSnapshot(getCaptureImageName(contextPrefix, notebook, c), {threshold: 0.3});
+    }
+
+    await page.notebook.close(true);
+  }
+}
+
+const testPlotUpdates = async (page: IJupyterLabPageFixture, tmpPath: string, theme: 'JupyterLab Light' | 'JupyterLab Dark') => {
+  const paths = klaw(path.resolve(__dirname, './notebooks'), {filter: item => filterUpdateNotebooks(item), nodir: true});
+  const notebooks = paths.map(item => path.basename(item.path));
+
+  const contextPrefix = theme == 'JupyterLab Light' ? 'light' : 'dark';
+  if (theme == 'JupyterLab Dark') {
+    page.theme.setTheme(theme);
+  }
+
+  for (const notebook of notebooks) {
+    let results = [];
+
+    await page.notebook.openByPath(`${tmpPath}/${notebook}`);
+    await page.notebook.activate(notebook);
+
+    const getCaptureImageName = (contextPrefix: string, notebook: string, id: number): string => {
+      return `${contextPrefix}-${notebook}-cell-${id}.png`;
+    };
+
+    let cellCount = 0;
+    await page.notebook.runCellByCell({
+      onAfterCellRun: async (cellIndex: number) => {
+        // Always get first cell output which must contain the plot
+        const cell = await page.notebook.getCellOutput(0);
+        if (cell) {
+          results.push(await cell.screenshot());
+          cellCount++;
+        }
+      }
+    });
+
+    await page.notebook.save();
+
+    for (let i = 0; i < cellCount; i++) {
+      expect(results[i]).toMatchSnapshot(getCaptureImageName(contextPrefix, notebook, i), {threshold: 0.3});
+    }
+
+    await page.notebook.close(true);
+  }
+};
+
+test.describe('ipyopenlayers Visual Regression', () => {
+  test.beforeEach(async ({ page, tmpPath }) => {
+    page.on("console", (message) => {
+      console.log('CONSOLE MSG ---', message.text());
+    });
+
+    await page.contents.uploadDirectory(
+      path.resolve(__dirname, './notebooks'),
+      tmpPath
+    );
+    await page.filebrowser.openDirectory(tmpPath);
+  });
+
+  test('Light theme: Check ipyopenlayers first renders', async ({
+    page,
+    tmpPath,
+  }) => {
+    await testCellOutputs(page, tmpPath, 'JupyterLab Light');
+  });
+
+  // test('Dark theme: Check ipyopenlayers first renders', async ({
+  //   page,
+  //   tmpPath,
+  // }) => {
+  //   await testCellOutputs(page, tmpPath, 'JupyterLab Dark');
+  // });
+
+  test('Light theme: Check ipyopenlayers update plot properties', async ({
+    page,
+    tmpPath,
+  }) => {
+    await testPlotUpdates(page, tmpPath, 'JupyterLab Light');
+  });
+
+  // test('Dark theme: Check ipyopenlayers update plot properties', async ({
+  //   page,
+  //   tmpPath,
+  // }) => {
+  //   await testPlotUpdates(page, tmpPath, 'JupyterLab Dark');
+  // });
+});