ci: add e2e tests

.github/workflows/e2e.yaml

@@ -0,0 +1,58 @@
+# Copyright 2024 Docker Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: E2E tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  e2e:
+    name: Run E2E tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install and start Docker Desktop
+        uses: docker/desktop-action/[email protected]
+
+      - name: Change Desktop settings and allow installing non marketplace extensions
+        run: |
+          jq '.onlyMarketplaceExtensions|=false' ~/.docker/desktop/settings.json >./new-settings.json
+          mv -f ./new-settings.json ~/.docker/desktop/settings.json
+
+      - name: Build extension
+        run: |
+          docker build --tag=projectcopacetic/copacetic-docker-desktop-extension:test .
+
+      - name: Kill Mono process running on port 8084 to unblock it # https://github.com/actions/runner-images/issues/2821
+        shell: bash
+        run: |
+          sudo fuser -k -n tcp 8084
+      - name: Enable corepack
+        run: corepack enable
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Run E2E tests
+        env:
+          SKIP_EXTENSION_IMAGE_BUILD: true
+        run: |
+          cd e2e
+          yarn install
+          xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" -- yarn test:e2e

e2e/.babelrc.json

@@ -0,0 +1,14 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "node": "14.17"
+        }
+      }
+    ],
+    "@babel/preset-typescript"
+  ],
+  "plugins": []
+}

e2e/.eslintrc

@@ -0,0 +1,24 @@
+{
+  "extends": [
+    "eslint:recommended",
+    "prettier"
+  ],
+  "parser": "babel-eslint",
+  "env": {
+    "es6": true,
+    "node": true,
+    "jest": true,
+    "browser": true
+  },
+  "plugins": [
+    "prettier"
+  ],
+  "rules": {
+    "prettier/prettier": [
+      "error"
+    ]
+  },
+  "globals": {
+    "page": true
+  }
+}

e2e/.gitignore

@@ -0,0 +1,3 @@
+screenshots
+.yarn
+node_modules

e2e/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all",
+  "tabWidth": 2
+}

e2e/.yarnrc.yml

@@ -0,0 +1,15 @@
+# Copyright 2024 Docker Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodeLinker: node-modules

e2e/e2e.yaml

@@ -0,0 +1,58 @@
+# Copyright 2024 Docker Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: E2E tests
+
+on:
+  push:
+    branches:
+      - ci-testing-workflow
+  pull_request:
+
+jobs:
+  e2e:
+    name: Run E2E tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install and start Docker Desktop
+        uses: docker/desktop-action/[email protected]
+
+      - name: Change Desktop settings and allow installing non marketplace extensions
+        run: |
+          jq '.onlyMarketplaceExtensions|=false' ~/.docker/desktop/settings.json >./new-settings.json
+          mv -f ./new-settings.json ~/.docker/desktop/settings.json
+
+      - name: Build extension
+        run: |
+          make build-extension TAG=test
+
+      - name: Kill Mono process running on port 8084 to unblock it # https://github.com/actions/runner-images/issues/2821
+        shell: bash
+        run: |
+          sudo fuser -k -n tcp 8084
+      - name: Enable corepack
+        run: corepack enable
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+      - name: Run E2E tests
+        env:
+          SKIP_EXTENSION_IMAGE_BUILD: true
+        run: |
+          cd e2e
+          yarn install
+          xvfb-run --auto-servernum --server-args="-screen 0 1920x1080x24" -- yarn test:e2e

e2e/globals.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Copyright 2024 Docker Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from 'puppeteer-core';
+
+declare global {
+  var page: Page | null;
+}

e2e/jest.config.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2024 Docker Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = {
+  verbose: true,
+  testPathIgnorePatterns: ['/node_modules/'],
+  testEnvironment: 'node',
+  preset: 'ts-jest',
+  testTimeout: 360000,
+};

e2e/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "electron-e2e",
+  "version": "1.0.0",
+  "description": "Testcafe setup to run docker desktop e2e tests",
+  "author": "docker",
+  "packageManager": "[email protected]",
+  "license": "ISC",
+  "devDependencies": {
+    "@babel/preset-env": "^7.24.8",
+    "@babel/preset-typescript": "^7.24.7",
+    "@docker/extension-test-helper": "^0.1.2",
+    "@jest/test-sequencer": "^29.7.0",
+    "@tsconfig/node16": "^16.1.3",
+    "@types/dockerode": "^3.3.30",
+    "@types/jest": "^29.5.12",
+    "@types/mkdirp": "^2.0.0",
+    "axios": "^1.7.2",
+    "babel-eslint": "^10.1.0",
+    "cross-env": "^7.0.3",
+    "dockerode": "^4.0.2",
+    "electron": "31.2.1",
+    "eslint": "^9.7.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-prettier": "^5.2.1",
+    "jest": "^29.7.0",
+    "prettier": "^3.3.3",
+    "ts-jest": "^29.2.3",
+    "typescript": "^4.5.5"
+  },
+  "dependencies": {
+    "@rumpl/node-eventsource": "^1.0.0",
+    "@types/bytes": "^3.1.4",
+    "bytes": "^3.1.2",
+    "find-process": "^1.4.4",
+    "fkill": "^9.0.0",
+    "get-port": "^7.1.0",
+    "got": "^14.4.1",
+    "mkdirp": "^3.0.1",
+    "puppeteer": "^22.13.1",
+    "puppeteer-core": "^22.13.1"
+  },
+  "scripts": {
+    "test:e2e": "yarn run test:e2e:packaged",
+    "test:e2e:packaged": "cross-env NODE_ENV=test WHICH_INSTANCE=packaged jest --colors --bail --runInBand --forceExit",
+    "pretest:e2e:dev": "(cd ../../client/desktop && yarn build:development)",
+    "test:e2e:dev": "cross-env NODE_ENV=test WHICH_INSTANCE=dev jest --colors --bail --runInBand --forceExit",
+    "test:puppeteer:debug": "cross-env NODE_ENV=test npx ndb jest --colors --bail --runInBand --forceExit",
+    "test:puppeteer:verbose": "cross-env NODE_ENV=test DEBUG=puppeteer:* jest --colors --bail --runInBand --forceExit",
+    "lint": "yarn run lint:js && yarn run check:ts",
+    "lint:js": "eslint .",
+    "check:ts": "tsc --noEmit",
+    "prettier:check": "prettier --check \"**/*.js\" \"**/*.ts\"",
+    "prettier": "prettier --write \"**/*.js\" \"**/*.ts\""
+  }
+}

e2e/test-copa-extension.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Copyright 2024 Docker Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { DesktopUI } from '@docker/extension-test-helper';
+import { exec as originalExec } from 'child_process';
+import { describe, beforeAll, afterAll, test } from '@jest/globals';
+import * as util from 'util';
+import { WaitForSelectorOptions } from 'puppeteer-core';
+
+export const exec = util.promisify(originalExec);
+
+let dashboard: DesktopUI;
+
+beforeAll(async () => {
+  if (process.env.SKIP_EXTENSION_IMAGE_BUILD != 'true') {
+    console.log('starting building extension...');
+    await exec(`projectcopacetic/copacetic-docker-desktop-extension:test`, {
+      cwd: '../',
+    });
+    console.log('extension built');
+  }
+  // await exec(
+  //   `docker container create --name e2e-test-container -v e2e-test-volume:/data hello-world`,
+  // );
+  // console.log('sample volume and container created');
+  await exec(
+    `docker extension install -f projectcopacetic/copacetic-docker-desktop-extension:test`,
+  );
+  console.log('extension installed');
+});
+
+afterAll(async () => {
+  await dashboard?.stop();
+  console.log('dashboard app closed');
+  await exec(`docker extension uninstall projectcopacetic/copacetic-docker-desktop-extension:test`);
+  console.log('extension uninstalled');
+});
+
+describe('Test Main Workflow of Extension', () => {
+  test('display the patched image', async () => {
+    dashboard = await DesktopUI.start();
+
+    const eFrame = await dashboard.navigateToExtension(
+      'projectcopacetic/copacetic-docker-desktop-extension',
+    );
+
+    const imageInput = await eFrame.waitForSelector("#image-select-combo-box");
+
+    console.log('input nginx:1.21.6 image');
+    await imageInput?.type('nginx:1.21.6');
+    await imageInput?.dispose();
+
+    const scanPatchImageButton = await eFrame.waitForSelector('#scan-or-patch-image-button');
+
+    console.log('click scan button');
+    await scanPatchImageButton?.click();
+
+    const vulnDisplay = await eFrame.waitForSelector('#loaded-vuln-display-page', { timeout: 120000 });
+    console.log('scan finished (vulns being displayed)')
+    await vulnDisplay?.dispose();
+
+    console.log('click patch button');
+    await scanPatchImageButton?.click();
+    await scanPatchImageButton?.dispose();
+
+    const loadingText = await eFrame.waitForSelector('#loading-patch-text');
+    console.log('confirm loading started');
+    await loadingText?.dispose();
+
+    const patchedImageElement = await eFrame.waitForSelector('#new-patched-image-name-text', { timeout: 240000 });
+    console.log("scan finished (success)");
+    let patchedImageText = "";
+    const evalText = await patchedImageElement?.evaluate(element => element.textContent);
+    if (evalText !== null && evalText !== undefined) {
+      patchedImageText = evalText;
+      console.log('created patched image: ' + evalText);
+    }
+    await patchedImageElement?.dispose();
+
+    expect(patchedImageText).toBe("nginx:1.21.6-patched!");
+
+    const output = await exec(`docker images`);
+    console.log(output.stdout);
+
+    expect(output.stdout.includes('nginx')).toBe(true);
+    expect(output.stdout.includes('1.21.6-patched')).toBe(true);
+  });
+});