Move multi-app-backup-and-restore test to Playwright (#4923)

* Move multi-app-backup-and-restore test to Playwright
replicatedhq · Sep 26, 2024 · ea8f786 · ea8f786
1 parent ff0adfa
commit ea8f786
Show file tree

Hide file tree

Showing 25 changed files with 147 additions and 62 deletions.
diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml
@@ -1,8 +1,8 @@
 name: 'KOTS E2E'
 description: 'Composite action for running KOTS e2e test'
 inputs:
-  test-focus:
-    description: 'Name of the test for FOCUS'
+  test-id:
+    description: 'ID of the test to run'
     required: true
   kots-namespace:
     description: 'KOTS namespace'
@@ -92,14 +92,14 @@ runs:
         instance-type: ${{ inputs.k8s-instance-type }}
         export-kubeconfig: true
 
-    - name: execute suite "${{ inputs.test-focus }}"
+    - name: execute suite "${{ inputs.test-id }}"
       env:
         TESTIM_ACCESS_TOKEN: ${{ inputs.testim-access-token }}
         REPLICATED_API_TOKEN: ${{ inputs.replicated-api-token }}
         KOTS_NAMESPACE: ${{ inputs.kots-namespace }}
       run: |
         make -C e2e test \
-          FOCUS="${{ inputs.test-focus }}" \
+          FOCUS="${{ inputs.test-id }}" \
           EXISTING_KUBECONFIG=$KUBECONFIG \
           TESTIM_BRANCH=${{ inputs.testim-branch }} \
           KOTSADM_IMAGE_REGISTRY=${{ inputs.kotsadm-image-registry }} \

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
@@ -592,7 +592,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         with:
           kots-namespace: 'qakotsregression'
-          test-focus: 'Regression'
+          test-id: '@regression'
           k8s-distribution: k3s
           k8s-version: v1.26
           testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
@@ -628,7 +628,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Smoke Test'
+          test-id: '@smoke-test'
           kots-namespace: 'smoke-test'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -846,7 +846,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Backup and Restore'
+          test-id: '@backup-and-restore'
           kots-namespace: 'backup-and-restore'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -883,7 +883,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'No Required Config'
+          test-id: '@no-required-config'
           kots-namespace: 'no-required-config'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1073,7 +1073,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Config Validation'
+          test-id: '@config-validation'
           kots-namespace: 'config-validation'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1109,7 +1109,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Version History Pagination'
+          test-id: '@version-history-pagination'
           kots-namespace: 'version-history-pagination'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1146,7 +1146,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Change License'
+          test-id: '@change-license'
           kots-namespace: 'change-license'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1183,7 +1183,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Change Channel'
+          test-id: '@change-channel'
           kots-namespace: 'change-channel'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2338,7 +2338,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Min KOTS Version'
+          test-id: '@min-kots-version'
           kots-namespace: 'min-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2404,7 +2404,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Target KOTS Version'
+          test-id: '@target-kots-version'
           kots-namespace: 'target-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2469,7 +2469,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Range KOTS Version'
+          test-id: '@range-kots-version'
           kots-namespace: 'range-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2857,12 +2857,10 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'multi-app-backup-and-restore'
+          test-id: '@multi-app-backup-and-restore'
           kots-namespace: 'multi-app-backup-and-restore'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
           replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
           kots-dockerhub-username: '${{ secrets.E2E_DOCKERHUB_USERNAME }}'
           kots-dockerhub-password: '${{ secrets.E2E_DOCKERHUB_PASSWORD }}'
@@ -2895,7 +2893,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Multi App Install'
+          test-id: '@multi-app-install'
           kots-namespace: 'multi-app-install'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2931,7 +2929,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'airgap-smoke-test'
+          test-id: '@airgap-smoke-test'
           kots-namespace: 'airgap-smoke-test'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -3944,7 +3942,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Support Bundle'
+          test-id: '@support-bundle'
           kots-namespace: 'support-bundle'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -3982,7 +3980,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'GitOps'
+          test-id: '@gitops'
           kots-namespace: 'gitops'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -4337,7 +4335,6 @@ jobs:
       - validate-min-kots-version
       - validate-target-kots-version
       - validate-range-kots-version
-      - validate-multi-app-backup-and-restore
       - validate-airgap-smoke-test
       - validate-support-bundle
       - validate-gitops
@@ -4347,6 +4344,7 @@ jobs:
       - validate-no-required-config
       - validate-config
       - validate-change-channel
+      - validate-multi-app-backup-and-restore
       - validate-multi-app-install
       # non-testim tests
       - validate-minimal-rbac

diff --git a/e2e/README.md b/e2e/README.md
@@ -4,7 +4,7 @@ E2E tests are run in build-test workflow on pull_request event.
 
 e2e_test.go uses Ginkgo to build a test suite from inventory.go and runs each test using testim/client.go or playwright/playwright.go
 
-Tests are parallelized using Gingko's test focus. Each workflow definition in .github/workflows/build-test.yaml must define a `test-focus` parameter that matches the `Test Name` property defined in inventory.go. Each e2e test workflow skips all tests but what is defined in `test-focus`. Please colocate new Playwright tests under the Playwright comment with the other pw tests.
+Tests are parallelized using Gingko's test focus. Each workflow definition in .github/workflows/build-test.yaml must define a `test-id` parameter that matches the `Test ID` property defined in inventory.go. Each e2e test workflow skips all tests but what is defined in `test-id`. Please colocate new Playwright tests under the Playwright comment with the other pw tests.
 
 New tests should be written with Playwright.
 
@@ -28,14 +28,13 @@ code --install-extension ms-playwright.playwright
 
 To add a new test that you've already added in the [kots-tests-app repo](https://github.com/replicatedhq/kots-test-apps) - do the following:
 
-- Update `.github/workflows/build-test.yaml` to include the new test. You can copy an existing pw entry like `validate-change-channel` and update the test-focus, kots-namespace, and any other parameters needed for the test.
+- Update `.github/workflows/build-test.yaml` to include the new test. You can copy an existing pw entry like `validate-change-channel` and update the `test-id`, kots-namespace, and any other parameters needed for the test.
 - Add the test to `e2e/inventory.go` , making sure the naming matches your kots-test-app and conforms to the naming convention of the other tests in the file:
 
 ```go
 func NewChangeChannel() Test {
 	return Test{
-		ID:          "change-channel",
-		Name:        "Change Channel",
+		ID:          "@change-channel",
 		Namespace:   "change-channel",
 		AppSlug:     "change-channel",
 		UpstreamURI: "change-channel/automated",
@@ -52,8 +51,8 @@ Entry(nil, inventory.NewChangeChannel()),
 - Create a new test directory in `e2e/playwright/tests` matching your test ID, with the corresponding test file:
 
 ```
-$ tree e2e/playwright/tests/change-channel
-e2e/playwright/tests/change-channel 
+$ tree e2e/playwright/tests/@change-channel
+e2e/playwright/tests/@change-channel 
 ├── license.yaml  // a test specific license if needed
 └── test.spec.ts  // the actual test file
 ```
@@ -77,7 +76,7 @@ To run an individual test run:
 
 ```bash
 make e2e \
-    FOCUS="Change License"
+    FOCUS="@change-license"
 ```
 
 To build and run with ttl.sh images:
@@ -103,7 +102,7 @@ To run against the okteto dev environment run:
 ```bash
 okteto context use https://replicated.okteto.dev
 make e2e \
-    FOCUS="Change License" \
+    FOCUS="@change-license" \
     EXISTING_KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
 ```
 
@@ -113,7 +112,7 @@ To skip cluster teardown in order to debug issues:
 
 ```bash
 $ make e2e \
-    FOCUS="Change License" \
+    FOCUS="@change-license" \
     SKIP_TEARDOWN=1
 ...
     To set kubecontext run:

diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go
@@ -101,6 +101,13 @@ var _ = BeforeSuite(func() {
 	kotsInstaller = kots.NewInstaller(kotsadmImageRegistry, kotsadmImageNamespace, kotsadmImageTag, airgap, kotsDockerhubUsername, kotsDockerhubPassword, isEKS)
 })
 
+var _ = ReportBeforeSuite(func(report Report) {
+	count := report.PreRunStats.SpecsThatWillRun
+	if count == 0 {
+		Fail("Did not match any tests")
+	}
+})
+
 var _ = AfterSuite(func() {
 	gexec.KillAndWait()
 })
@@ -204,7 +211,7 @@ var _ = Describe("E2E", func() {
 				testimRun.ShouldSucceed()
 			},
 			func(test inventory.Test) string {
-				return test.Name
+				return test.ID
 			},
 			Entry(nil, inventory.NewRegressionTest()),
 			Entry(nil, inventory.NewSmokeTest()),
@@ -223,7 +230,5 @@ var _ = Describe("E2E", func() {
 			Entry(nil, inventory.NewGitOps()),
 			Entry(nil, inventory.NewChangeChannel()),
 		)
-
 	})
-
 })