updates

.github/actions/kots-e2e/action.yml

@@ -1,8 +1,8 @@
 name: 'KOTS E2E'
 description: 'Composite action for running KOTS e2e test'
 inputs:
-  test-focus:
-    description: 'Name of the test for FOCUS'
+  test-id:
+    description: 'ID of the test to run'
     required: true
   kots-namespace:
     description: 'KOTS namespace'
@@ -92,14 +92,14 @@ runs:
         instance-type: ${{ inputs.k8s-instance-type }}
         export-kubeconfig: true
 
-    - name: execute suite "${{ inputs.test-focus }}"
+    - name: execute suite "${{ inputs.test-id }}"
       env:
         TESTIM_ACCESS_TOKEN: ${{ inputs.testim-access-token }}
         REPLICATED_API_TOKEN: ${{ inputs.replicated-api-token }}
         KOTS_NAMESPACE: ${{ inputs.kots-namespace }}
       run: |
         make -C e2e test \
-          FOCUS="${{ inputs.test-focus }}" \
+          FOCUS="${{ inputs.test-id }}" \
           EXISTING_KUBECONFIG=$KUBECONFIG \
           TESTIM_BRANCH=${{ inputs.testim-branch }} \
           KOTSADM_IMAGE_REGISTRY=${{ inputs.kotsadm-image-registry }} \

.github/workflows/build-test.yaml

@@ -592,7 +592,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         with:
           kots-namespace: 'qakotsregression'
-          test-focus: 'Regression'
+          test-id: '@regression'
           k8s-distribution: k3s
           k8s-version: v1.26
           testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
@@ -628,7 +628,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Smoke Test'
+          test-id: '@smoke-test'
           kots-namespace: 'smoke-test'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -846,7 +846,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Backup and Restore'
+          test-id: '@backup-and-restore'
           kots-namespace: 'backup-and-restore'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -883,7 +883,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'No Required Config'
+          test-id: '@no-required-config'
           kots-namespace: 'no-required-config'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1073,7 +1073,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Config Validation'
+          test-id: '@config-validation'
           kots-namespace: 'config-validation'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1109,7 +1109,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Version History Pagination'
+          test-id: '@version-history-pagination'
           kots-namespace: 'version-history-pagination'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1146,7 +1146,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Change License'
+          test-id: '@change-license'
           kots-namespace: 'change-license'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -1183,7 +1183,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Change Channel'
+          test-id: '@change-channel'
           kots-namespace: 'change-channel'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2338,7 +2338,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Min KOTS Version'
+          test-id: '@min-kots-version'
           kots-namespace: 'min-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2404,7 +2404,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Target KOTS Version'
+          test-id: '@target-kots-version'
           kots-namespace: 'target-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2469,7 +2469,7 @@ jobs:
       - uses: ./.github/actions/kots-e2e
         id: kots-e2e
         with:
-          test-focus: 'Range KOTS Version'
+          test-id: '@range-kots-version'
           kots-namespace: 'range-kots-version'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2857,7 +2857,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Multi App Backup and Restore'
+          test-id: '@multi-app-backup-and-restore'
           kots-namespace: 'multi-app-backup-and-restore'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2893,7 +2893,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Multi App Install'
+          test-id: '@multi-app-install'
           kots-namespace: 'multi-app-install'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -2929,7 +2929,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'airgap-smoke-test'
+          test-id: '@airgap-smoke-test'
           kots-namespace: 'airgap-smoke-test'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -3942,7 +3942,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'Support Bundle'
+          test-id: '@support-bundle'
           kots-namespace: 'support-bundle'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
@@ -3980,7 +3980,7 @@ jobs:
       - run: chmod +x bin/*
       - uses: ./.github/actions/kots-e2e
         with:
-          test-focus: 'GitOps'
+          test-id: '@gitops'
           kots-namespace: 'gitops'
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}

e2e/README.md

@@ -4,7 +4,7 @@ E2E tests are run in build-test workflow on pull_request event.
 
 e2e_test.go uses Ginkgo to build a test suite from inventory.go and runs each test using testim/client.go or playwright/playwright.go
 
-Tests are parallelized using Gingko's test focus. Each workflow definition in .github/workflows/build-test.yaml must define a `test-focus` parameter that matches the `Test Name` property defined in inventory.go. Each e2e test workflow skips all tests but what is defined in `test-focus`. Please colocate new Playwright tests under the Playwright comment with the other pw tests.
+Tests are parallelized using Gingko's test focus. Each workflow definition in .github/workflows/build-test.yaml must define a `test-id` parameter that matches the `Test ID` property defined in inventory.go. Each e2e test workflow skips all tests but what is defined in `test-id`. Please colocate new Playwright tests under the Playwright comment with the other pw tests.
 
 New tests should be written with Playwright.
 
@@ -28,14 +28,13 @@ code --install-extension ms-playwright.playwright
 
 To add a new test that you've already added in the [kots-tests-app repo](https://github.com/replicatedhq/kots-test-apps) - do the following:
 
-- Update `.github/workflows/build-test.yaml` to include the new test. You can copy an existing pw entry like `validate-change-channel` and update the test-focus, kots-namespace, and any other parameters needed for the test.
+- Update `.github/workflows/build-test.yaml` to include the new test. You can copy an existing pw entry like `validate-change-channel` and update the `test-id`, kots-namespace, and any other parameters needed for the test.
 - Add the test to `e2e/inventory.go` , making sure the naming matches your kots-test-app and conforms to the naming convention of the other tests in the file:
 
 ```go
 func NewChangeChannel() Test {
 	return Test{
 		ID:          "@change-channel",
-		Name:        "Change Channel",
 		Namespace:   "change-channel",
 		AppSlug:     "change-channel",
 		UpstreamURI: "change-channel/automated",
@@ -77,7 +76,7 @@ To run an individual test run:
 
 ```bash
 make e2e \
-    FOCUS="Change License"
+    FOCUS="@change-license"
 ```
 
 To build and run with ttl.sh images:
@@ -103,7 +102,7 @@ To run against the okteto dev environment run:
 ```bash
 okteto context use https://replicated.okteto.dev
 make e2e \
-    FOCUS="Change License" \
+    FOCUS="@change-license" \
     EXISTING_KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config}
 ```
 
@@ -113,7 +112,7 @@ To skip cluster teardown in order to debug issues:
 
 ```bash
 $ make e2e \
-    FOCUS="Change License" \
+    FOCUS="@change-license" \
     SKIP_TEARDOWN=1
 ...
     To set kubecontext run:

e2e/e2e_test.go

@@ -101,6 +101,13 @@ var _ = BeforeSuite(func() {
 	kotsInstaller = kots.NewInstaller(kotsadmImageRegistry, kotsadmImageNamespace, kotsadmImageTag, airgap, kotsDockerhubUsername, kotsDockerhubPassword, isEKS)
 })
 
+var _ = ReportBeforeSuite(func(report Report) {
+	count := report.PreRunStats.SpecsThatWillRun
+	if count == 0 {
+		Fail("Did not match any tests")
+	}
+})
+
 var _ = AfterSuite(func() {
 	gexec.KillAndWait()
 })
@@ -204,7 +211,7 @@ var _ = Describe("E2E", func() {
 				testimRun.ShouldSucceed()
 			},
 			func(test inventory.Test) string {
-				return test.Name
+				return test.ID
 			},
 			Entry(nil, inventory.NewRegressionTest()),
 			Entry(nil, inventory.NewSmokeTest()),
@@ -223,7 +230,5 @@ var _ = Describe("E2E", func() {
 			Entry(nil, inventory.NewGitOps()),
 			Entry(nil, inventory.NewChangeChannel()),
 		)
-
 	})
-
 })

e2e/inventory/inventory.go

@@ -21,7 +21,7 @@ const (
 
 func NewRegressionTest() Test {
 	return Test{
-		Name:            "Regression",
+		ID:              "@regression",
 		TestimLabel:     "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
 		Namespace:       "qakotsregression",
 		UpstreamURI:     "qakotsregression/type-existing-cluster-env-on-2",
@@ -36,7 +36,6 @@ func NewRegressionTest() Test {
 func NewSmokeTest() Test {
 	return Test{
 		ID:             "@smoke-test",
-		Name:           "Smoke Test",
 		Namespace:      "smoke-test",
 		AppSlug:        "qakotstestim",
 		UpstreamURI:    "qakotstestim/github-actions-qa",
@@ -46,7 +45,7 @@ func NewSmokeTest() Test {
 
 func NewAirgapSmokeTest() Test {
 	return Test{
-		Name:        "airgap-smoke-test",
+		ID:          "@airgap-smoke-test",
 		TestimSuite: "airgap-smoke-test",
 		Namespace:   "airgap-smoke-test",
 		UpstreamURI: "airgap-smoke-test/automated",
@@ -56,7 +55,6 @@ func NewAirgapSmokeTest() Test {
 func NewConfigValidation() Test {
 	return Test{
 		ID:          "@config-validation",
-		Name:        "Config Validation",
 		Namespace:   "config-validation",
 		AppSlug:     "config-validation-panda",
 		UpstreamURI: "config-validation-panda/automated",
@@ -66,7 +64,6 @@ func NewConfigValidation() Test {
 func NewBackupAndRestore() Test {
 	return Test{
 		ID:             "@backup-and-restore",
-		Name:           "Backup and Restore",
 		Namespace:      "backup-and-restore",
 		AppSlug:        "backup-and-restore",
 		UpstreamURI:    "backup-and-restore/automated",
@@ -77,7 +74,6 @@ func NewBackupAndRestore() Test {
 func NewNoRequiredConfig() Test {
 	return Test{
 		ID:          "@no-required-config",
-		Name:        "No Required Config",
 		Namespace:   "no-required-config",
 		AppSlug:     "no-required-config",
 		UpstreamURI: "no-required-config/automated",
@@ -86,7 +82,7 @@ func NewNoRequiredConfig() Test {
 
 func NewVersionHistoryPagination() Test {
 	return Test{
-		Name:        "Version History Pagination",
+		ID:          "@version-history-pagination",
 		TestimSuite: "version-history-pagination",
 		Namespace:   "version-history-pagination",
 		UpstreamURI: "version-history-pagination/automated",
@@ -95,7 +91,7 @@ func NewVersionHistoryPagination() Test {
 
 func NewChangeLicense() Test {
 	return Test{
-		Name:        "Change License",
+		ID:          "@change-license",
 		TestimSuite: "change-license",
 		Namespace:   "change-license",
 		UpstreamURI: "change-license/automated",
@@ -105,7 +101,6 @@ func NewChangeLicense() Test {
 func NewMultiAppBackupAndRestoreTest() Test {
 	return Test{
 		ID:             "@multi-app-backup-and-restore",
-		Name:           "Multi App Backup and Restore",
 		Namespace:      "multi-app-backup-and-restore",
 		AppSlug:        "multi-app-backup-and-restore",
 		UpstreamURI:    "multi-app-backup-and-restore/automated",
@@ -116,7 +111,6 @@ func NewMultiAppBackupAndRestoreTest() Test {
 func MultiAppTest() Test {
 	return Test{
 		ID:          "@multi-app-install",
-		Name:        "Multi App Install",
 		Namespace:   "multi-app-install",
 		AppSlug:     "mutli-app-install",
 		UpstreamURI: "mutli-app-install/automated",
@@ -125,7 +119,7 @@ func MultiAppTest() Test {
 
 func NewMinKotsVersion() Test {
 	return Test{
-		Name:                   "Min KOTS Version",
+		ID:                     "@min-kots-version",
 		TestimSuite:            "min-kots-version",
 		Namespace:              "min-kots-version",
 		UpstreamURI:            "min-kots-version/automated",
@@ -135,7 +129,7 @@ func NewMinKotsVersion() Test {
 
 func NewTargetKotsVersion() Test {
 	return Test{
-		Name:                   "Target KOTS Version",
+		ID:                     "@target-kots-version",
 		TestimSuite:            "target-kots-version",
 		Namespace:              "target-kots-version",
 		UpstreamURI:            "target-kots-version/automated",
@@ -145,7 +139,7 @@ func NewTargetKotsVersion() Test {
 
 func NewRangeKotsVersion() Test {
 	return Test{
-		Name:                   "Range KOTS Version",
+		ID:                     "@range-kots-version",
 		TestimSuite:            "range-kots-version",
 		Namespace:              "range-kots-version",
 		UpstreamURI:            "range-kots-version/automated",
@@ -155,7 +149,7 @@ func NewRangeKotsVersion() Test {
 
 func NewSupportBundle() Test {
 	return Test{
-		Name:        "Support Bundle",
+		ID:          "@support-bundle",
 		TestimSuite: "support-bundle",
 		Namespace:   "support-bundle",
 		UpstreamURI: "support-bundle-halibut/automated",
@@ -164,7 +158,7 @@ func NewSupportBundle() Test {
 
 func NewGitOps() Test {
 	return Test{
-		Name:        "GitOps",
+		ID:          "@gitops",
 		TestimSuite: "gitops",
 		Namespace:   "gitops",
 		UpstreamURI: "gitops-bobcat/automated",
@@ -174,7 +168,6 @@ func NewGitOps() Test {
 func NewChangeChannel() Test {
 	return Test{
 		ID:          "@change-channel",
-		Name:        "Change Channel",
 		Namespace:   "change-channel",
 		AppSlug:     "change-channel",
 		UpstreamURI: "change-channel/automated",

e2e/inventory/test.go

@@ -6,7 +6,6 @@ type TestimParams map[string]interface{}
 
 type Test struct {
 	ID                     string // must match directory name in e2e/playwright/tests
-	Name                   string // must match test-focus in .github/workflows/build-test.yaml
 	TestimSuite            string
 	TestimLabel            string
 	Namespace              string