feat(ci): change e2e tests to use new dr (#1634)

.github/workflows/ci.yaml

@@ -204,8 +204,8 @@ jobs:
           echo "K0S_VERSION=\"$K0S_VERSION\""
           echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
 
-  build-newdr:
-    name: Build new DR
+  build-legacydr:
+    name: Build legacy DR
     runs-on: embedded-cluster
     needs:
       - git-sha
@@ -239,7 +239,7 @@ jobs:
         env:
           APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
           APP_CHANNEL_SLUG: ci
-          RELEASE_YAML_DIR: e2e/kots-release-install-newdr
+          RELEASE_YAML_DIR: e2e/kots-release-install-legacydr
           S3_BUCKET: "tf-staging-embedded-cluster-bin"
           USES_DEV_BUCKET: "0"
           AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
@@ -251,20 +251,20 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           export K0S_VERSION=$(make print-K0S_VERSION)
-          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-newdr
-          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-newdr
+          export EC_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr
+          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-legacydr
           # avoid rate limiting
           export FIO_VERSION=$(gh release list --repo axboe/fio --json tagName,isLatest | jq -r '.[] | select(.isLatest==true)|.tagName' | cut -d- -f2)
 
           ./scripts/build-and-release.sh
-          cp output/bin/embedded-cluster output/bin/embedded-cluster-newdr
+          cp output/bin/embedded-cluster output/bin/embedded-cluster-legacydr
 
       - name: Upload release
         uses: actions/upload-artifact@v4
         with:
-          name: newdr-release
+          name: legacydr-release
           path: |
-            output/bin/embedded-cluster-newdr
+            output/bin/embedded-cluster-legacydr
 
       - name: Export k0s version
         id: export
@@ -509,7 +509,7 @@ jobs:
     needs:
       - git-sha
       - build-current
-      - build-newdr
+      - build-legacydr
       - build-previous-k0s
       - build-upgrade
       - find-previous-stable
@@ -564,9 +564,9 @@ jobs:
           ./scripts/ci-release-app.sh
 
           # promote a release with improved dr support
-          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-newdr"
-          export APP_VERSION="appver-${SHORT_SHA}-newdr"
-          export RELEASE_YAML_DIR=e2e/kots-release-install-newdr
+          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr"
+          export APP_VERSION="appver-${SHORT_SHA}-legacydr"
+          export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr
           ./scripts/ci-release-app.sh
 
           # then a noop upgrade
@@ -610,9 +610,9 @@ jobs:
           ./scripts/ci-release-app.sh
 
           # promote a release with improved dr support
-          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-newdr"
-          export APP_VERSION="appver-${SHORT_SHA}-newdr"
-          export RELEASE_YAML_DIR=e2e/kots-release-install-newdr
+          export EC_VERSION="$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*')-legacydr"
+          export APP_VERSION="appver-${SHORT_SHA}-legacydr"
+          export RELEASE_YAML_DIR=e2e/kots-release-install-legacydr
           ./scripts/ci-release-app.sh
 
           # and finally an app upgrade
@@ -654,7 +654,7 @@ jobs:
     needs:
       - git-sha
       - build-current
-      - build-newdr
+      - build-legacydr
       - build-previous-k0s
       - build-upgrade
       - find-previous-stable
@@ -684,7 +684,7 @@ jobs:
           - TestMultiNodeInstallation
           - TestMultiNodeHAInstallation
           - TestSingleNodeDisasterRecovery
-          - TestSingleNodeNewDisasterRecovery
+          - TestSingleNodeLegacyDisasterRecovery
           - TestSingleNodeResumeDisasterRecovery
           - TestMultiNodeHADisasterRecovery
           - TestSingleNodeInstallationNoopUpgrade
@@ -706,7 +706,7 @@ jobs:
       - name: Download new DR binary
         uses: actions/download-artifact@v4
         with:
-          name: newdr-release
+          name: legacydr-release
           path: output/bin
       - name: Setup go
         uses: actions/setup-go@v5
@@ -754,7 +754,7 @@ jobs:
     runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
     needs:
       - build-current
-      - build-newdr
+      - build-legacydr
       - build-previous-k0s
       - build-upgrade
       - find-previous-stable

e2e/kots-release-install-newdr/deployment-2.yaml → e2e/kots-release-install-legacydr/deployment-2.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: second
-  namespace: my-app
   labels:
     app: second
     replicated.com/disaster-recovery: app

e2e/kots-release-install-newdr/deployment.yaml → e2e/kots-release-install-legacydr/deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: nginx
-  namespace: my-app
   labels:
     app: example
     component: nginx

e2e/kots-release-install-newdr/kots-app.yaml → e2e/kots-release-install-legacydr/kots-app.yaml

@@ -7,10 +7,8 @@ spec:
   title: Embedded Cluster Smoke Test Staging App
   icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/kubernetes/icon/color/kubernetes-icon-color.png
   allowRollback: true
-  additionalNamespaces:
-    - my-app
   statusInformers:
-    - my-app/deployment/nginx
+    - deployment/nginx
   ports:
     - serviceName: "nginx"
       servicePort: 80

e2e/kots-release-install-newdr/backup.yaml → e2e/kots-release-install/backup.yaml

@@ -7,7 +7,10 @@ metadata:
 spec:
   ttl: 36h0m0s
   includedNamespaces:
-  - my-app
+  - kotsadm
+  orLabelSelectors:
+  - matchExpressions:
+    - { key: kots.io/kotsadm, operator: NotIn, values: ["true"] }
   hooks:
     resources:
     - name: test-hook

e2e/kots-release-install-newdr/restore.yaml → e2e/kots-release-install/restore.yaml

@@ -12,7 +12,10 @@ spec:
     resources:
     - name: restore-hook-1
       includedNamespaces:
-      - my-app
+      - kotsadm
+      labelSelector:
+        matchLabels:
+          app: example
       postHooks:
       - init:
           initContainers:

e2e/kots-release-upgrade/backup.yaml

@@ -0,0 +1,33 @@
+apiVersion: velero.io/v1
+kind: Backup
+metadata:
+  name: backup
+  annotations:
+    preserve: me
+spec:
+  ttl: 36h0m0s
+  includedNamespaces:
+  - kotsadm
+  orLabelSelectors:
+  - matchExpressions:
+    - { key: kots.io/kotsadm, operator: NotIn, values: ["true"] }
+  hooks:
+    resources:
+    - name: test-hook
+      includedResources:
+      - 'pods'
+      labelSelector:
+        matchLabels:
+          app: example
+          component: nginx
+      pre:
+      - exec:
+          container: nginx
+          command:
+          - /bin/uname
+          - -a
+      post:
+      - exec:
+          command:
+           - /bin/uname
+           - -a

e2e/kots-release-upgrade/restore.yaml

@@ -0,0 +1,27 @@
+apiVersion: velero.io/v1
+kind: Restore
+metadata:
+  name: restore
+  annotations:
+    preserve: me
+spec:
+  backupName: backup
+  includedNamespaces:
+  - '*'
+  hooks:
+    resources:
+    - name: restore-hook-1
+      includedNamespaces:
+      - kotsadm
+      labelSelector:
+        matchLabels:
+          app: example
+      postHooks:
+      - init:
+          initContainers:
+          - name: restore-hook-init1
+            image: proxy.replicated.com/anonymous/nginx:1.24-alpine
+            command:
+            - /bin/ash
+            - -c
+            - echo -n "FOOBARBAZ" > /tmp/foobarbaz

e2e/playwright/README.md

@@ -28,6 +28,7 @@
      labels:
        replicated.com/disaster-recovery: infra
        replicated.com/disaster-recovery-chart: admin-console
+       kots.io/kotsadm: "true"
    spec:
      type: NodePort
      ports:

e2e/restore_test.go

@@ -85,6 +85,12 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs = []string{appUpgradeVersion}
 
@@ -102,7 +108,7 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
-func TestSingleNodeNewDisasterRecovery(t *testing.T) {
+func TestSingleNodeLegacyDisasterRecovery(t *testing.T) {
 	t.Parallel()
 
 	requiredEnvVars := []string{
@@ -125,18 +131,13 @@ func TestSingleNodeNewDisasterRecovery(t *testing.T) {
 		Nodes:        1,
 		Distro:       "debian-bookworm",
 		LicensePath:  "snapshot-license.yaml",
-		ECBinaryPath: "../output/bin/embedded-cluster-newdr",
+		ECBinaryPath: "../output/bin/embedded-cluster-legacydr",
 	})
 	defer tc.Cleanup()
 
-	// Use an alternate data directory
-	withEnv := map[string]string{
-		"APP_NAMESPACE": "my-app",
-	}
-
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
 	line := []string{"single-node-install.sh", "ui"}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
 		t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
 	}
 
@@ -147,11 +148,11 @@ func TestSingleNodeNewDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to run playwright test deploy-app: %v", err)
 	}
 
-	appVersion := fmt.Sprintf("appver-%s-newdr", os.Getenv("SHORT_SHA"))
+	appVersion := fmt.Sprintf("appver-%s-legacydr", os.Getenv("SHORT_SHA"))
 
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", appVersion, k8sVersion()}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
 
@@ -182,18 +183,10 @@ func TestSingleNodeNewDisasterRecovery(t *testing.T) {
 
 	t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
 	line = []string{"check-installation-state.sh", appVersion, k8sVersion()}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
 		t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
 	}
 
-	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
-	line = []string{"check-post-restore-newdr.sh"}
-	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
-		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
-	}
-
-	// TODO: upgrade
-
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
@@ -277,6 +270,12 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
@@ -453,6 +452,12 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check installation state: %v", err)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
 	t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
 	line = []string{"airgap-update.sh"}
 	if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
@@ -656,6 +661,12 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post ha state: %v: %s: %s", err, stdout, stderr)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
 	appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
 	testArgs = []string{appUpgradeVersion}
 
@@ -921,5 +932,11 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to check post ha state: %v", err)
 	}
 
+	t.Logf("%s: checking post-restore state", time.Now().Format(time.RFC3339))
+	line = []string{"check-post-restore.sh"}
+	if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil {
+		t.Fatalf("fail to check post-restore state: %v: %s: %s", err, stdout, stderr)
+	}
+
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }

e2e/scripts/bypass-kurl-proxy.sh

@@ -15,6 +15,7 @@ main() {
     labels:
       replicated.com/disaster-recovery: infra
       replicated.com/disaster-recovery-chart: admin-console
+      kots.io/kotsadm: "true"
   spec:
     type: NodePort
     ports:

pkg/kotscli/kotscli.go

@@ -339,6 +339,7 @@ func CreateHostSupportBundle() error {
 			Namespace: "kotsadm",
 			Labels: map[string]string{
 				"troubleshoot.sh/kind":                   "support-bundle",
+				"kots.io/kotsadm":                        "true",
 				"replicated.com/disaster-recovery":       "infra",
 				"replicated.com/disaster-recovery-chart": "admin-console",
 			},