Move backup and restore test to playwright

.github/workflows/build-test.yaml

@@ -787,8 +787,6 @@ jobs:
           k8s-distribution: ${{ matrix.cluster.distribution }}
           k8s-version: ${{ matrix.cluster.version }}
           k8s-instance-type: ${{ matrix.cluster.instance_type }}
-          testim-access-token: '${{ secrets.TESTIM_ACCESS_TOKEN }}'
-          testim-branch: ${{ github.head_ref == 'main' && 'master' || github.head_ref }}
           aws-access-key-id: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_ACCESS_KEY_ID }}'
           aws-secret-access-key: '${{ secrets.E2E_SUPPORT_BUNDLE_AWS_SECRET_ACCESS_KEY }}'
           replicated-api-token: '${{ secrets.C11Y_MATRIX_TOKEN }}'
@@ -4128,8 +4126,6 @@ jobs:
       # testim tests
       - validate-existing-online-install-minimal
       - validate-smoke-test
-      - validate-backup-and-restore
-      - validate-no-required-config
       - validate-version-history-pagination
       - validate-change-license
       - validate-min-kots-version
@@ -4138,9 +4134,12 @@ jobs:
       - validate-multi-app-backup-and-restore
       - validate-multi-app-install
       - validate-airgap-smoke-test
-      - validate-config
       - validate-support-bundle
       - validate-gitops
+      # playwright tests
+      - validate-backup-and-restore
+      - validate-no-required-config
+      - validate-config
       # non-testim tests
       - validate-minimal-rbac
       - validate-minimal-rbac-override

e2e/inventory/inventory.go

@@ -64,9 +64,10 @@ func NewConfigValidation() Test {
 
 func NewBackupAndRestore() Test {
 	return Test{
+		ID:             "backup-and-restore",
 		Name:           "Backup and Restore",
-		TestimSuite:    "backup-and-restore",
 		Namespace:      "backup-and-restore",
+		AppSlug:        "backup-and-restore",
 		UpstreamURI:    "backup-and-restore/automated",
 		NeedsSnapshots: true,
 	}

e2e/playwright/package.json

@@ -9,6 +9,8 @@
   "license": "ISC",
   "devDependencies": {
     "@playwright/test": "^1.43.1",
-    "@types/node": "^20.12.7"
+    "@types/node": "^20.12.7",
+    "parse-duration": "^1.1.0",
+    "ts-retry": "^4.2.5"
   }
 }

e2e/playwright/tests/backup-and-restore/license.yaml

@@ -0,0 +1,23 @@
+apiVersion: kots.io/v1beta1
+kind: License
+metadata:
+  name: backup-and-restore
+spec:
+  appSlug: backup-and-restore
+  channelID: 25rOK81fdyhZjF7KI4051tX36jy
+  channelName: Automated
+  customerName: backup-and-restore
+  endpoint: https://replicated.app
+  entitlements:
+    expires_at:
+      description: License Expiration
+      title: Expiration
+      value: ""
+      valueType: String
+  isGitOpsSupported: true
+  isNewKotsUiEnabled: true
+  isSnapshotSupported: true
+  licenseID: 25rOkSz6kabQmMCH1ASb1dUHHKy
+  licenseSequence: 1
+  licenseType: dev
+  signature: eyJsaWNlbnNlRGF0YSI6ImV5SmhjR2xXWlhKemFXOXVJam9pYTI5MGN5NXBieTkyTVdKbGRHRXhJaXdpYTJsdVpDSTZJa3hwWTJWdWMyVWlMQ0p0WlhSaFpHRjBZU0k2ZXlKdVlXMWxJam9pWW1GamEzVndMV0Z1WkMxeVpYTjBiM0psSW4wc0luTndaV01pT25zaWJHbGpaVzV6WlVsRUlqb2lNalZ5VDJ0VGVqWnJZV0pSYlUxRFNERkJVMkl4WkZWSVNFdDVJaXdpYkdsalpXNXpaVlI1Y0dVaU9pSmtaWFlpTENKamRYTjBiMjFsY2s1aGJXVWlPaUppWVdOcmRYQXRZVzVrTFhKbGMzUnZjbVVpTENKaGNIQlRiSFZuSWpvaVltRmphM1Z3TFdGdVpDMXlaWE4wYjNKbElpd2lZMmhoYm01bGJFbEVJam9pTWpWeVQwczRNV1prZVdoYWFrWTNTMGswTURVeGRGZ3pObXA1SWl3aVkyaGhibTVsYkU1aGJXVWlPaUpCZFhSdmJXRjBaV1FpTENKc2FXTmxibk5sVTJWeGRXVnVZMlVpT2pFc0ltVnVaSEJ2YVc1MElqb2lhSFIwY0hNNkx5OXlaWEJzYVdOaGRHVmtMbUZ3Y0NJc0ltVnVkR2wwYkdWdFpXNTBjeUk2ZXlKbGVIQnBjbVZ6WDJGMElqcDdJblJwZEd4bElqb2lSWGh3YVhKaGRHbHZiaUlzSW1SbGMyTnlhWEIwYVc5dUlqb2lUR2xqWlc1elpTQkZlSEJwY21GMGFXOXVJaXdpZG1Gc2RXVWlPaUlpTENKMllXeDFaVlI1Y0dVaU9pSlRkSEpwYm1jaWZYMHNJbWx6UjJsMFQzQnpVM1Z3Y0c5eWRHVmtJanAwY25WbExDSnBjMU51WVhCemFHOTBVM1Z3Y0c5eWRHVmtJanAwY25WbExDSnBjMDVsZDB0dmRITlZhVVZ1WVdKc1pXUWlPblJ5ZFdWOWZRPT0iLCJpbm5lclNpZ25hdHVyZSI6ImV5SnNhV05sYm5ObFUybG5ibUYwZFhKbElqb2lUbWhRVDNWVVpHUmlabGRvVGpsUVQwWkpNekZoTjJoTlRscFlOMkZxVTJ4WGVWbFFRalpVTVd4VFZXcHZXRWxoTDNsWVFubDJSbGg0UVRKSU9WWkZkazUxYUZvMGVFcFliVWhUTDNac2RuVTFOVTlYVTBsQmNIb3lOMnh0VUZKM1RVbERaa2x0ZVVoeEsxRlJkSHBuV0RCSldpOUhablpyWjI1TmExRkJWVmRUWjFoc2VXc3pUVXhxYkZaV1VUbDBVVTlaYm5SaFJVTnJUMjF3YVVFeGVteFJha0ZsWVVORFFteFFPVFpxVEVoRWFtZFphM0ZVZDFjclJHWjJSRkJQVXk5NE9UTnpUR1pEUkRFclNGSTVhRGRVVDA1RFdUaGpjblZqVGtvNE9XZERUMWw1UlZkNGFYZHdaVWRHZEVoMlptMXJjVzVhSzJWcWMzSTVkR2d2Y1VWbFdFUmhRVXBqVjJWV1pDOWhjMlJGTkV4WksyRXdhR3hoWlVabVlTOTVMMGxGUVNzMlkyMUVURlF2V0daR1pVTkxLMDA1YVVoSVF6aEdZVGg1YTJObFNHOUNRMGhIY1VkcFNuVlNiRTVEU1V4bVZWRkJQVDBpTENKd2RXSnNhV05MWlhraU9pSXRMUzB0TFVKRlIwbE9JRkJWUWt4SlF5QkxSVmt0TFMwdExWeHVUVWxKUWtscVFVNUNaMnR4YUd0cFJ6bDNNRUpCVVVWR1FVRlBRMEZST0VGTlNVbENRMmRMUTBGUlJVRnRkMDQxYVc5alIwaFlUMGN3Wkd0SlVVdERjRnh1YTNwek9WSlJhVWxMT1d0VFMzQlRSSGRxZFZRd1MyMUJkbEozV2pReVZpczRVMWc0VG5STldqVk9UMnhtSzBadlptbHpTa1F5WVc5NmVXNXdOMHRNVGx4dVNWaFlhVWRKYkRWNlEzZEtOa1ZNVmtOTFJ5c3pSR3R4VVRCdVNXNWtjM0JOTTJNellsSmtMMHRPU2pseVlVcEVNWEUwUlhaR2N6UllPRmhzVTJKNU5GeHVabmhLYzBkeFIyNDBhekY2UkVwbVVFc3pUbE5GTmpsdlVEWmFTMVF2TjA1d1duWjRWazVVZGswNFEyMU9kVFpyYkRGdFJucHFaSEkxYldWcFptWnpLMXh1VTNoaVRHNUpRWEZKTDFsRk1FcGlRM0pEVlRNeU9URm1OblY2UkVrMVkzUXJURVIyVjFsaFFsRktkMVYzZUhocFRuWnJlbXc0WTJGRGNURlJNMXBYVDF4dVdVbERXVkZqT0U5QmVWcDJNelpKWVVKeU1FaE1TVUZ6VVhwc1dtWmhTV0pwZEVoNGVsTktUMjU0YVc5RFNEUkVPVzFtZUdweWJrOW5NelVyY1d0eldWeHVURkZKUkVGUlFVSmNiaTB0TFMwdFJVNUVJRkJWUWt4SlF5QkxSVmt0TFMwdExWeHVJaXdpYTJWNVUybG5ibUYwZFhKbElqb2laWGxLZW1GWFpIVlpXRkl4WTIxVmFVOXBTblpWVms1Q1ZFVmtZVTVHVGtaYWJFWkNXbnBDU2xsWFRqTlNNRkpaVkROT1FsVllUWGxpYWxKd1kyNUtSMVJGWkVWT2JteEVWRmRXVDJGNlFrZGtWbWQzVlZWb2FXUlZSbWxPVjJOM1l6SXhSbUpzWTNsT2ExVjNaRlpTYUZKRmQzWk5lbXhJVTFka1NWZFZlRXBOZVhNd1l6TndkMVV3YkZCUFJYQjFVVzV3TkdFeU5XNVdWMmh2WWpCWmVsUllTblZPYWtwMVZFY3hVVlJIVmxCWk0wRjRWVWhDVm1JelJsbFhXR1JIVWtkR01GUXdPVWRsUlRsNVdUSjBhbGt4WkVWUlZGcEVUV3hPZG1WWVpETlRWV3g2VmxoV2VXRXphRUpQVm1SR1kwZEtOVTFYT0ROVWJWWlVXbXhXYUdSNlVuSlhhM2hVWTFaR1dXRllWbE5qYkZaWVpGWldXazB5WjNsVVdGWnFaV3QzTVZWRlJtcFJhWFF6VFRGVmVGVXlSazlXTTI5NFVtNU5NMkZzYUZoUFZUbGhWRlJWZVZKRlJrMWlSemt5VlRCV01sVlZkRnBsU0VwTFpESTRNbFI2WnpWVlNFWldWMjV3V0dRd1VUTlZSekZQWTI1QmRsWkljRmRWYWxweFZqRldSRTR6YUdsUmFtaDZVbGhLVjFSdGRFTmpSV2d3WlZSR1lXTllhRnBSV0VwUFVXdFZOVTR4YkZaV2VYUk5aVWQwVG1Jd01XcGxTR00wVjBWU1RsTXlNVk5rTW5ST1ltcGplazR5YUhsYVZWSlJaVmh2Y2xVd1JUbFFVMGx6U1cxa2MySXlTbWhpUlhSc1pWVnNhMGxxYjJsWmJWSnNXbFJWTWs1VVdYZFpNbHBwVGtST2FrOVhTWGxQUjBwdFQxUm9iRmxYVG1oYWJVVXlUa1JaYVdaUlBUMGlmUT09In0=

e2e/playwright/tests/backup-and-restore/test.spec.ts

@@ -0,0 +1,74 @@
+import { test, expect } from '@playwright/test';
+import parse_duration from 'parse-duration';
+import { retry } from 'ts-retry';
+import { login, uploadLicense } from '../shared';
+
+const { execSync } = require("child_process");
+
+test('backup and restore', async ({ page }) => {
+  test.setTimeout(10 * 60 * 1000); // 10 minutes
+  await login(page);
+  await uploadLicense(page, expect);
+  await expect(page.locator('#app')).toContainText('Configure Backup and Restore', { timeout: 15000 });
+  await page.locator('#smtp_hostname-group').getByRole('textbox').click();
+  await page.locator('#smtp_hostname-group').getByRole('textbox').fill('hostname');
+  await page.locator('#smtp_username-group').getByRole('textbox').click();
+  await page.locator('#smtp_username-group').getByRole('textbox').fill('username');
+  await page.locator('input[type="password"]').click();
+  await page.locator('input[type="password"]').fill('password');
+  await page.getByRole('button', { name: 'Continue' }).click();
+  await expect(page.locator('#app')).toContainText('Ready', { timeout: 60000 });
+  await page.locator('.NavItem').getByText('Snapshots', { exact: true }).click();
+  await expect(page.locator('#app')).toContainText('No snapshots yet');
+  await page.getByRole('button', { name: 'Start a snapshot' }).click();
+  await expect(page.locator('#app')).toContainText('In Progress');
+  await expect(page.locator('#app')).toContainText('Completed', { timeout: 300000 });
+
+  const backupName = await page.locator('.card-item-title').textContent();
+  const restoreAdminConsoleCmd = `kubectl kots restore --from-backup ${backupName} --exclude-apps`;
+  console.log(restoreAdminConsoleCmd, "\n");
+  execSync(restoreAdminConsoleCmd, {stdio: 'inherit'});
+
+  // validate that only the admin console was restored
+  const getKotsadmPodAgeCommand = `kubectl get pod -l app=kotsadm -n ${process.env.NAMESPACE} | awk 'NR>1 {print $5}'`;
+  console.log(getKotsadmPodAgeCommand, "\n");
+  let kotsadmPodAge = parse_duration(execSync(getKotsadmPodAgeCommand).toString().trim());
+
+  const getAppPodAgeCommand = `kubectl get pod -l app=example,component=nginx -n ${process.env.NAMESPACE} | awk 'NR>1 {print $5}'`;
+  console.log(getAppPodAgeCommand, "\n");
+  let appPodAge = parse_duration(execSync(getAppPodAgeCommand).toString().trim());
+
+  // app pod should be older than kotsadm pod
+  let ageDiff = appPodAge! - kotsadmPodAge!;
+  console.log(`application pod is ${ageDiff}ms older than the kotsadm pod`);
+  if (ageDiff < 5000) {
+    throw new Error("Expected the application pod to be older than the kotsadm pod");
+  }
+
+  const restoreAppCommand = `kubectl kots restore --from-backup ${backupName} --exclude-admin-console`;
+  console.log(restoreAppCommand, "\n");
+  execSync(restoreAppCommand, {stdio: 'inherit'});
+
+  await retry(
+    () => {
+      const getAppPodCommand = `kubectl get pod -l app=example,component=nginx -n ${process.env.NAMESPACE} | grep example-nginx`;
+      console.log(getAppPodCommand, "\n");
+      execSync(getAppPodCommand, {stdio: 'inherit'});
+    },
+    { delay: 1000, maxTry: 10 }
+  );
+
+  // validate that only the app was restored
+  console.log(getAppPodAgeCommand, "\n");
+  appPodAge = parse_duration(execSync(getAppPodAgeCommand).toString().trim());
+
+  console.log(getKotsadmPodAgeCommand, "\n");
+  kotsadmPodAge = parse_duration(execSync(getKotsadmPodAgeCommand).toString().trim());
+
+  // kotsadm pod should be older than app pod
+  ageDiff = kotsadmPodAge! - appPodAge!;
+  console.log(`kotsadm pod is ${ageDiff}ms older than the application pod`);
+  if (ageDiff < 5000) {
+    throw new Error("Expected the kotsadm pod to be older than the application pod");
+  }
+});