test: [M3-8114] - Clean up Linodes, LKE clusters, and Firewalls after…

… Cypress runs (#11189)

* Clean up Linodes, Firewalls, and LKE Clusters after test run, and improve util docs

* Added changeset: Delete test Linodes, LKE clusters, and Firewalls after Cypress runs

packages/manager/.changeset/pr-11189-tests-1730310015248.md

@@ -0,0 +1,5 @@
+---
+"@linode/manager": Tests
+---
+
+Delete test Linodes, LKE clusters, and Firewalls after Cypress runs ([#11189](https://github.com/linode/manager/pull/11189))

packages/manager/cypress.config.ts

@@ -18,6 +18,7 @@ import { generateTestWeights } from './cypress/support/plugins/generate-weights'
 import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info';
 import cypressViteConfig from './cypress/vite.config';
 import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override';
+import { postRunCleanup } from './cypress/support/plugins/post-run-cleanup';
 
 /**
  * Exports a Cypress configuration object.
@@ -97,6 +98,7 @@ export default defineConfig({
         splitCypressRun,
         enableJunitReport(),
         generateTestWeights,
+        postRunCleanup,
       ]);
     },
   },

packages/manager/cypress/support/api/lke.ts

@@ -35,6 +35,12 @@ const isPoolReady = (pool: KubeNodePoolResponse): boolean =>
 /**
  * Delete all LKE clusters whose labels are prefixed with "cy-test-".
  *
+ * Sometimes when attempting to delete provisioning LKE clusters, the cluster
+ * becomes stuck and requires manual intervention to resolve. To reduce the risk
+ * of this happening, this function will only delete clusters that have finished
+ * provisioning (i.e. all nodes have `'ready'` status) or which have existed
+ * for at least an hour.
+ *
  * @returns Promise that resolves when test LKE clusters have been deleted.
  */
 export const deleteAllTestLkeClusters = async (): Promise<void> => {

packages/manager/cypress/support/plugins/post-run-cleanup.ts

@@ -0,0 +1,174 @@
+import { DateTime } from 'luxon';
+import { depaginate } from '../util/paginate';
+import { CypressPlugin } from './plugin';
+
+import {
+  deleteFirewall,
+  deleteKubernetesCluster,
+  deleteLinode,
+  Firewall,
+  getFirewalls,
+  getKubernetesClusters,
+  getLinodes,
+  getNodePools,
+  KubeNodePoolResponse,
+  KubernetesCluster,
+  Linode,
+  PoolNodeResponse,
+} from '@linode/api-v4';
+
+// TODO Refactor to use utilities after M3-8803.
+/*
+ * Cypress configuration and plugins are executed in Node.js where our
+ * path aliases `support`, `src`/`@src`, etc., are unavailable. Additionally,
+ * some Cypress-specific objects like `cy` and `Cypress` are unavailable.
+ *
+ * As a result, we cannot import any code which uses aliases, uses `cy`/`Cypress`,
+ * or imports any code which does (and so on...) from here. Because of this
+ * limitation, we can't import our existing utilities related to resource clean
+ * up and sadly must re-implement them here.
+ *
+ * M3-8803 seeks to reorganize our utilities to better distinguish which code
+ * is executed and expected to be available where, and after that point we
+ * should be able to refactor this plugin to take advantage of existing utilities
+ * like `deleteAllTestLinodes`, `deleteAllTestFirewalls`, etc.
+ */
+
+// Test resource label/name prefix.
+const TEST_TAG_PREFIX = 'cy-test-';
+
+// Desired number of items per page of a paginated API request.
+const PAGE_SIZE = 500;
+
+/*
+ * Determines if the given node pool is ready by checking the status of each node.
+ */
+const isPoolReady = (pool: KubeNodePoolResponse): boolean =>
+  pool.nodes.every((node: PoolNodeResponse) => node.status === 'ready');
+
+/**
+ * Deletes all test Linodes on the test account.
+ *
+ * This is a re-implementation of an existing util, `deleteAllTestLinodes`, in
+ * `support/api/linodes.ts`.
+ *
+ * @returns Promise that resolves when all test Linodes are deleted.
+ */
+const deleteTestLinodes = async () => {
+  const allLinodes = await depaginate<Linode>((page) =>
+    getLinodes({ page, page_size: PAGE_SIZE })
+  );
+
+  const deletePromises = allLinodes
+    .filter((linode: Linode) => linode.label.startsWith(TEST_TAG_PREFIX))
+    .map((linode: Linode) => deleteLinode(linode.id));
+
+  await Promise.all(deletePromises);
+};
+
+/**
+ * Deletes all test Firewalls on the test account.
+ *
+ * This is a re-implementation of an existing util, `deleteAllTestFirewalls`, in
+ * `support/api/firewalls.ts`.
+ *
+ * @returns Promise that resolves when all test Firewalls are deleted.
+ */
+const deleteTestFirewalls = async () => {
+  const allFirewalls = await depaginate<Firewall>((page) =>
+    getFirewalls({ page, page_size: PAGE_SIZE })
+  );
+
+  const deletePromises = allFirewalls
+    .filter((firewall: Firewall) => firewall.label.startsWith(TEST_TAG_PREFIX))
+    .map((firewall: Firewall) => deleteFirewall(firewall.id));
+
+  await Promise.all(deletePromises);
+};
+
+/**
+ * Deletes all running test LKE clusters on the test account.
+ *
+ * Sometimes when attempting to delete provisioning LKE clusters, the cluster
+ * becomes stuck and requires manual intervention to resolve. To reduce the risk
+ * of this happening, this function will only delete clusters that have finished
+ * provisioning (i.e. all nodes have `'ready'` status) or which have existed
+ * for at least an hour.
+ *
+ * This is a re-implementation of an existing util, `deleteAllTestLkeClusters`, in
+ * `support/api/lke.ts`.
+ *
+ * @returns Promise that resolves when all test LKE clusters are deleted.
+ */
+const deleteTestLkeClusters = async () => {
+  const allClusters = await depaginate<KubernetesCluster>((page) =>
+    getKubernetesClusters({ page, page_size: PAGE_SIZE })
+  );
+
+  const clusterDeletionPromises = allClusters
+    .filter((cluster: KubernetesCluster) =>
+      cluster.label.startsWith(TEST_TAG_PREFIX)
+    )
+    .map(async (cluster: KubernetesCluster) => {
+      const clusterCreateTime = DateTime.fromISO(cluster.created, {
+        zone: 'utc',
+      });
+      const createTimeElapsed = Math.abs(
+        clusterCreateTime.diffNow('minutes').minutes
+      );
+
+      // If the test cluster is older than 1 hour, delete it regardless of
+      // whether or not all of the Node Pools are ready; this is a safeguard
+      // to prevent LKE clusters with stuck pools from accumulating.
+      if (createTimeElapsed >= 60) {
+        return deleteKubernetesCluster(cluster.id);
+      }
+
+      // If the cluster is not older than 1 hour, only delete it if all of its
+      // Node Pools are ready.
+      const pools = await depaginate<KubeNodePoolResponse>((page: number) =>
+        getNodePools(cluster.id, { page, page_size: PAGE_SIZE })
+      );
+      if (pools.every(isPoolReady)) {
+        return deleteKubernetesCluster(cluster.id);
+      }
+      return;
+    });
+
+  await Promise.all(clusterDeletionPromises);
+};
+
+/*
+ * Human-friendly string describing the types of resources being deleted,
+ * and their corresponding deletion function.
+ */
+const resourceCleanUpItems = [
+  { name: 'Linodes', cleanUp: deleteTestLinodes },
+  // TODO Remove LKE cluster clean up once M3-8656 is complete because cluster cleanup will no longer be necessary.
+  { name: 'LKE Clusters', cleanUp: deleteTestLkeClusters },
+  { name: 'Firewalls', cleanUp: deleteTestFirewalls },
+];
+
+export const postRunCleanup: CypressPlugin = async (on) => {
+  on('after:run', async () => {
+    console.log('Performing post-run clean up:\n');
+
+    for (const resourceCleanUpItem of resourceCleanUpItems) {
+      console.log(`- Cleaning up test ${resourceCleanUpItem.name}...`);
+      try {
+        // Perform clean-up sequentially.
+        // eslint-disable-next-line no-await-in-loop
+        await resourceCleanUpItem.cleanUp();
+      } catch (e) {
+        console.error(
+          `\nAn error occurred while cleaning up test ${resourceCleanUpItem.name}:`
+        );
+        if (e.message) {
+          console.error(e.message);
+        }
+        console.error(e);
+      }
+    }
+    console.log('\nPost-run clean up is complete');
+  });
+};