From 7c547f63edb31f4ff063c895101f7c47a2060402 Mon Sep 17 00:00:00 2001 From: jdamore-linode <97627410+jdamore-linode@users.noreply.github.com> Date: Mon, 4 Nov 2024 14:39:20 -0500 Subject: [PATCH] test: [M3-8114] - Clean up Linodes, LKE clusters, and Firewalls after Cypress runs (#11189) * Clean up Linodes, Firewalls, and LKE Clusters after test run, and improve util docs * Added changeset: Delete test Linodes, LKE clusters, and Firewalls after Cypress runs --- .../pr-11189-tests-1730310015248.md | 5 + packages/manager/cypress.config.ts | 2 + packages/manager/cypress/support/api/lke.ts | 6 + .../support/plugins/post-run-cleanup.ts | 174 ++++++++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 packages/manager/.changeset/pr-11189-tests-1730310015248.md create mode 100644 packages/manager/cypress/support/plugins/post-run-cleanup.ts diff --git a/packages/manager/.changeset/pr-11189-tests-1730310015248.md b/packages/manager/.changeset/pr-11189-tests-1730310015248.md new file mode 100644 index 00000000000..0fbe1f93553 --- /dev/null +++ b/packages/manager/.changeset/pr-11189-tests-1730310015248.md @@ -0,0 +1,5 @@ +--- +"@linode/manager": Tests +--- + +Delete test Linodes, LKE clusters, and Firewalls after Cypress runs ([#11189](https://github.com/linode/manager/pull/11189)) diff --git a/packages/manager/cypress.config.ts b/packages/manager/cypress.config.ts index b51562d9f38..83903ec697c 100644 --- a/packages/manager/cypress.config.ts +++ b/packages/manager/cypress.config.ts @@ -18,6 +18,7 @@ import { generateTestWeights } from './cypress/support/plugins/generate-weights' import { logTestTagInfo } from './cypress/support/plugins/test-tagging-info'; import cypressViteConfig from './cypress/vite.config'; import { featureFlagOverrides } from './cypress/support/plugins/feature-flag-override'; +import { postRunCleanup } from './cypress/support/plugins/post-run-cleanup'; /** * Exports a Cypress configuration object. @@ -97,6 +98,7 @@ export default defineConfig({ splitCypressRun, enableJunitReport(), generateTestWeights, + postRunCleanup, ]); }, }, diff --git a/packages/manager/cypress/support/api/lke.ts b/packages/manager/cypress/support/api/lke.ts index 9551ca55460..14ed8339749 100644 --- a/packages/manager/cypress/support/api/lke.ts +++ b/packages/manager/cypress/support/api/lke.ts @@ -35,6 +35,12 @@ const isPoolReady = (pool: KubeNodePoolResponse): boolean => /** * Delete all LKE clusters whose labels are prefixed with "cy-test-". * + * Sometimes when attempting to delete provisioning LKE clusters, the cluster + * becomes stuck and requires manual intervention to resolve. To reduce the risk + * of this happening, this function will only delete clusters that have finished + * provisioning (i.e. all nodes have `'ready'` status) or which have existed + * for at least an hour. + * * @returns Promise that resolves when test LKE clusters have been deleted. */ export const deleteAllTestLkeClusters = async (): Promise => { diff --git a/packages/manager/cypress/support/plugins/post-run-cleanup.ts b/packages/manager/cypress/support/plugins/post-run-cleanup.ts new file mode 100644 index 00000000000..6be94af0a30 --- /dev/null +++ b/packages/manager/cypress/support/plugins/post-run-cleanup.ts @@ -0,0 +1,174 @@ +import { DateTime } from 'luxon'; +import { depaginate } from '../util/paginate'; +import { CypressPlugin } from './plugin'; + +import { + deleteFirewall, + deleteKubernetesCluster, + deleteLinode, + Firewall, + getFirewalls, + getKubernetesClusters, + getLinodes, + getNodePools, + KubeNodePoolResponse, + KubernetesCluster, + Linode, + PoolNodeResponse, +} from '@linode/api-v4'; + +// TODO Refactor to use utilities after M3-8803. +/* + * Cypress configuration and plugins are executed in Node.js where our + * path aliases `support`, `src`/`@src`, etc., are unavailable. Additionally, + * some Cypress-specific objects like `cy` and `Cypress` are unavailable. + * + * As a result, we cannot import any code which uses aliases, uses `cy`/`Cypress`, + * or imports any code which does (and so on...) from here. Because of this + * limitation, we can't import our existing utilities related to resource clean + * up and sadly must re-implement them here. + * + * M3-8803 seeks to reorganize our utilities to better distinguish which code + * is executed and expected to be available where, and after that point we + * should be able to refactor this plugin to take advantage of existing utilities + * like `deleteAllTestLinodes`, `deleteAllTestFirewalls`, etc. + */ + +// Test resource label/name prefix. +const TEST_TAG_PREFIX = 'cy-test-'; + +// Desired number of items per page of a paginated API request. +const PAGE_SIZE = 500; + +/* + * Determines if the given node pool is ready by checking the status of each node. + */ +const isPoolReady = (pool: KubeNodePoolResponse): boolean => + pool.nodes.every((node: PoolNodeResponse) => node.status === 'ready'); + +/** + * Deletes all test Linodes on the test account. + * + * This is a re-implementation of an existing util, `deleteAllTestLinodes`, in + * `support/api/linodes.ts`. + * + * @returns Promise that resolves when all test Linodes are deleted. + */ +const deleteTestLinodes = async () => { + const allLinodes = await depaginate((page) => + getLinodes({ page, page_size: PAGE_SIZE }) + ); + + const deletePromises = allLinodes + .filter((linode: Linode) => linode.label.startsWith(TEST_TAG_PREFIX)) + .map((linode: Linode) => deleteLinode(linode.id)); + + await Promise.all(deletePromises); +}; + +/** + * Deletes all test Firewalls on the test account. + * + * This is a re-implementation of an existing util, `deleteAllTestFirewalls`, in + * `support/api/firewalls.ts`. + * + * @returns Promise that resolves when all test Firewalls are deleted. + */ +const deleteTestFirewalls = async () => { + const allFirewalls = await depaginate((page) => + getFirewalls({ page, page_size: PAGE_SIZE }) + ); + + const deletePromises = allFirewalls + .filter((firewall: Firewall) => firewall.label.startsWith(TEST_TAG_PREFIX)) + .map((firewall: Firewall) => deleteFirewall(firewall.id)); + + await Promise.all(deletePromises); +}; + +/** + * Deletes all running test LKE clusters on the test account. + * + * Sometimes when attempting to delete provisioning LKE clusters, the cluster + * becomes stuck and requires manual intervention to resolve. To reduce the risk + * of this happening, this function will only delete clusters that have finished + * provisioning (i.e. all nodes have `'ready'` status) or which have existed + * for at least an hour. + * + * This is a re-implementation of an existing util, `deleteAllTestLkeClusters`, in + * `support/api/lke.ts`. + * + * @returns Promise that resolves when all test LKE clusters are deleted. + */ +const deleteTestLkeClusters = async () => { + const allClusters = await depaginate((page) => + getKubernetesClusters({ page, page_size: PAGE_SIZE }) + ); + + const clusterDeletionPromises = allClusters + .filter((cluster: KubernetesCluster) => + cluster.label.startsWith(TEST_TAG_PREFIX) + ) + .map(async (cluster: KubernetesCluster) => { + const clusterCreateTime = DateTime.fromISO(cluster.created, { + zone: 'utc', + }); + const createTimeElapsed = Math.abs( + clusterCreateTime.diffNow('minutes').minutes + ); + + // If the test cluster is older than 1 hour, delete it regardless of + // whether or not all of the Node Pools are ready; this is a safeguard + // to prevent LKE clusters with stuck pools from accumulating. + if (createTimeElapsed >= 60) { + return deleteKubernetesCluster(cluster.id); + } + + // If the cluster is not older than 1 hour, only delete it if all of its + // Node Pools are ready. + const pools = await depaginate((page: number) => + getNodePools(cluster.id, { page, page_size: PAGE_SIZE }) + ); + if (pools.every(isPoolReady)) { + return deleteKubernetesCluster(cluster.id); + } + return; + }); + + await Promise.all(clusterDeletionPromises); +}; + +/* + * Human-friendly string describing the types of resources being deleted, + * and their corresponding deletion function. + */ +const resourceCleanUpItems = [ + { name: 'Linodes', cleanUp: deleteTestLinodes }, + // TODO Remove LKE cluster clean up once M3-8656 is complete because cluster cleanup will no longer be necessary. + { name: 'LKE Clusters', cleanUp: deleteTestLkeClusters }, + { name: 'Firewalls', cleanUp: deleteTestFirewalls }, +]; + +export const postRunCleanup: CypressPlugin = async (on) => { + on('after:run', async () => { + console.log('Performing post-run clean up:\n'); + + for (const resourceCleanUpItem of resourceCleanUpItems) { + console.log(`- Cleaning up test ${resourceCleanUpItem.name}...`); + try { + // Perform clean-up sequentially. + // eslint-disable-next-line no-await-in-loop + await resourceCleanUpItem.cleanUp(); + } catch (e) { + console.error( + `\nAn error occurred while cleaning up test ${resourceCleanUpItem.name}:` + ); + if (e.message) { + console.error(e.message); + } + console.error(e); + } + } + console.log('\nPost-run clean up is complete'); + }); +};