BC-5628 cyclic data deletion

ansible/roles/schulcloud-server-core/tasks/main.yml

@@ -37,6 +37,13 @@
       template: onepassword.yml.j2
     when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool
 
+  - name: Admin API client secret (from 1Password)
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: onepassword-admin-api-client.yml.j2
+    when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool
+
   - name: remove old migration Job
     kubernetes.core.k8s:
       kubeconfig: ~/.kube/config
@@ -108,6 +115,12 @@
       namespace: "{{ NAMESPACE }}"
       template: api-delete-s3-files-cronjob.yml.j2
 
+  - name: Data deletion trigger CronJob
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: data-deletion-trigger-cronjob.yml.j2
+
   - name: AMQPFileStorageDeployment
     kubernetes.core.k8s:
       kubeconfig: ~/.kube/config

ansible/roles/schulcloud-server-core/templates/data-deletion-trigger-cronjob.yml.j2

@@ -0,0 +1,58 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: data-deletion-trigger
+    app.kubernetes.io/part-of: schulcloud-verbund
+    app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+    app.kubernetes.io/name: data-deletion-trigger
+    app.kubernetes.io/component: data-deletion
+    app.kubernetes.io/managed-by: ansible
+    git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+    git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+  name: data-deletion-trigger-cronjob
+spec:
+  concurrencyPolicy: Forbid
+  schedule: "{{ SERVER_DATA_DELETION_TRIGGER_CRONJOB_SCHEDULE|default("@hourly", true) }}"
+  jobTemplate:
+    metadata:
+      labels:
+        app: data-deletion-trigger
+        app.kubernetes.io/part-of: schulcloud-verbund
+        app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        app.kubernetes.io/name: data-deletion-trigger
+        app.kubernetes.io/component: data-deletion
+        app.kubernetes.io/managed-by: ansible
+        git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+        git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+    spec:
+      template:
+        spec:
+          containers:
+          - name: data-deletion-trigger-cronjob
+            image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            envFrom:
+            - secretRef:
+                name: admin-api-client-secret
+            command: ['/bin/sh', '-c']
+            args: ['npm run nest:start:deletion-console -- execution trigger']
+            resources:
+              limits:
+                cpu: {{ API_CPU_LIMITS|default("2000m", true) }}
+                memory: {{ API_MEMORY_LIMITS|default("2Gi", true) }}
+              requests:
+                cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
+                memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
+          restartPolicy: OnFailure
+        metadata:
+          labels:
+            app: data-deletion-trigger
+            cronjob: data-deletion-trigger
+            app.kubernetes.io/part-of: schulcloud-verbund
+            app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            app.kubernetes.io/name: data-deletion-trigger
+            app.kubernetes.io/component: data-deletion
+            app.kubernetes.io/managed-by: ansible
+            git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+            git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}

ansible/roles/schulcloud-server-core/templates/onepassword-admin-api-client.yml.j2

@@ -0,0 +1,7 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: admin-api-client-secret
+  namespace: {{ NAMESPACE }}
+spec:
+  itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/admin-api-client"

apps/server/src/modules/deletion/client/deletion.client.spec.ts

@@ -1,4 +1,4 @@
-import { of } from 'rxjs';
+import { of, throwError } from 'rxjs';
 import { AxiosResponse } from 'axios';
 import { HttpService } from '@nestjs/axios';
 import { ConfigService } from '@nestjs/config';
@@ -51,6 +51,23 @@ describe(DeletionClient.name, () => {
 	});
 
 	describe('queueDeletionRequest', () => {
+		describe('when sending the HTTP request failed', () => {
+			const setup = () => {
+				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
+
+				const error = new Error('unknown error');
+				httpService.post.mockReturnValueOnce(throwError(() => error));
+
+				return { input };
+			};
+
+			it('should catch and throw an error', async () => {
+				const { input } = setup();
+
+				await expect(client.queueDeletionRequest(input)).rejects.toThrow(Error);
+			});
+		});
+
 		describe('when received valid response with expected HTTP status code', () => {
 			const setup = () => {
 				const input = DeletionRequestInputBuilder.build('user', '652f1625e9bc1a13bdaae48b');
@@ -151,4 +168,55 @@ describe(DeletionClient.name, () => {
 			});
 		});
 	});
+
+	describe('executeDeletions', () => {
+		describe('when sending the HTTP request failed', () => {
+			const setup = () => {
+				const error = new Error('unknown error');
+				httpService.post.mockReturnValueOnce(throwError(() => error));
+			};
+
+			it('should catch and throw an error', async () => {
+				setup();
+
+				await expect(client.executeDeletions()).rejects.toThrow(Error);
+			});
+		});
+
+		describe('when received valid response with expected HTTP status code', () => {
+			const setup = () => {
+				const limit = 10;
+
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					status: 204,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+
+				return { limit };
+			};
+
+			it('should return proper output', async () => {
+				const { limit } = setup();
+
+				await expect(client.executeDeletions(limit)).resolves.not.toThrow();
+			});
+		});
+
+		describe('when received invalid HTTP status code in a response', () => {
+			const setup = () => {
+				const response: AxiosResponse<DeletionRequestOutput> = axiosResponseFactory.build({
+					status: 200,
+				});
+
+				httpService.post.mockReturnValueOnce(of(response));
+			};
+
+			it('should throw an exception', async () => {
+				setup();
+
+				await expect(client.executeDeletions()).rejects.toThrow(Error);
+			});
+		});
+	});
 });

apps/server/src/modules/deletion/client/deletion.client.ts

@@ -1,9 +1,9 @@
-import { firstValueFrom } from 'rxjs';
-import { AxiosResponse } from 'axios';
-import { Injectable } from '@nestjs/common';
 import { HttpService } from '@nestjs/axios';
+import { BadGatewayException, Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { DeletionRequestInput, DeletionRequestOutput, DeletionClientConfig } from './interface';
+import { ErrorUtils } from '@src/core/error/utils';
+import { firstValueFrom } from 'rxjs';
+import { DeletionClientConfig, DeletionRequestInput, DeletionRequestOutput } from './interface';
 
 @Injectable()
 export class DeletionClient {
@@ -13,6 +13,8 @@ export class DeletionClient {
 
 	private readonly postDeletionRequestsEndpoint: string;
 
+	private readonly postDeletionExecutionsEndpoint: string;
+
 	constructor(
 		private readonly httpService: HttpService,
 		private readonly configService: ConfigService<DeletionClientConfig, true>
@@ -22,36 +24,67 @@ export class DeletionClient {
 
 		// Prepare the POST /deletionRequests endpoint beforehand to not do it on every client call.
 		this.postDeletionRequestsEndpoint = new URL('/admin/api/v1/deletionRequests', this.baseUrl).toString();
+		this.postDeletionExecutionsEndpoint = new URL('/admin/api/v1/deletionExecutions', this.baseUrl).toString();
 	}
 
 	async queueDeletionRequest(input: DeletionRequestInput): Promise<DeletionRequestOutput> {
-		const request = this.httpService.post(this.postDeletionRequestsEndpoint, input, this.defaultHeaders());
-
-		return firstValueFrom(request)
-			.then((resp: AxiosResponse<DeletionRequestOutput>) => {
-				// Throw an error if any other status code (other than expected "202 Accepted" is returned).
-				if (resp.status !== 202) {
-					throw new Error(`invalid HTTP status code in a response from the server - ${resp.status} instead of 202`);
-				}
-
-				// Throw an error if server didn't return a requestId in a response (and it is
-				// required as it gives client the reference to the created deletion request).
-				if (!resp.data.requestId) {
-					throw new Error('no valid requestId returned from the server');
-				}
-
-				// Throw an error if server didn't return a deletionPlannedAt timestamp so the user
-				// will not be aware after which date the deletion request's execution will begin.
-				if (!resp.data.deletionPlannedAt) {
-					throw new Error('no valid deletionPlannedAt returned from the server');
-				}
-
-				return resp.data;
-			})
-			.catch((err: Error) => {
-				// Throw an error if sending/processing deletion request by the client failed in any way.
-				throw new Error(`failed to send/process a deletion request: ${err.toString()}`);
-			});
+		try {
+			const request = this.httpService.post<DeletionRequestOutput>(
+				this.postDeletionRequestsEndpoint,
+				input,
+				this.defaultHeaders()
+			);
+
+			const resp = await firstValueFrom(request);
+
+			// Throw an error if any other status code (other than expected "202 Accepted" is returned).
+			if (resp.status !== 202) {
+				throw new Error(`invalid HTTP status code in a response from the server - ${resp.status} instead of 202`);
+			}
+
+			// Throw an error if server didn't return a requestId in a response (and it is
+			// required as it gives client the reference to the created deletion request).
+			if (!resp.data.requestId) {
+				throw new Error('no valid requestId returned from the server');
+			}
+
+			// Throw an error if server didn't return a deletionPlannedAt timestamp so the user
+			// will not be aware after which date the deletion request's execution will begin.
+			if (!resp.data.deletionPlannedAt) {
+				throw Error('no valid deletionPlannedAt returned from the server');
+			}
+
+			return resp.data;
+		} catch (err) {
+			// Throw an error if sending deletion request has failed.
+			throw new BadGatewayException('DeletionClient:queueDeletionRequest', ErrorUtils.createHttpExceptionOptions(err));
+		}
+	}
+
+	async executeDeletions(limit?: number): Promise<void> {
+		let requestConfig = {};
+
+		if (limit && limit > 0) {
+			requestConfig = { ...this.defaultHeaders(), params: { limit } };
+		} else {
+			requestConfig = { ...this.defaultHeaders() };
+		}
+
+		try {
+			const request = this.httpService.post(this.postDeletionExecutionsEndpoint, null, requestConfig);
+
+			const resp = await firstValueFrom(request);
+
+			if (resp.status !== 204) {
+				// Throw an error if any other status code (other than expected "204 No Content" is returned).
+				const err = new Error(`invalid HTTP status code in a response from the server - ${resp.status} instead of 204`);
+
+				throw new BadGatewayException('DeletionClient:executeDeletions', ErrorUtils.createHttpExceptionOptions(err));
+			}
+		} catch (err: unknown) {
+			// Throw an error if sending deletion request(s) execution trigger has failed.
+			throw new BadGatewayException('DeletionClient:executeDeletions', ErrorUtils.createHttpExceptionOptions(err));
+		}
 	}
 
 	private apiKeyHeader() {

apps/server/src/modules/deletion/console/builder/deletion-execution-trigger-result.builder.spec.ts

@@ -0,0 +1,45 @@
+import { DeletionExecutionTriggerResult, DeletionExecutionTriggerStatus } from '../interface';
+import { DeletionExecutionTriggerResultBuilder } from './deletion-execution-trigger-result.builder';
+
+describe(DeletionExecutionTriggerResultBuilder.name, () => {
+	describe(DeletionExecutionTriggerResultBuilder.buildSuccess.name, () => {
+		describe('when called', () => {
+			const setup = () => {
+				const expectedOutput: DeletionExecutionTriggerResult = { status: DeletionExecutionTriggerStatus.SUCCESS };
+
+				return { expectedOutput };
+			};
+
+			it('should return valid object indicating success', () => {
+				const { expectedOutput } = setup();
+
+				const output = DeletionExecutionTriggerResultBuilder.buildSuccess();
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+
+	describe(DeletionExecutionTriggerResultBuilder.buildFailure.name, () => {
+		describe('when called with proper arguments', () => {
+			const setup = () => {
+				const error = new Error('test error message');
+
+				const expectedOutput: DeletionExecutionTriggerResult = {
+					status: DeletionExecutionTriggerStatus.FAILURE,
+					error: error.toString(),
+				};
+
+				return { error, expectedOutput };
+			};
+
+			it('should return valid object with expected values', () => {
+				const { error, expectedOutput } = setup();
+
+				const output = DeletionExecutionTriggerResultBuilder.buildFailure(error);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});

apps/server/src/modules/deletion/console/builder/deletion-execution-trigger-result.builder.ts

@@ -0,0 +1,21 @@
+import { DeletionExecutionTriggerResult, DeletionExecutionTriggerStatus } from '../interface';
+
+export class DeletionExecutionTriggerResultBuilder {
+	private static build(status: DeletionExecutionTriggerStatus, error?: string): DeletionExecutionTriggerResult {
+		const output: DeletionExecutionTriggerResult = { status };
+
+		if (error) {
+			output.error = error;
+		}
+
+		return output;
+	}
+
+	static buildSuccess(): DeletionExecutionTriggerResult {
+		return this.build(DeletionExecutionTriggerStatus.SUCCESS);
+	}
+
+	static buildFailure(err: Error): DeletionExecutionTriggerResult {
+		return this.build(DeletionExecutionTriggerStatus.FAILURE, err.toString());
+	}
+}

apps/server/src/modules/deletion/console/builder/index.ts

@@ -1 +1,3 @@
 export * from './push-delete-requests-options.builder';
+export * from './trigger-deletion-execution-options.builder';
+export * from './deletion-execution-trigger-result.builder';

apps/server/src/modules/deletion/console/builder/trigger-deletion-execution-options.builder.spec.ts

@@ -0,0 +1,24 @@
+import { TriggerDeletionExecutionOptions } from '../interface';
+import { TriggerDeletionExecutionOptionsBuilder } from './trigger-deletion-execution-options.builder';
+
+describe(TriggerDeletionExecutionOptionsBuilder.name, () => {
+	describe(TriggerDeletionExecutionOptionsBuilder.build.name, () => {
+		describe('when called with proper arguments', () => {
+			const setup = () => {
+				const limit = 1000;
+
+				const expectedOutput: TriggerDeletionExecutionOptions = { limit };
+
+				return { limit, expectedOutput };
+			};
+
+			it('should return valid object with expected values', () => {
+				const { limit, expectedOutput } = setup();
+
+				const output = TriggerDeletionExecutionOptionsBuilder.build(limit);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});