BC-6420 - unsynced users deletion (#4868)

* crate entity, repo, do, mapper for synchronization and deletion-reconcilation.uc

* add synchronization service and tests

* changes in reconcilation UC

* add lastSyncedAt to userEntity and change create in synchronization service

* create synchronization module and move files from deletion module into it

* add new interface, builders, method in userrepo and new methos in userservice, ande test cases

* some changes

* mofdify acount, user service and synchronization UC

* add some tests, remove userIdAndExternalId interface, builder

* changes in synchronizationEntity

* add staus to synchronizationEntity

* add update method in repo in synchronization module

* fix imports, add update method and test  in service

* add synchromization.module, fix import, modify uc

* fix in acount service test

* change folder structure in synchronization module, fix test in repo

* coments about chunk

* add loggable, and extension of UC

* add test cases to loggable and to userService

* add tests to useCase in synchronizationModule

* add test to UC

* BC-6223-add chunks

* add test cases in UC

* add tests

* remove private methon

* refactor

* add tests and change loop

* add test to UC

* add config

* sum with promise.all

* fixes typo, changes test in UC

* rename testConfig file

* add new command (for queueing unsynchronized entities for deletion)

* add new Ansible role for moin.schule sync with users deletion queueing CronJob task

* Update apps/server/src/shared/repo/user/user.repo.integration.spec.ts

Co-authored-by: Bartosz Nowicki <[email protected]>

* fix some issues after review

* fiz error cases in uc

* add test in UC

* modify SynchronizationErrorLoggableException

* main logic impl

* fix tests

* remove files that were not resolved automatically after merge

* Pr fix and test implementation part1

* test impl part2

* test impl part 3

* test implementation part 3

* add needed modules and providers

* add mongoDb flag

* fix import order

* import fix

* import chnages

* refactor dependencies

---------

Co-authored-by: WojciechGrancow <[email protected]>
Co-authored-by: Wojciech Grancow <[email protected]>
Co-authored-by: micners <[email protected]>
Co-authored-by: WojciechGrancow <[email protected]>
Co-authored-by: Szymon Szafoni <[email protected]>
Co-authored-by: sszafGCA <[email protected]>

ansible/roles/moin-schule-sync/tasks/main.yml

@@ -18,3 +18,10 @@
     kubeconfig: ~/.kube/config
     namespace: "{{ NAMESPACE }}"
     template: moin-schule-users-sync-cronjob-configmap.yml.j2
+
+- name: unsynced moin.schule users deletion queueing CronJob
+  when: WITH_MOIN_SCHULE is defined and WITH_MOIN_SCHULE|bool == true and WITH_UNSYNCED_ENTITIES_DELETION is defined and WITH_UNSYNCED_ENTITIES_DELETION|bool == true
+  kubernetes.core.k8s:
+    kubeconfig: ~/.kube/config
+    namespace: "{{ NAMESPACE }}"
+    template: moin-schule-users-deletion-queueing-cronjob.yml.j2

ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2

@@ -0,0 +1,61 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: moin-schule-users-deletion-queueing-cronjob
+    app.kubernetes.io/part-of: schulcloud-verbund
+    app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+    app.kubernetes.io/name: moin-schule-users-deletion-queueing-cronjob
+    app.kubernetes.io/component: sync
+    app.kubernetes.io/managed-by: ansible
+    git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+    git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+  name: moin-schule-users-deletion-queueing-cronjob
+spec:
+  schedule: "{{ MOIN_SCHULE_USERS_DELETION_QUEUEING_CRONJOB_SCHEDULE|default("@hourly", true) }}"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: moin-schule-users-deletion-queueing-cronjob
+            image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            envFrom:
+            - secretRef:
+                name: moin-schule-sync-secret
+            command: ['/bin/sh','-c']
+            args: ['npm run nest:start:deletion-console -- queue unsynced --systemId $SYSTEM_ID']
+            resources:
+              limits:
+                cpu: {{ API_CPU_LIMITS|default("2000m", true) }}
+                memory: {{ API_MEMORY_LIMITS|default("2Gi", true) }}
+              requests:
+                cpu: {{ API_CPU_REQUESTS|default("100m", true) }}
+                memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }}
+          restartPolicy: OnFailure
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+          affinity:
+            podAffinity:
+              preferredDuringSchedulingIgnoredDuringExecution:
+              - weight: 100
+                podAffinityTerm:
+                  labelSelector:
+                    matchExpressions:
+                    - key: app.kubernetes.io/part-of
+                      operator: In
+                      values:
+                      - schulcloud-verbund
+                  topologyKey: "kubernetes.io/hostname"
+                  namespaceSelector: {}
+{% endif %}
+        metadata:
+          labels:
+            app: moin-schule-users-deletion-queueing-cronjob
+            app.kubernetes.io/part-of: schulcloud-verbund
+            app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+            app.kubernetes.io/name: moin-schule-users-deletion-queueing-cronjob
+            app.kubernetes.io/component: sync
+            app.kubernetes.io/managed-by: ansible
+            git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+            git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}

apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.spec.ts

@@ -0,0 +1,49 @@
+import { ObjectId } from 'bson';
+import { UnsyncedEntitiesOptions } from '../interface';
+import { UnsyncedEntitiesOptionsBuilder } from './unsynced-entities-options.builder';
+
+describe(UnsyncedEntitiesOptionsBuilder.name, () => {
+	describe(UnsyncedEntitiesOptionsBuilder.build.name, () => {
+		describe('when called with valid arguments', () => {
+			const setup = () => {
+				const systemId = new ObjectId().toHexString();
+				const unsyncedForMinutes = 3600;
+				const targetRefDomain = 'school';
+				const deleteInMinutes = 43200;
+				const callsDelayMs = 100;
+
+				const expectedOutput: UnsyncedEntitiesOptions = {
+					systemId,
+					unsyncedForMinutes,
+					targetRefDomain,
+					deleteInMinutes,
+					callsDelayMs,
+				};
+
+				return {
+					systemId,
+					unsyncedForMinutes,
+					targetRefDomain,
+					deleteInMinutes,
+					callsDelayMs,
+					expectedOutput,
+				};
+			};
+
+			it('should return valid options object with expected values', () => {
+				const { systemId, unsyncedForMinutes, targetRefDomain, deleteInMinutes, callsDelayMs, expectedOutput } =
+					setup();
+
+				const output = UnsyncedEntitiesOptionsBuilder.build(
+					systemId,
+					unsyncedForMinutes,
+					targetRefDomain,
+					deleteInMinutes,
+					callsDelayMs
+				);
+
+				expect(output).toStrictEqual(expectedOutput);
+			});
+		});
+	});
+});

apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.ts

@@ -0,0 +1,20 @@
+import { EntityId } from '@shared/domain/types';
+import { UnsyncedEntitiesOptions } from '../interface';
+
+export class UnsyncedEntitiesOptionsBuilder {
+	static build(
+		systemId: EntityId,
+		unsyncedForMinutes: number,
+		targetRefDomain?: string,
+		deleteInMinutes?: number,
+		callsDelayMs?: number
+	): UnsyncedEntitiesOptions {
+		return {
+			systemId,
+			unsyncedForMinutes,
+			targetRefDomain,
+			deleteInMinutes,
+			callsDelayMs,
+		};
+	}
+}

apps/server/src/modules/deletion-console/deletion-console.module.ts

@@ -1,30 +1,48 @@
 import { Module } from '@nestjs/common';
-import { HttpModule } from '@nestjs/axios';
-import { ConfigModule } from '@nestjs/config';
 import { ConsoleModule } from 'nestjs-console';
+import { ConfigModule } from '@nestjs/config';
+import { MikroOrmModule } from '@mikro-orm/nestjs';
+import { HttpModule } from '@nestjs/axios';
 import { ConsoleWriterModule } from '@infra/console';
-import { createConfigModuleOptions } from '@src/config';
-import { DeletionClient } from './deletion-client';
+import { UserModule } from '@modules/user';
+import { ALL_ENTITIES } from '@shared/domain/entity';
+import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config';
+import { defaultMikroOrmOptions } from '@modules/server';
+import { AccountModule } from '@modules/account';
 import { getDeletionClientConfig } from './deletion-client/deletion-client.config';
+import { FileEntity } from '../files/entity';
+import { DeletionClient } from './deletion-client';
 import { DeletionQueueConsole } from './deletion-queue.console';
-import { DeletionExecutionConsole } from './deletion-execution.console';
-import { BatchDeletionService } from './services';
 import { BatchDeletionUc, DeletionExecutionUc } from './uc';
+import { BatchDeletionService } from './services';
+import { DeletionExecutionConsole } from './deletion-execution.console';
 
 @Module({
 	imports: [
 		ConsoleModule,
 		ConsoleWriterModule,
-		HttpModule,
+		UserModule,
 		ConfigModule.forRoot(createConfigModuleOptions(getDeletionClientConfig)),
+		MikroOrmModule.forRoot({
+			...defaultMikroOrmOptions,
+			type: 'mongo',
+			clientUrl: DB_URL,
+			password: DB_PASSWORD,
+			user: DB_USERNAME,
+			allowGlobalContext: true,
+			entities: [...ALL_ENTITIES, FileEntity],
+			debug: true,
+		}),
+		AccountModule,
+		HttpModule,
 	],
 	providers: [
 		DeletionClient,
-		BatchDeletionService,
-		BatchDeletionUc,
-		DeletionExecutionUc,
 		DeletionQueueConsole,
+		BatchDeletionUc,
+		BatchDeletionService,
 		DeletionExecutionConsole,
+		DeletionExecutionUc,
 	],
 })
 export class DeletionConsoleModule {}

apps/server/src/modules/deletion-console/deletion-queue.console.spec.ts

@@ -1,9 +1,11 @@
+import { ObjectId } from 'bson';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ConsoleWriterService } from '@infra/console';
 import { createMock } from '@golevelup/ts-jest';
 import { DeletionQueueConsole } from './deletion-queue.console';
 import { PushDeleteRequestsOptionsBuilder } from './builder';
 import { BatchDeletionUc } from './uc';
+import { UnsyncedEntitiesOptionsBuilder } from './builder/unsynced-entities-options.builder';
 
 describe(DeletionQueueConsole.name, () => {
 	let module: TestingModule;
@@ -76,4 +78,34 @@ describe(DeletionQueueConsole.name, () => {
 			});
 		});
 	});
+
+	describe('unsyncedEntities', () => {
+		describe('when called with an invalid "unsyncedForMinutes" option', () => {
+			const setup = () => {
+				const options = UnsyncedEntitiesOptionsBuilder.build(new ObjectId().toHexString(), 15);
+
+				return { options };
+			};
+
+			it('should throw an exception', async () => {
+				const { options } = setup();
+
+				await expect(console.unsyncedEntities(options)).rejects.toThrow();
+			});
+		});
+
+		describe('when called with valid options', () => {
+			const setup = () => {
+				const options = UnsyncedEntitiesOptionsBuilder.build(new ObjectId().toHexString(), 3600);
+
+				return { options };
+			};
+
+			it('should not throw any exception indicating invalid options', async () => {
+				const { options } = setup();
+
+				await expect(console.unsyncedEntities(options)).resolves.not.toThrow();
+			});
+		});
+	});
 });

apps/server/src/modules/deletion-console/deletion-queue.console.ts

@@ -1,9 +1,29 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { Console, Command } from 'nestjs-console';
+import { Console, Command, CommandOption } from 'nestjs-console';
 import { ConsoleWriterService } from '@infra/console';
-import { PushDeletionRequestsOptions } from './interface';
+import { PushDeletionRequestsOptions, UnsyncedEntitiesOptions } from './interface';
 import { BatchDeletionUc } from './uc';
 
+const sharedCommandOptions: CommandOption[] = [
+	{
+		flags: '-trd, --targetRefDomain <value>',
+		description: 'Name of the target ref domain.',
+		required: false,
+		defaultValue: 'user',
+	},
+	{
+		flags: '-dim, --deleteInMinutes <value>',
+		description: 'Number of minutes after which the data deletion process should begin.',
+		required: false,
+		defaultValue: 43200, // 43200 minutes = 30 days
+	},
+	{
+		flags: '-cdm, --callsDelayMs <value>',
+		description: 'Delay between all the performed client calls, in milliseconds.',
+		required: false,
+	},
+];
+
 @Console({ command: 'queue', description: 'Console providing an access to the deletion queue.' })
 export class DeletionQueueConsole {
 	constructor(private consoleWriter: ConsoleWriterService, private batchDeletionUc: BatchDeletionUc) {}
@@ -17,21 +37,7 @@ export class DeletionQueueConsole {
 				description: 'Path of the file containing all the references to the data that should be deleted.',
 				required: true,
 			},
-			{
-				flags: '-trd, --targetRefDomain <value>',
-				description: 'Name of the target ref domain.',
-				required: false,
-			},
-			{
-				flags: '-dim, --deleteInMinutes <value>',
-				description: 'Number of minutes after which the data deletion process should begin.',
-				required: false,
-			},
-			{
-				flags: '-cdm, --callsDelayMs <value>',
-				description: 'Delay between all the performed client calls, in milliseconds.',
-				required: false,
-			},
+			...sharedCommandOptions,
 		],
 	})
 	async pushDeletionRequests(options: PushDeletionRequestsOptions): Promise<void> {
@@ -44,4 +50,45 @@ export class DeletionQueueConsole {
 
 		this.consoleWriter.info(JSON.stringify(summary));
 	}
+
+	@Command({
+		command: 'unsynced',
+		description: 'Finds unsynchronized users and queue them for deletion.',
+		options: [
+			{
+				flags: '-si, --systemId <value>',
+				description: 'ID of a synchronized system.',
+				required: true,
+			},
+			{
+				flags: '-ufm, --unsyncedForMinutes <value>',
+				description:
+					'Number of minutes that must have passed before entity can be considered unsynchronized. Minimum value: 60.',
+				required: false,
+				defaultValue: 10080, // 10080 minutes = 7 days
+			},
+			...sharedCommandOptions,
+		],
+	})
+	async unsyncedEntities(options: UnsyncedEntitiesOptions): Promise<void> {
+		if (options.unsyncedForMinutes < 60) {
+			throw new Error(`invalid "unsyncedForMinutes" option value - minimum value is 60`);
+		}
+
+		this.consoleWriter.info(
+			JSON.stringify({ message: 'starting queueing unsynchronized entities for deletion', options })
+		);
+
+		const summary = await this.batchDeletionUc.deleteUnsynchronizedRefs(
+			options.systemId,
+			options.unsyncedForMinutes,
+			options.targetRefDomain,
+			options.deleteInMinutes,
+			options.callsDelayMs
+		);
+
+		this.consoleWriter.info(
+			JSON.stringify({ message: 'successfully finished queueing unsynchronized entities for deletion', summary })
+		);
+	}
 }

apps/server/src/modules/deletion-console/interface/index.ts

@@ -1,4 +1,5 @@
 export * from './push-delete-requests-options.interface';
 export * from './trigger-deletion-execution-options.interface';
+export * from './unsynced-entities-options.interface';
 export * from './deletion-execution-trigger-status.enum';
 export * from './deletion-execution-trigger-result';

apps/server/src/modules/deletion-console/interface/unsynced-entities-options.interface.ts

@@ -0,0 +1,9 @@
+import { EntityId } from '@shared/domain/types';
+
+export interface UnsyncedEntitiesOptions {
+	systemId: EntityId;
+	unsyncedForMinutes: number;
+	targetRefDomain?: string;
+	deleteInMinutes?: number;
+	callsDelayMs?: number;
+}