diff --git a/ansible/roles/moin-schule-sync/tasks/main.yml b/ansible/roles/moin-schule-sync/tasks/main.yml index 4efee08f12d..3e8cc3ff4d5 100644 --- a/ansible/roles/moin-schule-sync/tasks/main.yml +++ b/ansible/roles/moin-schule-sync/tasks/main.yml @@ -18,3 +18,10 @@ kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" template: moin-schule-users-sync-cronjob-configmap.yml.j2 + +- name: unsynced moin.schule users deletion queueing CronJob + when: WITH_MOIN_SCHULE is defined and WITH_MOIN_SCHULE|bool == true and WITH_UNSYNCED_ENTITIES_DELETION is defined and WITH_UNSYNCED_ENTITIES_DELETION|bool == true + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: moin-schule-users-deletion-queueing-cronjob.yml.j2 diff --git a/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 b/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 new file mode 100644 index 00000000000..9fff4a9337c --- /dev/null +++ b/ansible/roles/moin-schule-sync/templates/moin-schule-users-deletion-queueing-cronjob.yml.j2 @@ -0,0 +1,61 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + namespace: {{ NAMESPACE }} + labels: + app: moin-schule-users-deletion-queueing-cronjob + app.kubernetes.io/part-of: schulcloud-verbund + app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }} + app.kubernetes.io/name: moin-schule-users-deletion-queueing-cronjob + app.kubernetes.io/component: sync + app.kubernetes.io/managed-by: ansible + git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }} + git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }} + name: moin-schule-users-deletion-queueing-cronjob +spec: + schedule: "{{ MOIN_SCHULE_USERS_DELETION_QUEUEING_CRONJOB_SCHEDULE|default("@hourly", true) }}" + jobTemplate: + spec: + template: + spec: + containers: + - name: moin-schule-users-deletion-queueing-cronjob + image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + envFrom: + - secretRef: + name: moin-schule-sync-secret + command: ['/bin/sh','-c'] + args: ['npm run nest:start:deletion-console -- queue unsynced --systemId $SYSTEM_ID'] + resources: + limits: + cpu: {{ API_CPU_LIMITS|default("2000m", true) }} + memory: {{ API_MEMORY_LIMITS|default("2Gi", true) }} + requests: + cpu: {{ API_CPU_REQUESTS|default("100m", true) }} + memory: {{ API_MEMORY_REQUESTS|default("150Mi", true) }} + restartPolicy: OnFailure +{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %} + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/part-of + operator: In + values: + - schulcloud-verbund + topologyKey: "kubernetes.io/hostname" + namespaceSelector: {} +{% endif %} + metadata: + labels: + app: moin-schule-users-deletion-queueing-cronjob + app.kubernetes.io/part-of: schulcloud-verbund + app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }} + app.kubernetes.io/name: moin-schule-users-deletion-queueing-cronjob + app.kubernetes.io/component: sync + app.kubernetes.io/managed-by: ansible + git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }} + git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }} diff --git a/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.spec.ts b/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.spec.ts new file mode 100644 index 00000000000..222a8734fa2 --- /dev/null +++ b/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.spec.ts @@ -0,0 +1,49 @@ +import { ObjectId } from 'bson'; +import { UnsyncedEntitiesOptions } from '../interface'; +import { UnsyncedEntitiesOptionsBuilder } from './unsynced-entities-options.builder'; + +describe(UnsyncedEntitiesOptionsBuilder.name, () => { + describe(UnsyncedEntitiesOptionsBuilder.build.name, () => { + describe('when called with valid arguments', () => { + const setup = () => { + const systemId = new ObjectId().toHexString(); + const unsyncedForMinutes = 3600; + const targetRefDomain = 'school'; + const deleteInMinutes = 43200; + const callsDelayMs = 100; + + const expectedOutput: UnsyncedEntitiesOptions = { + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes, + callsDelayMs, + }; + + return { + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes, + callsDelayMs, + expectedOutput, + }; + }; + + it('should return valid options object with expected values', () => { + const { systemId, unsyncedForMinutes, targetRefDomain, deleteInMinutes, callsDelayMs, expectedOutput } = + setup(); + + const output = UnsyncedEntitiesOptionsBuilder.build( + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes, + callsDelayMs + ); + + expect(output).toStrictEqual(expectedOutput); + }); + }); + }); +}); diff --git a/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.ts b/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.ts new file mode 100644 index 00000000000..8d7d5c8bcd9 --- /dev/null +++ b/apps/server/src/modules/deletion-console/builder/unsynced-entities-options.builder.ts @@ -0,0 +1,20 @@ +import { EntityId } from '@shared/domain/types'; +import { UnsyncedEntitiesOptions } from '../interface'; + +export class UnsyncedEntitiesOptionsBuilder { + static build( + systemId: EntityId, + unsyncedForMinutes: number, + targetRefDomain?: string, + deleteInMinutes?: number, + callsDelayMs?: number + ): UnsyncedEntitiesOptions { + return { + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes, + callsDelayMs, + }; + } +} diff --git a/apps/server/src/modules/deletion-console/deletion-console.module.ts b/apps/server/src/modules/deletion-console/deletion-console.module.ts index cf3a1dee2e7..45898df09b8 100644 --- a/apps/server/src/modules/deletion-console/deletion-console.module.ts +++ b/apps/server/src/modules/deletion-console/deletion-console.module.ts @@ -1,30 +1,48 @@ import { Module } from '@nestjs/common'; -import { HttpModule } from '@nestjs/axios'; -import { ConfigModule } from '@nestjs/config'; import { ConsoleModule } from 'nestjs-console'; +import { ConfigModule } from '@nestjs/config'; +import { MikroOrmModule } from '@mikro-orm/nestjs'; +import { HttpModule } from '@nestjs/axios'; import { ConsoleWriterModule } from '@infra/console'; -import { createConfigModuleOptions } from '@src/config'; -import { DeletionClient } from './deletion-client'; +import { UserModule } from '@modules/user'; +import { ALL_ENTITIES } from '@shared/domain/entity'; +import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config'; +import { defaultMikroOrmOptions } from '@modules/server'; +import { AccountModule } from '@modules/account'; import { getDeletionClientConfig } from './deletion-client/deletion-client.config'; +import { FileEntity } from '../files/entity'; +import { DeletionClient } from './deletion-client'; import { DeletionQueueConsole } from './deletion-queue.console'; -import { DeletionExecutionConsole } from './deletion-execution.console'; -import { BatchDeletionService } from './services'; import { BatchDeletionUc, DeletionExecutionUc } from './uc'; +import { BatchDeletionService } from './services'; +import { DeletionExecutionConsole } from './deletion-execution.console'; @Module({ imports: [ ConsoleModule, ConsoleWriterModule, - HttpModule, + UserModule, ConfigModule.forRoot(createConfigModuleOptions(getDeletionClientConfig)), + MikroOrmModule.forRoot({ + ...defaultMikroOrmOptions, + type: 'mongo', + clientUrl: DB_URL, + password: DB_PASSWORD, + user: DB_USERNAME, + allowGlobalContext: true, + entities: [...ALL_ENTITIES, FileEntity], + debug: true, + }), + AccountModule, + HttpModule, ], providers: [ DeletionClient, - BatchDeletionService, - BatchDeletionUc, - DeletionExecutionUc, DeletionQueueConsole, + BatchDeletionUc, + BatchDeletionService, DeletionExecutionConsole, + DeletionExecutionUc, ], }) export class DeletionConsoleModule {} diff --git a/apps/server/src/modules/deletion-console/deletion-queue.console.spec.ts b/apps/server/src/modules/deletion-console/deletion-queue.console.spec.ts index 07676ad08e4..df3602a80d2 100644 --- a/apps/server/src/modules/deletion-console/deletion-queue.console.spec.ts +++ b/apps/server/src/modules/deletion-console/deletion-queue.console.spec.ts @@ -1,9 +1,11 @@ +import { ObjectId } from 'bson'; import { Test, TestingModule } from '@nestjs/testing'; import { ConsoleWriterService } from '@infra/console'; import { createMock } from '@golevelup/ts-jest'; import { DeletionQueueConsole } from './deletion-queue.console'; import { PushDeleteRequestsOptionsBuilder } from './builder'; import { BatchDeletionUc } from './uc'; +import { UnsyncedEntitiesOptionsBuilder } from './builder/unsynced-entities-options.builder'; describe(DeletionQueueConsole.name, () => { let module: TestingModule; @@ -76,4 +78,34 @@ describe(DeletionQueueConsole.name, () => { }); }); }); + + describe('unsyncedEntities', () => { + describe('when called with an invalid "unsyncedForMinutes" option', () => { + const setup = () => { + const options = UnsyncedEntitiesOptionsBuilder.build(new ObjectId().toHexString(), 15); + + return { options }; + }; + + it('should throw an exception', async () => { + const { options } = setup(); + + await expect(console.unsyncedEntities(options)).rejects.toThrow(); + }); + }); + + describe('when called with valid options', () => { + const setup = () => { + const options = UnsyncedEntitiesOptionsBuilder.build(new ObjectId().toHexString(), 3600); + + return { options }; + }; + + it('should not throw any exception indicating invalid options', async () => { + const { options } = setup(); + + await expect(console.unsyncedEntities(options)).resolves.not.toThrow(); + }); + }); + }); }); diff --git a/apps/server/src/modules/deletion-console/deletion-queue.console.ts b/apps/server/src/modules/deletion-console/deletion-queue.console.ts index eb5ac7dcd6b..2cea000a2ab 100644 --- a/apps/server/src/modules/deletion-console/deletion-queue.console.ts +++ b/apps/server/src/modules/deletion-console/deletion-queue.console.ts @@ -1,9 +1,29 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { Console, Command } from 'nestjs-console'; +import { Console, Command, CommandOption } from 'nestjs-console'; import { ConsoleWriterService } from '@infra/console'; -import { PushDeletionRequestsOptions } from './interface'; +import { PushDeletionRequestsOptions, UnsyncedEntitiesOptions } from './interface'; import { BatchDeletionUc } from './uc'; +const sharedCommandOptions: CommandOption[] = [ + { + flags: '-trd, --targetRefDomain ', + description: 'Name of the target ref domain.', + required: false, + defaultValue: 'user', + }, + { + flags: '-dim, --deleteInMinutes ', + description: 'Number of minutes after which the data deletion process should begin.', + required: false, + defaultValue: 43200, // 43200 minutes = 30 days + }, + { + flags: '-cdm, --callsDelayMs ', + description: 'Delay between all the performed client calls, in milliseconds.', + required: false, + }, +]; + @Console({ command: 'queue', description: 'Console providing an access to the deletion queue.' }) export class DeletionQueueConsole { constructor(private consoleWriter: ConsoleWriterService, private batchDeletionUc: BatchDeletionUc) {} @@ -17,21 +37,7 @@ export class DeletionQueueConsole { description: 'Path of the file containing all the references to the data that should be deleted.', required: true, }, - { - flags: '-trd, --targetRefDomain ', - description: 'Name of the target ref domain.', - required: false, - }, - { - flags: '-dim, --deleteInMinutes ', - description: 'Number of minutes after which the data deletion process should begin.', - required: false, - }, - { - flags: '-cdm, --callsDelayMs ', - description: 'Delay between all the performed client calls, in milliseconds.', - required: false, - }, + ...sharedCommandOptions, ], }) async pushDeletionRequests(options: PushDeletionRequestsOptions): Promise { @@ -44,4 +50,45 @@ export class DeletionQueueConsole { this.consoleWriter.info(JSON.stringify(summary)); } + + @Command({ + command: 'unsynced', + description: 'Finds unsynchronized users and queue them for deletion.', + options: [ + { + flags: '-si, --systemId ', + description: 'ID of a synchronized system.', + required: true, + }, + { + flags: '-ufm, --unsyncedForMinutes ', + description: + 'Number of minutes that must have passed before entity can be considered unsynchronized. Minimum value: 60.', + required: false, + defaultValue: 10080, // 10080 minutes = 7 days + }, + ...sharedCommandOptions, + ], + }) + async unsyncedEntities(options: UnsyncedEntitiesOptions): Promise { + if (options.unsyncedForMinutes < 60) { + throw new Error(`invalid "unsyncedForMinutes" option value - minimum value is 60`); + } + + this.consoleWriter.info( + JSON.stringify({ message: 'starting queueing unsynchronized entities for deletion', options }) + ); + + const summary = await this.batchDeletionUc.deleteUnsynchronizedRefs( + options.systemId, + options.unsyncedForMinutes, + options.targetRefDomain, + options.deleteInMinutes, + options.callsDelayMs + ); + + this.consoleWriter.info( + JSON.stringify({ message: 'successfully finished queueing unsynchronized entities for deletion', summary }) + ); + } } diff --git a/apps/server/src/modules/deletion-console/interface/index.ts b/apps/server/src/modules/deletion-console/interface/index.ts index b15a668b53e..fda9b64ef5f 100644 --- a/apps/server/src/modules/deletion-console/interface/index.ts +++ b/apps/server/src/modules/deletion-console/interface/index.ts @@ -1,4 +1,5 @@ export * from './push-delete-requests-options.interface'; export * from './trigger-deletion-execution-options.interface'; +export * from './unsynced-entities-options.interface'; export * from './deletion-execution-trigger-status.enum'; export * from './deletion-execution-trigger-result'; diff --git a/apps/server/src/modules/deletion-console/interface/unsynced-entities-options.interface.ts b/apps/server/src/modules/deletion-console/interface/unsynced-entities-options.interface.ts new file mode 100644 index 00000000000..6761a7b5712 --- /dev/null +++ b/apps/server/src/modules/deletion-console/interface/unsynced-entities-options.interface.ts @@ -0,0 +1,9 @@ +import { EntityId } from '@shared/domain/types'; + +export interface UnsyncedEntitiesOptions { + systemId: EntityId; + unsyncedForMinutes: number; + targetRefDomain?: string; + deleteInMinutes?: number; + callsDelayMs?: number; +} diff --git a/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.spec.ts b/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.spec.ts index 558f9a73276..1c556d7b00d 100644 --- a/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.spec.ts +++ b/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.spec.ts @@ -1,6 +1,8 @@ import { ObjectId } from '@mikro-orm/mongodb'; import { Test, TestingModule } from '@nestjs/testing'; import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { UserService } from '@modules/user'; +import { AccountService } from '@modules/account'; import { BatchDeletionUc } from './batch-deletion.uc'; import { BatchDeletionService, ReferencesService } from '../services'; import { QueueDeletionRequestOutput } from '../services/interface'; @@ -12,6 +14,8 @@ describe(BatchDeletionUc.name, () => { let module: TestingModule; let uc: BatchDeletionUc; let batchDeletionService: DeepMocked; + let accountService: DeepMocked; + let userService: DeepMocked; beforeAll(async () => { module = await Test.createTestingModule({ @@ -21,11 +25,21 @@ describe(BatchDeletionUc.name, () => { provide: BatchDeletionService, useValue: createMock(), }, + { + provide: UserService, + useValue: createMock(), + }, + { + provide: AccountService, + useValue: createMock(), + }, ], }).compile(); uc = module.get(BatchDeletionUc); batchDeletionService = module.get(BatchDeletionService); + accountService = module.get(AccountService); + userService = module.get(UserService); }); beforeEach(() => { @@ -196,4 +210,178 @@ describe(BatchDeletionUc.name, () => { }); }); }); + + describe('deleteUnsynchronizedRefs', () => { + describe('when called with valid arguments', () => { + describe('when batch deletion service returns an expected amount of outputs', () => { + describe('when only successful executions took place', () => { + const setup = () => { + const targetRefsCount = 3; + const systemId = new ObjectId().toHexString(); + const unsyncedForMinutes = 60; + + const targetRefIds: string[] = []; + const outputs: QueueDeletionRequestOutput[] = []; + const userIds: string[] = []; + + for (let i = 0; i < targetRefsCount; i += 1) { + targetRefIds.push(new ObjectId().toHexString()); + userIds.push(new ObjectId().toHexString()); + outputs.push(QueueDeletionRequestOutputBuilder.buildSuccess(new ObjectId().toHexString(), new Date())); + } + + jest.spyOn(userService, 'findUnsynchronizedUserIds').mockResolvedValueOnce(userIds); + jest.spyOn(accountService, 'findByUserIdsAndSystemId').mockResolvedValueOnce(targetRefIds); + + batchDeletionService.queueDeletionRequests.mockResolvedValueOnce([outputs[0], outputs[1], outputs[2]]); + + const targetRefDomain = 'user'; + const deleteInMinutes = 60; + + const expectedSummaryFieldsDetails: BatchDeletionSummaryDetail[] = []; + + for (let i = 0; i < targetRefIds.length; i += 1) { + expectedSummaryFieldsDetails.push( + BatchDeletionSummaryDetailBuilder.build( + QueueDeletionRequestInputBuilder.build(targetRefDomain, targetRefIds[i], deleteInMinutes), + outputs[i] + ) + ); + } + + const expectedSummaryFields = { + overallStatus: BatchDeletionSummaryOverallStatus.SUCCESS, + successCount: 3, + failureCount: 0, + details: expectedSummaryFieldsDetails, + }; + + return { + targetRefDomain, + deleteInMinutes, + expectedSummaryFields, + systemId, + unsyncedForMinutes, + }; + }; + + it('should return proper summary with all the successes and a successful overall status', async () => { + const { targetRefDomain, deleteInMinutes, expectedSummaryFields, systemId, unsyncedForMinutes } = setup(); + + const summary: BatchDeletionSummary = await uc.deleteUnsynchronizedRefs( + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes + ); + + expect(summary.executionTimeMilliseconds).toBeGreaterThan(0); + expect(summary).toMatchObject(expectedSummaryFields); + }); + }); + + describe('when both successful and failed executions took place', () => { + const setup = () => { + const targetRefsCount = 3; + const systemId = new ObjectId().toHexString(); + const unsyncedForMinutes = 120; + + const targetRefIds: string[] = []; + const userIds: string[] = []; + + for (let i = 0; i < targetRefsCount; i += 1) { + targetRefIds.push(new ObjectId().toHexString()); + userIds.push(new ObjectId().toHexString()); + } + + const targetRefDomain = 'user'; + const deleteInMinutes = 60; + + jest.spyOn(userService, 'findUnsynchronizedUserIds').mockResolvedValueOnce(userIds); + jest.spyOn(accountService, 'findByUserIdsAndSystemId').mockResolvedValueOnce(targetRefIds); + + const outputs = [ + QueueDeletionRequestOutputBuilder.buildSuccess(new ObjectId().toHexString(), new Date()), + QueueDeletionRequestOutputBuilder.buildError(new Error('some error occurred...')), + QueueDeletionRequestOutputBuilder.buildSuccess(new ObjectId().toHexString(), new Date()), + ]; + + batchDeletionService.queueDeletionRequests.mockResolvedValueOnce([outputs[0], outputs[1], outputs[2]]); + + const expectedSummaryFieldsDetails: BatchDeletionSummaryDetail[] = []; + + for (let i = 0; i < targetRefIds.length; i += 1) { + expectedSummaryFieldsDetails.push( + BatchDeletionSummaryDetailBuilder.build( + QueueDeletionRequestInputBuilder.build(targetRefDomain, targetRefIds[i], deleteInMinutes), + outputs[i] + ) + ); + } + + const expectedSummaryFields = { + overallStatus: BatchDeletionSummaryOverallStatus.FAILURE, + successCount: 2, + failureCount: 1, + details: expectedSummaryFieldsDetails, + }; + + return { targetRefDomain, deleteInMinutes, expectedSummaryFields, systemId, unsyncedForMinutes }; + }; + + it('should return proper summary with all the successes and failures', async () => { + const { targetRefDomain, deleteInMinutes, expectedSummaryFields, systemId, unsyncedForMinutes } = setup(); + + const summary: BatchDeletionSummary = await uc.deleteUnsynchronizedRefs( + systemId, + unsyncedForMinutes, + targetRefDomain, + deleteInMinutes + ); + + expect(summary.executionTimeMilliseconds).toBeGreaterThan(0); + expect(summary).toMatchObject(expectedSummaryFields); + }); + }); + }); + + describe('when batch deletion service returns an invalid amount of outputs', () => { + const setup = () => { + const targetRefsCount = 3; + const systemId = new ObjectId().toHexString(); + const unsyncedForMinutes = 120; + + const targetRefIds: string[] = []; + const userIds: string[] = []; + + for (let i = 0; i < targetRefsCount; i += 1) { + targetRefIds.push(new ObjectId().toHexString()); + userIds.push(new ObjectId().toHexString()); + } + + jest.spyOn(userService, 'findUnsynchronizedUserIds').mockResolvedValueOnce(userIds); + jest.spyOn(accountService, 'findByUserIdsAndSystemId').mockResolvedValueOnce(targetRefIds); + + const outputs: QueueDeletionRequestOutput[] = []; + + for (let i = 0; i < targetRefsCount - 1; i += 1) { + targetRefIds.push(new ObjectId().toHexString()); + outputs.push(QueueDeletionRequestOutputBuilder.buildSuccess(new ObjectId().toHexString(), new Date())); + } + + batchDeletionService.queueDeletionRequests.mockResolvedValueOnce(outputs); + + return { systemId, unsyncedForMinutes }; + }; + + it('should throw an error', async () => { + const { systemId, unsyncedForMinutes } = setup(); + + const func = () => uc.deleteUnsynchronizedRefs(systemId, unsyncedForMinutes); + + await expect(func()).rejects.toThrow(); + }); + }); + }); + }); }); diff --git a/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.ts b/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.ts index 9e8ae027063..ce765e4df5f 100644 --- a/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.ts +++ b/apps/server/src/modules/deletion-console/uc/batch-deletion.uc.ts @@ -1,4 +1,6 @@ import { Injectable } from '@nestjs/common'; +import { AccountService } from '@modules/account'; +import { UserService } from '@modules/user'; import { BatchDeletionSummaryBuilder, BatchDeletionSummaryDetailBuilder } from '../builder'; import { BatchDeletionService, ReferencesService } from '../services'; import { QueueDeletionRequestInputBuilder } from '../services/builder'; @@ -7,7 +9,11 @@ import { BatchDeletionSummary, BatchDeletionSummaryOverallStatus } from './inter @Injectable() export class BatchDeletionUc { - constructor(private readonly batchDeletionService: BatchDeletionService) {} + constructor( + private readonly batchDeletionService: BatchDeletionService, + private readonly userService: UserService, + private readonly accountService: AccountService + ) {} async deleteRefsFromTxtFile( refsFilePath: string, @@ -18,13 +24,44 @@ export class BatchDeletionUc { // First, load all the references from the provided text file (with given path). const refsFromTxtFile = ReferencesService.loadFromTxtFile(refsFilePath); + return this.buildInputsQueueDeletionRequestsAndReturnSummary( + refsFromTxtFile, + targetRefDomain, + deleteInMinutes, + callsDelayMilliseconds + ); + } + + async deleteUnsynchronizedRefs( + systemId: string, + unsyncedForMinutes = 10080, + targetRefDomain = 'user', + deleteInMinutes = 43200, // 43200 minutes = 720 hours = 30 days + callsDelayMilliseconds?: number + ): Promise { + const unsynchronizedUserIds = await this.userService.findUnsynchronizedUserIds(unsyncedForMinutes); + + const accountIds = await this.accountService.findByUserIdsAndSystemId(unsynchronizedUserIds, systemId); + + return this.buildInputsQueueDeletionRequestsAndReturnSummary( + accountIds, + targetRefDomain, + deleteInMinutes, + callsDelayMilliseconds + ); + } + + private async buildInputsQueueDeletionRequestsAndReturnSummary( + refs: string[], + targetRefDomain: string, + deleteInMinutes: number, + callsDelayMilliseconds?: number + ): Promise { const inputs: QueueDeletionRequestInput[] = []; // For each reference found in a given file, add it to the inputs // array (with added targetRefDomain and deleteInMinutes fields). - refsFromTxtFile.forEach((ref) => - inputs.push(QueueDeletionRequestInputBuilder.build(targetRefDomain, ref, deleteInMinutes)) - ); + refs.forEach((ref) => inputs.push(QueueDeletionRequestInputBuilder.build(targetRefDomain, ref, deleteInMinutes))); // Measure the overall queueing execution time by setting the start... const startTime = performance.now(); diff --git a/apps/server/src/modules/user/service/user.service.spec.ts b/apps/server/src/modules/user/service/user.service.spec.ts index 3b8816bf1bd..88778dfb5a7 100644 --- a/apps/server/src/modules/user/service/user.service.spec.ts +++ b/apps/server/src/modules/user/service/user.service.spec.ts @@ -904,4 +904,42 @@ describe('UserService', () => { }); }); }); + describe('findUnsynchronizedUserIds', () => { + const setup = () => { + const currentDate = new Date(); + const dateA = new Date(currentDate.getTime() - 120 * 60000); + const dateB = new Date(currentDate.getTime() - 3600 * 60000); + const unsyncedForMinutes = 60; + const userA = userFactory.buildWithId({ lastSyncedAt: dateA }); + const userB = userFactory.buildWithId({ lastSyncedAt: dateB }); + + const foundUsers = [userA.id, userB.id]; + + return { + foundUsers, + unsyncedForMinutes, + }; + }; + + describe('when findUnsynchronizedUserIds is called', () => { + it('should call findUnsynchronizedUserIds and retrun array with found users', async () => { + const { unsyncedForMinutes, foundUsers } = setup(); + + userRepo.findUnsynchronizedUserIds.mockResolvedValueOnce(foundUsers); + + const result = await service.findUnsynchronizedUserIds(unsyncedForMinutes); + + expect(result).toEqual(foundUsers); + }); + + it('should call findUnsynchronizedUserIds and return empty array', async () => { + const { unsyncedForMinutes } = setup(); + + userRepo.findUnsynchronizedUserIds.mockResolvedValueOnce([]); + const result = await service.findUnsynchronizedUserIds(unsyncedForMinutes); + + expect(result).toEqual([]); + }); + }); + }); }); diff --git a/apps/server/src/modules/user/service/user.service.ts b/apps/server/src/modules/user/service/user.service.ts index c1e1a6c2372..1a5a3337880 100644 --- a/apps/server/src/modules/user/service/user.service.ts +++ b/apps/server/src/modules/user/service/user.service.ts @@ -265,6 +265,13 @@ export class UserService implements DeletionService, IEventHandler { + const unsyncedForMiliseconds = unsyncedForMinutes * 60000; + const differenceBetweenCurrentDateAndUnsyncedTime = new Date().getTime() - unsyncedForMiliseconds; + const dateOfLastSyncToBeLookedFrom = new Date(differenceBetweenCurrentDateAndUnsyncedTime); + return this.userRepo.findUnsynchronizedUserIds(dateOfLastSyncToBeLookedFrom); + } + public async removeCalendarEvents(userId: EntityId): Promise { let extractedOperationReport: DomainOperationReport[] = []; const results = await this.calendarService.deleteUserData(userId); diff --git a/apps/server/src/shared/repo/user/user.repo.integration.spec.ts b/apps/server/src/shared/repo/user/user.repo.integration.spec.ts index 589d9bfb9d7..ae1227669e1 100644 --- a/apps/server/src/shared/repo/user/user.repo.integration.spec.ts +++ b/apps/server/src/shared/repo/user/user.repo.integration.spec.ts @@ -679,4 +679,38 @@ describe('user repo', () => { }); }); }); + + describe('findUnsynchronizedUserIds', () => { + describe('when user meets criteria', () => { + const setup = async () => { + const currentDate = new Date(); + const dateB = new Date(currentDate.getTime() - 120 * 60000); + const dateC = new Date(currentDate.getTime() - 3600 * 60000); + const dateToCheckFrom = new Date(currentDate.getTime() - 60 * 60000); + const userA = userFactory.buildWithId({ lastSyncedAt: currentDate }); + const userB = userFactory.buildWithId({ lastSyncedAt: dateB }); + const userC = userFactory.buildWithId({ lastSyncedAt: dateC }); + + await em.persistAndFlush([userA, userB, userC]); + em.clear(); + + const userIds = [userB.id, userC.id]; + + return { + userIds, + dateToCheckFrom, + }; + }; + + it('should find users with appropriate value of lastSyncedAt field', async () => { + const { userIds, dateToCheckFrom } = await setup(); + + const result = await repo.findUnsynchronizedUserIds(dateToCheckFrom); + + expect(result.length).toBe(2); + expect(result).toContain(userIds[0]); + expect(result).toContain(userIds[1]); + }); + }); + }); }); diff --git a/apps/server/src/shared/repo/user/user.repo.ts b/apps/server/src/shared/repo/user/user.repo.ts index 4809929c1f5..8cad0e63734 100644 --- a/apps/server/src/shared/repo/user/user.repo.ts +++ b/apps/server/src/shared/repo/user/user.repo.ts @@ -241,4 +241,14 @@ export class UserRepo extends BaseRepo { { lastSyncedAt: new Date() } ); } + + public async findUnsynchronizedUserIds(dateOfLastSyncToBeLookedFrom: Date): Promise { + const foundUsers = await this._em.find(User, { + lastSyncedAt: { + $lte: dateOfLastSyncToBeLookedFrom, + }, + }); + + return foundUsers.map((user) => user.id); + } }