Spsh-1574: Fortschrittsanzeige & Bulk-Transaktionen

src/modules/import/api/import.controller.integration-spec.ts

@@ -786,15 +786,20 @@ describe('Import API', () => {
     describe('/GET importstatus by id', () => {
         it('should return 200 OK with import ststus', async () => {
             const importVorgang: ImportVorgang<true> = await importVorgangRepository.save(
-                DoFactory.createImportVorgang(false, { status: ImportStatus.COMPLETED }),
+                DoFactory.createImportVorgang(false, { status: ImportStatus.COMPLETED, totalDataItemImported: 100 }),
             );
 
             const response: Response = await request(app.getHttpServer() as App)
                 .get(`/import/${importVorgang.id}/status`)
                 .send();
 
             expect(response.status).toBe(200);
-            expect(response.body).toEqual({ status: ImportStatus.COMPLETED } as ImportVorgangStatusResponse);
+            expect(response.body).toBeInstanceOf(Object);
+            expect(response.body).toEqual({
+                dataItemCount: 100,
+                status: ImportStatus.COMPLETED,
+                totalDataItemImported: 100,
+            } as ImportVorgangStatusResponse);
         });
 
         it('should return 404 if importvorgang does not exist', async () => {

src/modules/import/api/importvorgang-status.response.ts

@@ -3,10 +3,18 @@ import { ImportStatus, ImportStatusName } from '../domain/import.enums.js';
 import { ImportVorgang } from '../domain/import-vorgang.js';
 
 export class ImportVorgangStatusResponse {
+    @ApiProperty()
+    public dataItemCount: number;
+
     @ApiProperty({ enum: ImportStatus, enumName: ImportStatusName })
     public status: ImportStatus;
 
+    @ApiProperty()
+    public totalDataItemImported: number;
+
     public constructor(importVorgang: ImportVorgang<true>) {
+        this.dataItemCount = importVorgang.dataItemCount;
         this.status = importVorgang.status;
+        this.totalDataItemImported = importVorgang.totalDataItemImported;
     }
 }

src/modules/import/domain/import-event-handler.spec.ts

@@ -23,6 +23,7 @@ import { ClassLogger } from '../../../core/logging/class-logger.js';
 import { ImportExecutedEvent } from '../../../shared/events/import-executed.event.js';
 import { RolleNurAnPassendeOrganisationError } from '../../personenkontext/specification/error/rolle-nur-an-passende-organisation.js';
 import { ImportPasswordEncryptor } from './import-password-encryptor.js';
+import { ImportDomainError } from './import-domain.error.js';
 
 describe('ImportEventHandler', () => {
     let module: TestingModule;
@@ -167,7 +168,12 @@ describe('ImportEventHandler', () => {
             personenkontextCreationServiceMock.createPersonWithPersonenkontexte.mockResolvedValueOnce(error);
             organisationRepoMock.findById.mockResolvedValueOnce(schule);
 
-            await sut.handleExecuteImport(event);
+            const importDomainError: DomainError = new ImportDomainError(
+                `The creation of person with personenkontexte for the import transaction:${importvorgang.id} failed`,
+                importvorgang.id,
+            );
+
+            await expect(sut.handleExecuteImport(event)).rejects.toThrowError(importDomainError);
 
             expect(loggerMock.error).toHaveBeenCalledWith(
                 `System hat versucht einen neuen Benutzer für ${importDataItem.vorname} ${importDataItem.nachname} anzulegen. Fehler: ${error.message}`,
@@ -206,7 +212,12 @@ describe('ImportEventHandler', () => {
             personenkontextCreationServiceMock.createPersonWithPersonenkontexte.mockResolvedValueOnce(pks);
             organisationRepoMock.findById.mockResolvedValueOnce(schule);
 
-            await sut.handleExecuteImport(event);
+            const error: DomainError = new ImportDomainError(
+                `The creation for a password for the person with ID ${person.id} for the import transaction:${importvorgang.id} has failed`,
+                importvorgang.id,
+            );
+
+            await expect(sut.handleExecuteImport(event)).rejects.toThrowError(error);
 
             expect(person.newPassword).toBeUndefined();
             expect(loggerMock.error).toHaveBeenCalledWith(`Person with ID ${person.id} has no start password!`);
@@ -258,8 +269,8 @@ describe('ImportEventHandler', () => {
             expect(loggerMock.info).toHaveBeenCalledWith(
                 `System hat einen neuen Benutzer ${person.referrer} (${person.id}) angelegt.`,
             );
-            expect(importDataRepositoryMock.save).toHaveBeenCalled();
-            expect(importVorgangRepositoryMock.save).toHaveBeenCalledTimes(1);
+            expect(importDataRepositoryMock.replaceAll).toHaveBeenCalled();
+            expect(importVorgangRepositoryMock.save).toHaveBeenCalledTimes(2);
         });
     });
 });

src/modules/import/domain/import-event-handler.ts

@@ -18,12 +18,16 @@ import { OrganisationByIdAndName } from './import-workflow.js';
 import { Injectable } from '@nestjs/common';
 import { ClassLogger } from '../../../core/logging/class-logger.js';
 import { ImportPasswordEncryptor } from './import-password-encryptor.js';
+import { PersonPermissions } from '../../authentication/domain/person-permissions.js';
+import { ImportDomainError } from './import-domain.error.js';
 @Injectable()
 export class ImportEventHandler {
     public selectedOrganisationId!: string;
 
     public selectedRolleId!: string;
 
+    private readonly NUMBER_OF_USERS_PRO_REQUEST: number = 25;
+
     public constructor(
         private readonly organisationRepository: OrganisationRepository,
         private readonly importDataRepository: ImportDataRepository,
@@ -51,7 +55,6 @@ export class ImportEventHandler {
             }
         });
 
-        const importDataItemsWithLoginInfo: ImportDataItem<true>[] = [];
         const importvorgangId: string = event.importVorgangId;
         const importVorgang: Option<ImportVorgang<true>> = await this.importVorgangRepository.findById(importvorgangId);
         if (!importVorgang) {
@@ -64,9 +67,37 @@ export class ImportEventHandler {
             return this.logger.error(`No import data itemns found for Importvorgang:${importvorgangId}`);
         }
 
+        /* eslint-disable no-await-in-loop */
+        while (importDataItems.length > 0) {
+            const dataItemsToImport: ImportDataItem<true>[] = importDataItems.splice(
+                0,
+                this.NUMBER_OF_USERS_PRO_REQUEST,
+            );
+            await this.savePersonWithPersonenkontext(
+                importVorgang,
+                dataItemsToImport,
+                klassenByIDandName,
+                event.permissions,
+            );
+
+            importVorgang.incrementTotalImportDataItems(dataItemsToImport.length);
+            await this.importVorgangRepository.save(importVorgang);
+        }
+
+        importVorgang.finish();
+        await this.importVorgangRepository.save(importVorgang);
+    }
+
+    private async savePersonWithPersonenkontext(
+        importVorgang: ImportVorgang<true>,
+        dataItems: ImportDataItem<true>[],
+        klassenByIDandName: OrganisationByIdAndName[],
+        permissions: PersonPermissions,
+    ): Promise<void> {
+        const importDataItemsWithLoginInfo: ImportDataItem<true>[] = [];
         //We must create every peron individually otherwise it cannot assign the correct username when we have multiple users with the same name
         /* eslint-disable no-await-in-loop */
-        for (const importDataItem of importDataItems) {
+        for (const importDataItem of dataItems) {
             const klasse: OrganisationByIdAndName | undefined = klassenByIDandName.find(
                 (organisationByIdAndName: OrganisationByIdAndName) =>
                     organisationByIdAndName.name === importDataItem.klasse,
@@ -93,7 +124,7 @@ export class ImportEventHandler {
 
             const savedPersonWithPersonenkontext: DomainError | PersonPersonenkontext =
                 await this.personenkontextCreationService.createPersonWithPersonenkontexte(
-                    event.permissions,
+                    permissions,
                     importDataItem.vorname,
                     importDataItem.nachname,
                     createPersonenkontexte,
@@ -104,14 +135,22 @@ export class ImportEventHandler {
                     `System hat einen neuen Benutzer ${savedPersonWithPersonenkontext.person.referrer} (${savedPersonWithPersonenkontext.person.id}) angelegt.`,
                 );
             } else {
-                return this.logger.error(
+                this.logger.error(
                     `System hat versucht einen neuen Benutzer für ${importDataItem.vorname} ${importDataItem.nachname} anzulegen. Fehler: ${savedPersonWithPersonenkontext.message}`,
                 );
+
+                throw new ImportDomainError(
+                    `The creation of person with personenkontexte for the import transaction:${importVorgang.id} failed`,
+                    importVorgang.id,
+                );
             }
 
             if (!savedPersonWithPersonenkontext.person.newPassword) {
-                return this.logger.error(
-                    `Person with ID ${savedPersonWithPersonenkontext.person.id} has no start password!`,
+                this.logger.error(`Person with ID ${savedPersonWithPersonenkontext.person.id} has no start password!`);
+
+                throw new ImportDomainError(
+                    `The creation for a password for the person with ID ${savedPersonWithPersonenkontext.person.id} for the import transaction:${importVorgang.id} has failed`,
+                    importVorgang.id,
                 );
             }
 
@@ -134,14 +173,6 @@ export class ImportEventHandler {
             );
         }
         /* eslint-disable no-await-in-loop */
-
-        await Promise.allSettled(
-            importDataItemsWithLoginInfo.map(async (importDataItem: ImportDataItem<true>) =>
-                this.importDataRepository.save(importDataItem),
-            ),
-        );
-
-        importVorgang.finish();
-        await this.importVorgangRepository.save(importVorgang);
+        await this.importDataRepository.replaceAll(importDataItemsWithLoginInfo);
     }
 }

src/modules/import/domain/import-vorgang.ts

@@ -10,6 +10,7 @@ export class ImportVorgang<WasPersisted extends boolean> {
         public organisationsname: string,
         public dataItemCount: number,
         public status: Persisted<ImportStatus, WasPersisted>,
+        public totalDataItemImported: number,
         public importByPersonId?: string,
         public rolleId?: string,
         public organisationId?: string,
@@ -24,6 +25,7 @@ export class ImportVorgang<WasPersisted extends boolean> {
         organisationsname: string,
         dataItemCount: number,
         status: ImportStatus,
+        totalDataItemImported: number,
         importByPersonId?: string,
         rolleId?: string,
         organisationId?: string,
@@ -37,6 +39,7 @@ export class ImportVorgang<WasPersisted extends boolean> {
             organisationsname,
             dataItemCount,
             status,
+            totalDataItemImported,
             importByPersonId,
             rolleId,
             organisationId,
@@ -61,6 +64,7 @@ export class ImportVorgang<WasPersisted extends boolean> {
             organisationsname,
             dataItemCount,
             undefined,
+            0,
             importByPersonId,
             rolleId,
             organisationId,
@@ -90,4 +94,8 @@ export class ImportVorgang<WasPersisted extends boolean> {
     public finish(): void {
         this.status = ImportStatus.FINISHED;
     }
+
+    public incrementTotalImportDataItems(totalDataItemImported: number): void {
+        this.totalDataItemImported += totalDataItemImported;
+    }
 }

src/modules/import/domain/import-workflow.ts

@@ -186,56 +186,63 @@ export class ImportWorkflow {
         );
 
         const savedImportvorgang: ImportVorgang<true> = await this.importVorgangRepository.save(importVorgang);
+        const totalImportDataItems: number = parsedDataItems.length;
+        /* eslint-disable no-await-in-loop */
+        while (parsedDataItems.length > 0) {
+            const dataItems: CSVImportDataItemDTO[] = parsedDataItems.splice(0, 50);
 
-        const promises: Promise<ImportDataItem<true>>[] = parsedDataItems.map((value: CSVImportDataItemDTO) => {
-            const importDataItemErrors: string[] = [];
+            const importDataItems: ImportDataItem<false>[] = dataItems.map((value: CSVImportDataItemDTO) => {
+                const importDataItemErrors: string[] = [];
 
-            // Validate object
-            for (const error of validateSync(value, { forbidUnknownValues: true })) {
-                if (error.constraints) {
-                    for (const message of Object.values(error.constraints)) {
-                        importDataItemErrors.push(message);
+                // Validate object
+                for (const error of validateSync(value, { forbidUnknownValues: true })) {
+                    if (error.constraints) {
+                        for (const message of Object.values(error.constraints)) {
+                            importDataItemErrors.push(message);
+                        }
                     }
                 }
-            }
 
-            if (value.klasse) {
-                const klasse: OrganisationByIdAndName | undefined = klassenByIDandName.find(
-                    (organisationByIdAndName: OrganisationByIdAndName) => organisationByIdAndName.name === value.klasse, //Klassennamen sind case sensitive
-                );
+                if (value.klasse) {
+                    const klasse: OrganisationByIdAndName | undefined = klassenByIDandName.find(
+                        (organisationByIdAndName: OrganisationByIdAndName) =>
+                            organisationByIdAndName.name === value.klasse, //Klassennamen sind case sensitive
+                    );
 
-                //Only check if the Klasse exists
-                //Do not need to check if the Klasse can be assigned to rolle for now, because we only impport RollenArt=LERN
-                if (!klasse) {
-                    importDataItemErrors.push(ImportDomainErrorI18nTypes.IMPORT_DATA_ITEM_KLASSE_NOT_FOUND);
+                    //Only check if the Klasse exists
+                    //Do not need to check if the Klasse can be assigned to rolle for now, because we only impport RollenArt=LERN
+                    if (!klasse) {
+                        importDataItemErrors.push(ImportDomainErrorI18nTypes.IMPORT_DATA_ITEM_KLASSE_NOT_FOUND);
+                    }
                 }
-            }
 
-            const importDataItem: ImportDataItem<false> = ImportDataItem.createNew(
-                savedImportvorgang.id,
-                value.nachname,
-                value.vorname,
-                value.klasse,
-                value.personalnummer,
-                importDataItemErrors,
-            );
+                const importDataItem: ImportDataItem<false> = ImportDataItem.createNew(
+                    savedImportvorgang.id,
+                    value.nachname,
+                    value.vorname,
+                    value.klasse,
+                    value.personalnummer,
+                    importDataItemErrors,
+                );
 
-            if (importDataItemErrors.length > 0) {
-                invalidImportDataItems.push(importDataItem);
-            }
+                if (importDataItemErrors.length > 0) {
+                    invalidImportDataItems.push(importDataItem);
+                }
 
-            return this.importDataRepository.save(importDataItem);
-        });
+                return importDataItem;
+            });
 
-        await Promise.all(promises);
+            await this.importDataRepository.createAll(importDataItems);
+        }
+        /* eslint-disable no-await-in-loop */
 
         savedImportvorgang.validate(invalidImportDataItems.length);
         await this.importVorgangRepository.save(savedImportvorgang);
 
         return {
             importVorgangId: savedImportvorgang.id,
             isValid: invalidImportDataItems.length === 0,
-            totalImportDataItems: parsedDataItems.length,
+            totalImportDataItems: totalImportDataItems,
             totalInvalidImportDataItems: invalidImportDataItems.length,
             invalidImportDataItems,
         };
@@ -250,11 +257,6 @@ export class ImportWorkflow {
             };
         }
 
-        // Get all import data items with importvorgangId
-        //Optimierung: für das folgeTicket mit z.B. 800 Lehrer , muss der thread so manipuliert werden (sobald ein Resultat da ist, wird der nächste request abgeschickt)
-        //Optimierung: Process 10 dataItems at time for createPersonWithPersonenkontexte
-        // const offset: number = 0;
-        // const limit: number = 10;
         const importVorgang: Option<ImportVorgang<true>> = await this.importVorgangRepository.findById(importvorgangId);
         if (!importVorgang) {
             this.logger.warning(`Importvorgang: ${importvorgangId} not found`);