From 3c704adcda2d69d49fc801fecb732e70836f34b4 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 2 Jan 2024 15:37:11 +0100 Subject: [PATCH 01/30] Add fileExists property to fileRecord --- .../src/modules/files-storage/entity/filerecord.entity.ts | 4 ++++ .../modules/files-storage/service/files-storage.service.ts | 3 +++ 2 files changed, 7 insertions(+) diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts index 9278d93a4fa..85e0a3ca401 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts @@ -120,6 +120,9 @@ export class FileRecord extends BaseEntityWithTimestamps { @Enum() parentType: FileRecordParentType; + @Property() + fileExists: boolean; + @Index() @Property({ fieldName: 'parent' }) _parentId: ObjectId; @@ -161,6 +164,7 @@ export class FileRecord extends BaseEntityWithTimestamps { this.name = props.name; this.mimeType = props.mimeType; this.parentType = props.parentType; + this.fileExists = false; this._parentId = new ObjectId(props.parentId); if (props.creatorId !== undefined) { this._creatorId = new ObjectId(props.creatorId); diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 449971b5542..aebf5d5596e 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -180,6 +180,9 @@ export class FilesStorageService { // The actual file size is set here because it is known only after the whole file is streamed. fileRecord.size = await fileSizePromise; this.throwErrorIfFileIsTooBig(fileRecord.size); + + fileRecord.fileExists = true; + await this.fileRecordRepo.save(fileRecord); if (!useStreamToAntivirus || !fileRecord.isPreviewPossible()) { From 59972ce4530a4fb8ec42a00d9c70a30ccb32d544 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 2 Jan 2024 15:47:53 +0100 Subject: [PATCH 02/30] Add fileExists to response --- .../files-storage/controller/dto/file-storage.response.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts index dc595d62d9a..401096d43f9 100644 --- a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts +++ b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts @@ -14,6 +14,7 @@ export class FileRecordResponse { this.creatorId = fileRecord.creatorId; this.mimeType = fileRecord.mimeType; this.parentType = fileRecord.parentType; + this.fileExists = fileRecord.fileExists; this.deletedSince = fileRecord.deletedSince; this.previewStatus = fileRecord.getPreviewStatus(); } @@ -46,6 +47,9 @@ export class FileRecordResponse { @ApiProperty({ enum: FileRecordParentType, enumName: 'FileRecordParentType' }) parentType: FileRecordParentType; + @ApiProperty() + fileExists: boolean; + @ApiProperty({ enum: PreviewStatus, enumName: 'PreviewStatus' }) previewStatus: PreviewStatus; From 4548b8f2814b3805b1d40b6817f3b4486af5feab Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Thu, 4 Jan 2024 15:08:49 +0100 Subject: [PATCH 03/30] Add isUploading --- .../controller/dto/file-storage.response.ts | 4 ++-- .../modules/files-storage/entity/filerecord.entity.ts | 11 ++++++++--- .../src/modules/files-storage/helper/file-record.ts | 1 + .../files-storage/service/files-storage.service.ts | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts index 401096d43f9..4e99ba916ee 100644 --- a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts +++ b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts @@ -14,7 +14,7 @@ export class FileRecordResponse { this.creatorId = fileRecord.creatorId; this.mimeType = fileRecord.mimeType; this.parentType = fileRecord.parentType; - this.fileExists = fileRecord.fileExists; + this.isUploading = fileRecord.isUploading; this.deletedSince = fileRecord.deletedSince; this.previewStatus = fileRecord.getPreviewStatus(); } @@ -48,7 +48,7 @@ export class FileRecordResponse { parentType: FileRecordParentType; @ApiProperty() - fileExists: boolean; + isUploading?: boolean; @ApiProperty({ enum: PreviewStatus, enumName: 'PreviewStatus' }) previewStatus: PreviewStatus; diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts index 85e0a3ca401..d9354ef454f 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts @@ -80,6 +80,7 @@ export interface FileRecordProperties { schoolId: EntityId; deletedSince?: Date; isCopyFrom?: EntityId; + isUploading?: boolean; } interface ParentInfo { @@ -120,8 +121,8 @@ export class FileRecord extends BaseEntityWithTimestamps { @Enum() parentType: FileRecordParentType; - @Property() - fileExists: boolean; + @Property({ nullable: true }) + isUploading?: boolean; @Index() @Property({ fieldName: 'parent' }) @@ -164,7 +165,7 @@ export class FileRecord extends BaseEntityWithTimestamps { this.name = props.name; this.mimeType = props.mimeType; this.parentType = props.parentType; - this.fileExists = false; + this.isUploading = props.isUploading; this._parentId = new ObjectId(props.parentId); if (props.creatorId !== undefined) { this._creatorId = new ObjectId(props.creatorId); @@ -315,4 +316,8 @@ export class FileRecord extends BaseEntityWithTimestamps { public removeCreatorId(): void { this.creatorId = undefined; } + + public markAsLoaded(): void { + this.isUploading = undefined; + } } diff --git a/apps/server/src/modules/files-storage/helper/file-record.ts b/apps/server/src/modules/files-storage/helper/file-record.ts index ed291661735..80a73249441 100644 --- a/apps/server/src/modules/files-storage/helper/file-record.ts +++ b/apps/server/src/modules/files-storage/helper/file-record.ts @@ -36,6 +36,7 @@ export function createFileRecord( parentId: params.parentId, creatorId: userId, schoolId: params.schoolId, + isUploading: true, }); return entity; diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index aebf5d5596e..0b16acb6708 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -181,7 +181,7 @@ export class FilesStorageService { fileRecord.size = await fileSizePromise; this.throwErrorIfFileIsTooBig(fileRecord.size); - fileRecord.fileExists = true; + fileRecord.markAsLoaded(); await this.fileRecordRepo.save(fileRecord); From 2798ef10a2cad0ac07c1fb6901632d16177832c9 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Fri, 5 Jan 2024 09:23:21 +0100 Subject: [PATCH 04/30] Add tests --- .../entity/filerecord.entity.spec.ts | 36 ++++++++++++++++--- .../files-storage-upload.service.spec.ts | 34 ++++++++++++++++-- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts index 13c17ddcad8..de9328b9f67 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts @@ -32,7 +32,7 @@ describe('FileRecord Entity', () => { }; }); - it('should provide the target id as entity id', () => { + it('should provide target id', () => { const parentId = new ObjectId().toHexString(); const fileRecord = new FileRecord({ ...props, @@ -41,7 +41,7 @@ describe('FileRecord Entity', () => { expect(fileRecord.parentId).toEqual(parentId); }); - it('should provide the creator id as entity id', () => { + it('should provide creator id', () => { const creatorId = new ObjectId().toHexString(); const fileRecord = new FileRecord({ ...props, @@ -50,7 +50,7 @@ describe('FileRecord Entity', () => { expect(fileRecord.creatorId).toEqual(creatorId); }); - it('should provide the school id as entity id', () => { + it('should provide school id', () => { const schoolId = new ObjectId().toHexString(); const fileRecord = new FileRecord({ ...props, @@ -59,7 +59,7 @@ describe('FileRecord Entity', () => { expect(fileRecord.schoolId).toEqual(schoolId); }); - it('should provide the isCopyFrom as entity id', () => { + it('should provide isCopyFrom', () => { const isCopyFrom = new ObjectId().toHexString(); const fileRecord = new FileRecord({ ...props, @@ -67,6 +67,15 @@ describe('FileRecord Entity', () => { }); expect(fileRecord.isCopyFrom).toEqual(isCopyFrom); }); + + it('should provide isUploading', () => { + const isUploading = true; + const fileRecord = new FileRecord({ + ...props, + isUploading, + }); + expect(fileRecord.isUploading).toEqual(isUploading); + }); }); describe('when embedding the security status', () => { @@ -852,4 +861,23 @@ describe('FileRecord Entity', () => { }); }); }); + + describe('markAsLoaded is called', () => { + describe('WHEN isUploading is true', () => { + const setup = () => { + const isUploading = true; + const fileRecord = fileRecordFactory.build({ isUploading }); + + return { fileRecord, isUploading }; + }; + + it('should set it to undefined', () => { + const { fileRecord } = setup(); + expect(fileRecord.isUploading).toBe(true); + const result = fileRecord.markAsLoaded(); + + expect(result).toBe(undefined); + }); + }); + }); }); diff --git a/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts b/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts index 765de1077bd..6ee5001cd03 100644 --- a/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts @@ -1,10 +1,10 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; import { ObjectId } from '@mikro-orm/mongodb'; import { BadRequestException } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { readableStreamWithFileTypeFactory } from '@shared/testing/factory/readable-stream-with-file-type.factory'; import { LegacyLogger } from '@src/core/logger'; @@ -187,6 +187,35 @@ describe('FilesStorageService upload methods', () => { expect(getFileRecordsOfParentSpy).toHaveBeenCalledWith(params.parentId); }); + it('should call fileRecordRepo.save in first call with isUploading: true', async () => { + const { params, file, userId } = setup(); + + // haveBeenCalledWith can't be use here because fileRecord is a reference and + // it will always compare the final state of the object + let param: FileRecord | undefined; + + fileRecordRepo.save.mockReset(); + fileRecordRepo.save.mockImplementationOnce(async (fr) => { + if (fr instanceof FileRecord && !fr._id) { + fr._id = new ObjectId(); + } + + param = JSON.parse(JSON.stringify(fr)) as FileRecord; + + return Promise.resolve(); + }); + + fileRecordRepo.save.mockImplementationOnce(async (fr) => { + if (fr instanceof FileRecord && !fr._id) { + fr._id = new ObjectId(); + } + }); + + await service.uploadFile(userId, params, file); + + expect(param).toMatchObject({ isUploading: true }); + }); + it('should call fileRecordRepo.save twice with correct params', async () => { const { params, file, fileSize, userId, readableStreamWithFileType, expectedFileRecord } = setup(); @@ -201,6 +230,7 @@ describe('FilesStorageService upload methods', () => { size: fileSize, createdAt: expect.any(Date), updatedAt: expect.any(Date), + isUploading: undefined, }) ); }); From 5eec0311c26901ff32be5295fa8e833cf9382673 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Fri, 5 Jan 2024 11:19:13 +0100 Subject: [PATCH 05/30] Add promise resolve to test mocks --- .../service/files-storage-upload.service.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts b/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts index 6ee5001cd03..63049e0c587 100644 --- a/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts @@ -157,6 +157,8 @@ describe('FilesStorageService upload methods', () => { if (fr instanceof FileRecord && !fr._id) { fr._id = new ObjectId(); } + + return Promise.resolve(); }); return { @@ -209,6 +211,8 @@ describe('FilesStorageService upload methods', () => { if (fr instanceof FileRecord && !fr._id) { fr._id = new ObjectId(); } + + return Promise.resolve(); }); await service.uploadFile(userId, params, file); From 304624e7d62b5db56b8066f0cadc68381634287e Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Fri, 5 Jan 2024 13:32:26 +0100 Subject: [PATCH 06/30] Use ApiPropertyOptional for isUploading prop --- .../files-storage/controller/dto/file-storage.response.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts index 4e99ba916ee..094589673a3 100644 --- a/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts +++ b/apps/server/src/modules/files-storage/controller/dto/file-storage.response.ts @@ -47,7 +47,7 @@ export class FileRecordResponse { @ApiProperty({ enum: FileRecordParentType, enumName: 'FileRecordParentType' }) parentType: FileRecordParentType; - @ApiProperty() + @ApiPropertyOptional() isUploading?: boolean; @ApiProperty({ enum: PreviewStatus, enumName: 'PreviewStatus' }) From 78175166be52838bee3b1478bf674a2f15247901 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 9 Jan 2024 13:57:29 +0100 Subject: [PATCH 07/30] Add apiheader for If-None-Match --- .../modules/files-storage/controller/files-storage.controller.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 7269336c44c..8fc7bf17c60 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -127,6 +127,7 @@ export class FilesStorageController { @ApiResponse({ status: 422, type: UnprocessableEntityException }) @ApiResponse({ status: 500, type: InternalServerErrorException }) @ApiHeader({ name: 'Range', required: false }) + @ApiHeader({ name: 'If-None-Match', required: false }) @Get('/preview/:fileRecordId/:fileName') @RequestTimeout(config().INCOMING_REQUEST_TIMEOUT) async downloadPreview( From aec41e4b1d60ad08136f2aa78d6f3d070bfc490f Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 9 Jan 2024 14:12:26 +0100 Subject: [PATCH 08/30] Add console logs --- .../controller/files-storage.controller.ts | 2 ++ .../service/files-storage.service.ts | 31 +++++++++++++++++++ .../files-storage/uc/files-storage.uc.ts | 2 ++ 3 files changed, 35 insertions(+) diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 8fc7bf17c60..6d2d58eb3e8 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -86,9 +86,11 @@ export class FilesStorageController { @CurrentUser() currentUser: ICurrentUser, @Req() req: Request ): Promise { + console.time('controller'); const fileRecord = await this.filesStorageUC.upload(currentUser.userId, params, req); const response = FileRecordMapper.mapToFileRecordResponse(fileRecord); + console.timeEnd('controller'); return response; } diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 0b16acb6708..0b99fcdf808 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -84,13 +84,27 @@ export class FilesStorageService { // upload public async uploadFile(userId: EntityId, params: FileRecordParams, file: FileDto): Promise { + console.log('createFileRecord start'); + console.time('createFileRecord'); const { fileRecord, stream } = await this.createFileRecord(file, params, userId); + console.timeEnd('createFileRecord'); + console.log('createFileRecord end'); + // MimeType Detection consumes part of the stream, so the restored stream is passed on file.data = stream; file.mimeType = fileRecord.mimeType; + + console.log('save start'); + console.time('save'); await this.fileRecordRepo.save(fileRecord); + console.timeEnd('save'); + console.log('save end'); + console.log('createFileInStorageAndRollbackOnError start'); + console.time('createFileInStorageAndRollbackOnError'); await this.createFileInStorageAndRollbackOnError(fileRecord, params, file); + console.timeEnd('createFileInStorageAndRollbackOnError'); + console.log('createFileInStorageAndRollbackOnError end'); return fileRecord; } @@ -100,8 +114,17 @@ export class FilesStorageService { params: FileRecordParams, userId: EntityId ): Promise<{ fileRecord: FileRecord; stream: Readable }> { + console.log('resolveFileName start'); + console.time('resolveFileName'); const fileName = await this.resolveFileName(file, params); + console.timeEnd('resolveFileName'); + console.log('resolveFileName end'); + + console.log('detectMimeType start'); + console.time('detectMimeType'); const { mimeType, stream } = await this.detectMimeType(file); + console.timeEnd('detectMimeType'); + console.log('detectMimeType end'); // Create fileRecord with 0 as initial file size, because it is overwritten later anyway. const fileRecord = createFileRecord(fileName, 0, mimeType, params, userId); @@ -167,10 +190,14 @@ export class FilesStorageService { if (useStreamToAntivirus && fileRecord.isPreviewPossible()) { const streamToAntivirus = file.data.pipe(new PassThrough()); + console.log('upload and virus check start'); + console.time('upload and virus check '); const [, antivirusClientResponse] = await Promise.all([ this.storageClient.create(filePath, file), this.antivirusService.checkStream(streamToAntivirus), ]); + console.log('upload and virus check end'); + console.timeEnd('upload and virus check '); const { status, reason } = FileRecordMapper.mapScanResultParamsToDto(antivirusClientResponse); fileRecord.updateSecurityCheckStatus(status, reason); } else { @@ -183,7 +210,11 @@ export class FilesStorageService { fileRecord.markAsLoaded(); + console.log('second save start'); + console.time('second save'); await this.fileRecordRepo.save(fileRecord); + console.log('second save end'); + console.timeEnd('second save'); if (!useStreamToAntivirus || !fileRecord.isPreviewPossible()) { await this.sendToAntivirus(fileRecord); diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index a9fae9db118..5f9eab8d427 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -70,6 +70,8 @@ export class FilesStorageUC { try { const record = await this.filesStorageService.uploadFile(userId, params, fileDto); + console.log('filerecord returned in uc', record); + resolve(record); } catch (error) { req.unpipe(bb); From 37ab24dbcf6ab6bf502268e19a9e2d1485a5ddec Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Wed, 10 Jan 2024 08:20:25 +0100 Subject: [PATCH 09/30] Revert "Add console logs" This reverts commit aec41e4b1d60ad08136f2aa78d6f3d070bfc490f. --- .../controller/files-storage.controller.ts | 2 -- .../service/files-storage.service.ts | 31 ------------------- .../files-storage/uc/files-storage.uc.ts | 2 -- 3 files changed, 35 deletions(-) diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 6d2d58eb3e8..8fc7bf17c60 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -86,11 +86,9 @@ export class FilesStorageController { @CurrentUser() currentUser: ICurrentUser, @Req() req: Request ): Promise { - console.time('controller'); const fileRecord = await this.filesStorageUC.upload(currentUser.userId, params, req); const response = FileRecordMapper.mapToFileRecordResponse(fileRecord); - console.timeEnd('controller'); return response; } diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 0b99fcdf808..0b16acb6708 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -84,27 +84,13 @@ export class FilesStorageService { // upload public async uploadFile(userId: EntityId, params: FileRecordParams, file: FileDto): Promise { - console.log('createFileRecord start'); - console.time('createFileRecord'); const { fileRecord, stream } = await this.createFileRecord(file, params, userId); - console.timeEnd('createFileRecord'); - console.log('createFileRecord end'); - // MimeType Detection consumes part of the stream, so the restored stream is passed on file.data = stream; file.mimeType = fileRecord.mimeType; - - console.log('save start'); - console.time('save'); await this.fileRecordRepo.save(fileRecord); - console.timeEnd('save'); - console.log('save end'); - console.log('createFileInStorageAndRollbackOnError start'); - console.time('createFileInStorageAndRollbackOnError'); await this.createFileInStorageAndRollbackOnError(fileRecord, params, file); - console.timeEnd('createFileInStorageAndRollbackOnError'); - console.log('createFileInStorageAndRollbackOnError end'); return fileRecord; } @@ -114,17 +100,8 @@ export class FilesStorageService { params: FileRecordParams, userId: EntityId ): Promise<{ fileRecord: FileRecord; stream: Readable }> { - console.log('resolveFileName start'); - console.time('resolveFileName'); const fileName = await this.resolveFileName(file, params); - console.timeEnd('resolveFileName'); - console.log('resolveFileName end'); - - console.log('detectMimeType start'); - console.time('detectMimeType'); const { mimeType, stream } = await this.detectMimeType(file); - console.timeEnd('detectMimeType'); - console.log('detectMimeType end'); // Create fileRecord with 0 as initial file size, because it is overwritten later anyway. const fileRecord = createFileRecord(fileName, 0, mimeType, params, userId); @@ -190,14 +167,10 @@ export class FilesStorageService { if (useStreamToAntivirus && fileRecord.isPreviewPossible()) { const streamToAntivirus = file.data.pipe(new PassThrough()); - console.log('upload and virus check start'); - console.time('upload and virus check '); const [, antivirusClientResponse] = await Promise.all([ this.storageClient.create(filePath, file), this.antivirusService.checkStream(streamToAntivirus), ]); - console.log('upload and virus check end'); - console.timeEnd('upload and virus check '); const { status, reason } = FileRecordMapper.mapScanResultParamsToDto(antivirusClientResponse); fileRecord.updateSecurityCheckStatus(status, reason); } else { @@ -210,11 +183,7 @@ export class FilesStorageService { fileRecord.markAsLoaded(); - console.log('second save start'); - console.time('second save'); await this.fileRecordRepo.save(fileRecord); - console.log('second save end'); - console.timeEnd('second save'); if (!useStreamToAntivirus || !fileRecord.isPreviewPossible()) { await this.sendToAntivirus(fileRecord); diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 5f9eab8d427..a9fae9db118 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -70,8 +70,6 @@ export class FilesStorageUC { try { const record = await this.filesStorageService.uploadFile(userId, params, fileDto); - console.log('filerecord returned in uc', record); - resolve(record); } catch (error) { req.unpipe(bb); From 4e5697a57052655ce5eb48042d64d57d4121f877 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Wed, 10 Jan 2024 08:24:14 +0100 Subject: [PATCH 10/30] Turn on mikroorm debug --- apps/server/src/modules/files-storage/files-storage.module.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index 0b049951767..ddd1188038b 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -49,6 +49,7 @@ const defaultMikroOrmOptions: MikroOrmModuleSyncOptions = { password: DB_PASSWORD, user: DB_USERNAME, entities: [...ALL_ENTITIES, FileRecord, FileRecordSecurityCheck], + debug: true, // debug: true, // use it for locally debugging of querys }), From 1489825db88b9fb69a49937c5bf7233273801e29 Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Wed, 10 Jan 2024 10:39:18 +0100 Subject: [PATCH 11/30] BC-5714 - add req context --- .../files-storage/controller/files-storage.controller.ts | 8 +++++++- package.json | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 8fc7bf17c60..281116a7ff4 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -1,3 +1,4 @@ +import { MikroORM, UseRequestContext } from '@mikro-orm/core'; import { Authenticate, CurrentUser, ICurrentUser } from '@modules/authentication'; import { BadRequestException, @@ -51,7 +52,11 @@ import { @Authenticate('jwt') @Controller('file') export class FilesStorageController { - constructor(private readonly filesStorageUC: FilesStorageUC) {} + constructor( + private readonly filesStorageUC: FilesStorageUC, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + private readonly orm: MikroORM // don't remove it, we need it for @UseRequestContext + ) {} @ApiOperation({ summary: 'Upload file from url' }) @ApiResponse({ status: 201, type: FileRecordResponse }) @@ -79,6 +84,7 @@ export class FilesStorageController { @ApiResponse({ status: 403, type: ForbiddenException }) @ApiResponse({ status: 500, type: InternalServerErrorException }) @ApiConsumes('multipart/form-data') + @UseRequestContext() @Post('/upload/:schoolId/:parentType/:parentId') async upload( @Body() _: FileParams, diff --git a/package.json b/package.json index c4d7d4026e5..9c63e4ab70c 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", + "nest:start:files-storage:debug::qaonce": "nest start files-storage --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", From 49aab2292cf71d604c3770a21446eb4331fd6f2b Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Thu, 11 Jan 2024 13:00:57 +0100 Subject: [PATCH 12/30] Rename markAsLoaded to markAsUploaded --- .../src/modules/files-storage/entity/filerecord.entity.spec.ts | 2 +- .../src/modules/files-storage/entity/filerecord.entity.ts | 2 +- .../src/modules/files-storage/service/files-storage.service.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts index de9328b9f67..4caac8fafc6 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts @@ -874,7 +874,7 @@ describe('FileRecord Entity', () => { it('should set it to undefined', () => { const { fileRecord } = setup(); expect(fileRecord.isUploading).toBe(true); - const result = fileRecord.markAsLoaded(); + const result = fileRecord.markAsUploaded(); expect(result).toBe(undefined); }); diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts index d9354ef454f..ac615d15f65 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts @@ -317,7 +317,7 @@ export class FileRecord extends BaseEntityWithTimestamps { this.creatorId = undefined; } - public markAsLoaded(): void { + public markAsUploaded(): void { this.isUploading = undefined; } } diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 0b16acb6708..2898258b814 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -181,7 +181,7 @@ export class FilesStorageService { fileRecord.size = await fileSizePromise; this.throwErrorIfFileIsTooBig(fileRecord.size); - fileRecord.markAsLoaded(); + fileRecord.markAsUploaded(); await this.fileRecordRepo.save(fileRecord); From c01c009d1bd12da26f8adc2088bb5cd579c477b6 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Thu, 11 Jan 2024 13:03:00 +0100 Subject: [PATCH 13/30] Remove mikororm debug mode --- apps/server/src/modules/files-storage/files-storage.module.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index ddd1188038b..0b049951767 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -49,7 +49,6 @@ const defaultMikroOrmOptions: MikroOrmModuleSyncOptions = { password: DB_PASSWORD, user: DB_USERNAME, entities: [...ALL_ENTITIES, FileRecord, FileRecordSecurityCheck], - debug: true, // debug: true, // use it for locally debugging of querys }), From b5cc09943d4180806a7e898f336dded597a13ff8 Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Thu, 11 Jan 2024 13:51:53 +0100 Subject: [PATCH 14/30] BC-5714 - add ingress config --- .../schulcloud-server-core/templates/api-files-ingress.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 index 3eabd5d2659..fe377635efd 100644 --- a/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 @@ -14,6 +14,7 @@ metadata: nginx.ingress.kubernetes.io/http2-max-header-size: 96k nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k nginx.ingress.kubernetes.io/proxy-buffer-size: 96k + nginx.ingress.kubernetes.io/proxy-request-buffering: "off" {% if CLUSTER_ISSUER is defined %} cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }} {% endif %} From 2bcff0ce5c1eb2f6dd63c8f1eea7046106d0de28 Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Thu, 11 Jan 2024 14:12:47 +0100 Subject: [PATCH 15/30] Revert "BC-5714 - add req context" This reverts commit 1489825db88b9fb69a49937c5bf7233273801e29. --- .../files-storage/controller/files-storage.controller.ts | 8 +------- package.json | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 281116a7ff4..8fc7bf17c60 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -1,4 +1,3 @@ -import { MikroORM, UseRequestContext } from '@mikro-orm/core'; import { Authenticate, CurrentUser, ICurrentUser } from '@modules/authentication'; import { BadRequestException, @@ -52,11 +51,7 @@ import { @Authenticate('jwt') @Controller('file') export class FilesStorageController { - constructor( - private readonly filesStorageUC: FilesStorageUC, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - private readonly orm: MikroORM // don't remove it, we need it for @UseRequestContext - ) {} + constructor(private readonly filesStorageUC: FilesStorageUC) {} @ApiOperation({ summary: 'Upload file from url' }) @ApiResponse({ status: 201, type: FileRecordResponse }) @@ -84,7 +79,6 @@ export class FilesStorageController { @ApiResponse({ status: 403, type: ForbiddenException }) @ApiResponse({ status: 500, type: InternalServerErrorException }) @ApiConsumes('multipart/form-data') - @UseRequestContext() @Post('/upload/:schoolId/:parentType/:parentId') async upload( @Body() _: FileParams, diff --git a/package.json b/package.json index 86c976caf9b..656edf9a2d4 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,6 @@ "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", - "nest:start:files-storage:debug::qaonce": "nest start files-storage --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", From fdc2e71f3cdce40ed739b2759f3e43a804c9c1c4 Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Thu, 11 Jan 2024 16:39:21 +0100 Subject: [PATCH 16/30] BC-5714 - remove await --- apps/server/src/modules/files-storage/uc/files-storage.uc.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index a9fae9db118..0daab02355e 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -69,8 +69,7 @@ export class FilesStorageUC { const fileDto = FileDtoBuilder.buildFromRequest(info, file); try { - const record = await this.filesStorageService.uploadFile(userId, params, fileDto); - resolve(record); + resolve(this.filesStorageService.uploadFile(userId, params, fileDto)); } catch (error) { req.unpipe(bb); reject(error); From 970f94a8b6f39f0b743db1bf5375379be256465c Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Thu, 11 Jan 2024 18:15:08 +0100 Subject: [PATCH 17/30] add error logging --- .../modules/files-storage/service/files-storage.service.ts | 1 + apps/server/src/modules/files-storage/uc/files-storage.uc.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 2898258b814..2de2cd8a1d4 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -189,6 +189,7 @@ export class FilesStorageService { await this.sendToAntivirus(fileRecord); } } catch (error) { + this.logger.error(`create file failed for fileRecordId ${fileRecord.id}`, error); await this.storageClient.delete([filePath]); await this.fileRecordRepo.delete(fileRecord); throw error; diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 0daab02355e..7355aab1a15 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -69,8 +69,10 @@ export class FilesStorageUC { const fileDto = FileDtoBuilder.buildFromRequest(info, file); try { - resolve(this.filesStorageService.uploadFile(userId, params, fileDto)); + const result = await this.filesStorageService.uploadFile(userId, params, fileDto); + resolve(result); } catch (error) { + this.logger.error({ message: 'could not upload file', error: error as Error }); req.unpipe(bb); reject(error); } From ba492b428319643bab0b1e4233825e118177c6dc Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Fri, 12 Jan 2024 11:24:05 +0100 Subject: [PATCH 18/30] try request scope --- apps/server/src/modules/files-storage/uc/files-storage.uc.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 7355aab1a15..fa0701cf853 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -1,7 +1,7 @@ import { AuthorizationContext } from '@modules/authorization'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; -import { Injectable, NotFoundException } from '@nestjs/common'; +import { Injectable, NotFoundException, Scope } from '@nestjs/common'; import { Counted, EntityId } from '@shared/domain/types'; import { LegacyLogger } from '@src/core/logger'; import { AxiosRequestConfig, AxiosResponse } from 'axios'; @@ -29,7 +29,7 @@ import { FileDtoBuilder, FilesStorageMapper } from '../mapper'; import { FilesStorageService } from '../service/files-storage.service'; import { PreviewService } from '../service/preview.service'; -@Injectable() +@Injectable({ scope: Scope.REQUEST }) export class FilesStorageUC { constructor( private logger: LegacyLogger, From e9fd73f9ad1f1bacf91cc956557c5975d2f0609c Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Fri, 12 Jan 2024 11:50:25 +0100 Subject: [PATCH 19/30] Add scope to repo --- apps/server/src/modules/files-storage/repo/filerecord.repo.ts | 4 ++-- apps/server/src/modules/files-storage/uc/files-storage.uc.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/server/src/modules/files-storage/repo/filerecord.repo.ts b/apps/server/src/modules/files-storage/repo/filerecord.repo.ts index 8333c5b7021..404462c5add 100644 --- a/apps/server/src/modules/files-storage/repo/filerecord.repo.ts +++ b/apps/server/src/modules/files-storage/repo/filerecord.repo.ts @@ -1,11 +1,11 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Scope } from '@nestjs/common'; import { IFindOptions, SortOrder } from '@shared/domain/interface'; import { Counted, EntityId } from '@shared/domain/types'; import { BaseRepo } from '@shared/repo'; import { FileRecord } from '../entity'; import { FileRecordScope } from './filerecord-scope'; -@Injectable() +@Injectable({ scope: Scope.REQUEST }) export class FileRecordRepo extends BaseRepo { get entityName() { return FileRecord; diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index fa0701cf853..7355aab1a15 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -1,7 +1,7 @@ import { AuthorizationContext } from '@modules/authorization'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; -import { Injectable, NotFoundException, Scope } from '@nestjs/common'; +import { Injectable, NotFoundException } from '@nestjs/common'; import { Counted, EntityId } from '@shared/domain/types'; import { LegacyLogger } from '@src/core/logger'; import { AxiosRequestConfig, AxiosResponse } from 'axios'; @@ -29,7 +29,7 @@ import { FileDtoBuilder, FilesStorageMapper } from '../mapper'; import { FilesStorageService } from '../service/files-storage.service'; import { PreviewService } from '../service/preview.service'; -@Injectable({ scope: Scope.REQUEST }) +@Injectable() export class FilesStorageUC { constructor( private logger: LegacyLogger, From c6027eaa835f650984033f3f152484202609ddba Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Fri, 12 Jan 2024 12:27:28 +0100 Subject: [PATCH 20/30] add scope to MikroOrm Module --- apps/server/src/modules/files-storage/files-storage.module.ts | 4 +++- apps/server/src/modules/files-storage/repo/filerecord.repo.ts | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index 0b049951767..e6f4efb5ac4 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -5,7 +5,7 @@ import { RabbitMQWrapperModule } from '@infra/rabbitmq'; import { S3ClientModule } from '@infra/s3-client'; import { Dictionary, IPrimaryKey } from '@mikro-orm/core'; import { MikroOrmModule, MikroOrmModuleSyncOptions } from '@mikro-orm/nestjs'; -import { Module, NotFoundException } from '@nestjs/common'; +import { Module, NotFoundException, Scope } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { ALL_ENTITIES } from '@shared/domain/entity'; import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config'; @@ -49,6 +49,8 @@ const defaultMikroOrmOptions: MikroOrmModuleSyncOptions = { password: DB_PASSWORD, user: DB_USERNAME, entities: [...ALL_ENTITIES, FileRecord, FileRecordSecurityCheck], + registerRequestContext: false, // disable the automatic context + scope: Scope.REQUEST, // use nestjs request scopes instead // debug: true, // use it for locally debugging of querys }), diff --git a/apps/server/src/modules/files-storage/repo/filerecord.repo.ts b/apps/server/src/modules/files-storage/repo/filerecord.repo.ts index 404462c5add..8333c5b7021 100644 --- a/apps/server/src/modules/files-storage/repo/filerecord.repo.ts +++ b/apps/server/src/modules/files-storage/repo/filerecord.repo.ts @@ -1,11 +1,11 @@ -import { Injectable, Scope } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { IFindOptions, SortOrder } from '@shared/domain/interface'; import { Counted, EntityId } from '@shared/domain/types'; import { BaseRepo } from '@shared/repo'; import { FileRecord } from '../entity'; import { FileRecordScope } from './filerecord-scope'; -@Injectable({ scope: Scope.REQUEST }) +@Injectable() export class FileRecordRepo extends BaseRepo { get entityName() { return FileRecord; From 336ba02b92ab223091f0d21a26aee581a899c1a7 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Fri, 12 Jan 2024 13:39:50 +0100 Subject: [PATCH 21/30] Create new async request context in uc --- .../src/modules/files-storage/files-storage.module.ts | 4 +--- .../src/modules/files-storage/uc/files-storage.uc.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index e6f4efb5ac4..0b049951767 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -5,7 +5,7 @@ import { RabbitMQWrapperModule } from '@infra/rabbitmq'; import { S3ClientModule } from '@infra/s3-client'; import { Dictionary, IPrimaryKey } from '@mikro-orm/core'; import { MikroOrmModule, MikroOrmModuleSyncOptions } from '@mikro-orm/nestjs'; -import { Module, NotFoundException, Scope } from '@nestjs/common'; +import { Module, NotFoundException } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { ALL_ENTITIES } from '@shared/domain/entity'; import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config'; @@ -49,8 +49,6 @@ const defaultMikroOrmOptions: MikroOrmModuleSyncOptions = { password: DB_PASSWORD, user: DB_USERNAME, entities: [...ALL_ENTITIES, FileRecord, FileRecordSecurityCheck], - registerRequestContext: false, // disable the automatic context - scope: Scope.REQUEST, // use nestjs request scopes instead // debug: true, // use it for locally debugging of querys }), diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 7355aab1a15..7ff269380bb 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -1,3 +1,4 @@ +import { EntityManager, RequestContext } from '@mikro-orm/core'; import { AuthorizationContext } from '@modules/authorization'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; @@ -36,7 +37,8 @@ export class FilesStorageUC { private readonly authorizationReferenceService: AuthorizationReferenceService, private readonly httpService: HttpService, private readonly filesStorageService: FilesStorageService, - private readonly previewService: PreviewService + private readonly previewService: PreviewService, + private readonly em: EntityManager ) { this.logger.setContext(FilesStorageUC.name); } @@ -69,7 +71,11 @@ export class FilesStorageUC { const fileDto = FileDtoBuilder.buildFromRequest(info, file); try { - const result = await this.filesStorageService.uploadFile(userId, params, fileDto); + const result = await RequestContext.createAsync(this.em, async () => { + const record = await this.filesStorageService.uploadFile(userId, params, fileDto); + + return record; + }); resolve(result); } catch (error) { this.logger.error({ message: 'could not upload file', error: error as Error }); From e1e3a1c5b734bac1c6b98495d2ec60c555cb8aae Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 11:38:04 +0100 Subject: [PATCH 22/30] Remove await from busboy on file --- .../files-storage/uc/files-storage.uc.ts | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 7ff269380bb..4b3ed1cd7ff 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -65,23 +65,30 @@ export class FilesStorageUC { private async uploadFileWithBusboy(userId: EntityId, params: FileRecordParams, req: Request): Promise { const promise = new Promise((resolve, reject) => { const bb = busboy({ headers: req.headers, defParamCharset: 'utf8' }); + let promise2: Promise; - // eslint-disable-next-line @typescript-eslint/no-misused-promises - bb.on('file', async (_name, file, info) => { + bb.on('file', (_name, file, info) => { const fileDto = FileDtoBuilder.buildFromRequest(info, file); - try { - const result = await RequestContext.createAsync(this.em, async () => { - const record = await this.filesStorageService.uploadFile(userId, params, fileDto); + promise2 = RequestContext.createAsync(this.em, () => { + const record = this.filesStorageService.uploadFile(userId, params, fileDto); - return record; + return record; + }); + }); + + bb.on('finish', () => { + promise2 + .then((result) => { + resolve(result); + console.log('fileclose', result); + return result; + }) + .catch((error) => { + this.logger.error({ message: 'could not upload file', error: error as Error }); + req.unpipe(bb); + reject(error); }); - resolve(result); - } catch (error) { - this.logger.error({ message: 'could not upload file', error: error as Error }); - req.unpipe(bb); - reject(error); - } }); req.pipe(bb); From c4eb4fe2a590f675e84ac59aa747d7dc833418e3 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 11:39:14 +0100 Subject: [PATCH 23/30] Only starte file storage api server on debug command --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 656edf9a2d4..7dbc59cfe9b 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "nest:start:management:prod": "node dist/apps/server/apps/management.app", "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", - "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", + "nest:start:files-storage:debug": "nest start files-storage --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", From d35829f94eab166500fffb495cd0259530ed4b2c Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 13:40:43 +0100 Subject: [PATCH 24/30] Clean up code --- .../modules/files-storage/uc/files-storage.uc.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 4b3ed1cd7ff..ca3d5f28ac1 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -65,12 +65,12 @@ export class FilesStorageUC { private async uploadFileWithBusboy(userId: EntityId, params: FileRecordParams, req: Request): Promise { const promise = new Promise((resolve, reject) => { const bb = busboy({ headers: req.headers, defParamCharset: 'utf8' }); - let promise2: Promise; + let fileRecordPromise: Promise; bb.on('file', (_name, file, info) => { const fileDto = FileDtoBuilder.buildFromRequest(info, file); - promise2 = RequestContext.createAsync(this.em, () => { + fileRecordPromise = RequestContext.createAsync(this.em, () => { const record = this.filesStorageService.uploadFile(userId, params, fileDto); return record; @@ -78,14 +78,9 @@ export class FilesStorageUC { }); bb.on('finish', () => { - promise2 - .then((result) => { - resolve(result); - console.log('fileclose', result); - return result; - }) + fileRecordPromise + .then((result) => resolve(result)) .catch((error) => { - this.logger.error({ message: 'could not upload file', error: error as Error }); req.unpipe(bb); reject(error); }); From aeb1639b6c5983de1be673d10c3019c50cf4d7b8 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 13:51:32 +0100 Subject: [PATCH 25/30] Rollback package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7dbc59cfe9b..656edf9a2d4 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "nest:start:management:prod": "node dist/apps/server/apps/management.app", "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", - "nest:start:files-storage:debug": "nest start files-storage --debug --watch", + "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", From f71d61699a1fb3184f3454563ffe710a68f7b594 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 13:51:41 +0100 Subject: [PATCH 26/30] Remove error log --- .../src/modules/files-storage/service/files-storage.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 2de2cd8a1d4..2898258b814 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -189,7 +189,6 @@ export class FilesStorageService { await this.sendToAntivirus(fileRecord); } } catch (error) { - this.logger.error(`create file failed for fileRecordId ${fileRecord.id}`, error); await this.storageClient.delete([filePath]); await this.fileRecordRepo.delete(fileRecord); throw error; From e1f4916a1d6ba5971b4352b3760623b7a66912e9 Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Mon, 15 Jan 2024 14:40:10 +0100 Subject: [PATCH 27/30] Revert "Clean up code" This reverts commit d35829f94eab166500fffb495cd0259530ed4b2c. --- .../modules/files-storage/uc/files-storage.uc.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index ca3d5f28ac1..4b3ed1cd7ff 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -65,12 +65,12 @@ export class FilesStorageUC { private async uploadFileWithBusboy(userId: EntityId, params: FileRecordParams, req: Request): Promise { const promise = new Promise((resolve, reject) => { const bb = busboy({ headers: req.headers, defParamCharset: 'utf8' }); - let fileRecordPromise: Promise; + let promise2: Promise; bb.on('file', (_name, file, info) => { const fileDto = FileDtoBuilder.buildFromRequest(info, file); - fileRecordPromise = RequestContext.createAsync(this.em, () => { + promise2 = RequestContext.createAsync(this.em, () => { const record = this.filesStorageService.uploadFile(userId, params, fileDto); return record; @@ -78,9 +78,14 @@ export class FilesStorageUC { }); bb.on('finish', () => { - fileRecordPromise - .then((result) => resolve(result)) + promise2 + .then((result) => { + resolve(result); + console.log('fileclose', result); + return result; + }) .catch((error) => { + this.logger.error({ message: 'could not upload file', error: error as Error }); req.unpipe(bb); reject(error); }); From 584de6fd4195435b78236b3cb84235506067e85f Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 16 Jan 2024 09:00:15 +0100 Subject: [PATCH 28/30] Adjust file storage uc tests --- .../uc/files-storage-upload.uc.spec.ts | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage-upload.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-upload.uc.spec.ts index 5b9e9b410f3..336745140af 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-upload.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-upload.uc.spec.ts @@ -1,6 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { AntivirusService } from '@infra/antivirus'; import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; import { Action } from '@modules/authorization'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; @@ -110,6 +111,10 @@ describe('FilesStorageUC upload methods', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); @@ -285,13 +290,21 @@ describe('FilesStorageUC upload methods', () => { mimeType: fileRecord.mimeType, }; + let resolveUploadFile: (value: FileRecord | PromiseLike) => void; + const fileRecordPromise = new Promise((resolve) => { + resolveUploadFile = resolve; + }); + filesStorageService.uploadFile.mockImplementationOnce(() => fileRecordPromise); + request.pipe.mockImplementation((requestStream) => { requestStream.emit('file', 'file', readable, fileInfo); + + requestStream.emit('finish'); + resolveUploadFile(fileRecord); + return requestStream; }); - filesStorageService.uploadFile.mockResolvedValueOnce(fileRecord); - return { params, userId, request, fileRecord, readable, fileInfo }; }; @@ -314,7 +327,6 @@ describe('FilesStorageUC upload methods', () => { const file = FileDtoBuilder.buildFromRequest(fileInfo, readable); await filesStorageUC.upload(userId, params, request); - expect(filesStorageService.uploadFile).toHaveBeenCalledWith(userId, params, file); }); @@ -333,9 +345,16 @@ describe('FilesStorageUC upload methods', () => { const fileRecord = fileRecords[0]; const request = createRequest(); const readable = Readable.from('abc'); + const error = new Error('test'); const size = request.headers['content-length']; + let rejectUploadFile: (value: Error) => void; + const fileRecordPromise = new Promise((resolve, reject) => { + rejectUploadFile = reject; + }); + filesStorageService.uploadFile.mockImplementationOnce(() => fileRecordPromise); + request.get.mockReturnValue(size); request.pipe.mockImplementation((requestStream) => { requestStream.emit('file', 'file', readable, { @@ -344,12 +363,12 @@ describe('FilesStorageUC upload methods', () => { mimeType: fileRecord.mimeType, }); + requestStream.emit('finish'); + rejectUploadFile(error); + return requestStream; }); - const error = new Error('test'); - filesStorageService.uploadFile.mockRejectedValueOnce(error); - return { params, userId, request, error }; }; From 04a398a4da5aa50f01bd6eefbe4eb452748c7cad Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 16 Jan 2024 10:45:59 +0100 Subject: [PATCH 29/30] Add entity manager to uc tests --- .../files-storage/uc/files-storage-copy.uc.spec.ts | 5 +++++ .../files-storage/uc/files-storage-delete.uc.spec.ts | 5 +++++ .../uc/files-storage-download-preview.uc.spec.ts | 11 ++++++++--- .../uc/files-storage-download.uc.spec.ts | 11 ++++++++--- .../files-storage/uc/files-storage-get.uc.spec.ts | 11 ++++++++--- .../files-storage/uc/files-storage-restore.uc.spec.ts | 11 ++++++++--- .../files-storage/uc/files-storage-update.uc.spec.ts | 11 ++++++++--- 7 files changed, 50 insertions(+), 15 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage-copy.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-copy.uc.spec.ts index 0f72cd326e6..9dda2df44da 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-copy.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-copy.uc.spec.ts @@ -1,6 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { AntivirusService } from '@infra/antivirus'; import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; import { Action } from '@modules/authorization'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; @@ -110,6 +111,10 @@ describe('FilesStorageUC', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts index b4ea26c20b1..8606fcda798 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts @@ -1,6 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { AntivirusService } from '@infra/antivirus'; import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; @@ -93,6 +94,10 @@ describe('FilesStorageUC delete methods', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts index d34f004d73b..2e2f89ee224 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts @@ -1,13 +1,14 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { ForbiddenException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { SingleFileParams } from '../controller/dto'; import { FileRecord } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -79,6 +80,10 @@ describe('FilesStorageUC', () => { provide: HttpService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-download.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-download.uc.spec.ts index bcb5b2ec827..3654fdabe8e 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-download.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-download.uc.spec.ts @@ -1,13 +1,14 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { ForbiddenException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { SingleFileParams } from '../controller/dto'; import { FileRecord } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -70,6 +71,10 @@ describe('FilesStorageUC', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-get.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-get.uc.spec.ts index a4f3e0f8b2f..e1e94c8d4ad 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-get.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-get.uc.spec.ts @@ -1,12 +1,13 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { FileRecordParams } from '../controller/dto'; import { FileRecord, FileRecordParentType } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -73,6 +74,10 @@ describe('FilesStorageUC', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-restore.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-restore.uc.spec.ts index dc811c566a4..6ee67c75cb6 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-restore.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-restore.uc.spec.ts @@ -1,13 +1,14 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { ForbiddenException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { FileRecordParams, SingleFileParams } from '../controller/dto'; import { FileRecord, FileRecordParentType } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -88,6 +89,10 @@ describe('FilesStorageUC', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); diff --git a/apps/server/src/modules/files-storage/uc/files-storage-update.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-update.uc.spec.ts index 5b126c8ea2a..f07914a8915 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-update.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-update.uc.spec.ts @@ -1,12 +1,13 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { S3ClientAdapter } from '@infra/s3-client'; +import { EntityManager } from '@mikro-orm/core'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { Test, TestingModule } from '@nestjs/testing'; -import { AntivirusService } from '@infra/antivirus'; -import { S3ClientAdapter } from '@infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { RenameFileParams, ScanResultParams, SingleFileParams } from '../controller/dto'; import { FileRecord } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -67,6 +68,10 @@ describe('FilesStorageUC', () => { provide: PreviewService, useValue: createMock(), }, + { + provide: EntityManager, + useValue: createMock(), + }, ], }).compile(); From 339dfe17deedeb58e30b1c60dafb1546512bd4af Mon Sep 17 00:00:00 2001 From: Max Bischof Date: Tue, 16 Jan 2024 13:13:17 +0100 Subject: [PATCH 30/30] Rename promise2 to fileRecordPromise --- .../modules/files-storage/uc/files-storage.uc.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 4b3ed1cd7ff..ca3d5f28ac1 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -65,12 +65,12 @@ export class FilesStorageUC { private async uploadFileWithBusboy(userId: EntityId, params: FileRecordParams, req: Request): Promise { const promise = new Promise((resolve, reject) => { const bb = busboy({ headers: req.headers, defParamCharset: 'utf8' }); - let promise2: Promise; + let fileRecordPromise: Promise; bb.on('file', (_name, file, info) => { const fileDto = FileDtoBuilder.buildFromRequest(info, file); - promise2 = RequestContext.createAsync(this.em, () => { + fileRecordPromise = RequestContext.createAsync(this.em, () => { const record = this.filesStorageService.uploadFile(userId, params, fileDto); return record; @@ -78,14 +78,9 @@ export class FilesStorageUC { }); bb.on('finish', () => { - promise2 - .then((result) => { - resolve(result); - console.log('fileclose', result); - return result; - }) + fileRecordPromise + .then((result) => resolve(result)) .catch((error) => { - this.logger.error({ message: 'could not upload file', error: error as Error }); req.unpipe(bb); reject(error); });