Merge branch 'main' into BC-5170-no-preview-for-larger-files

apps/server/src/modules/files-storage/controller/api-test/files-storage-copy-files.api.spec.ts

@@ -4,7 +4,7 @@ import { ExecutionContext, INestApplication } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ApiValidationError } from '@shared/common';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import {
 	cleanupCollections,

apps/server/src/modules/files-storage/controller/api-test/files-storage-delete-files.api.spec.ts

@@ -4,7 +4,7 @@ import { ExecutionContext, INestApplication } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ApiValidationError } from '@shared/common';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import {
 	cleanupCollections,

apps/server/src/modules/files-storage/controller/api-test/files-storage-download-upload.api.spec.ts

@@ -4,7 +4,7 @@ import { ExecutionContext, INestApplication } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ApiValidationError } from '@shared/common';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { cleanupCollections, mapUserToCurrentUser, roleFactory, schoolFactory, userFactory } from '@shared/testing';
 import { ICurrentUser } from '@src/modules/authentication';

apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts

@@ -4,7 +4,7 @@ import { ExecutionContext, INestApplication, NotFoundException, StreamableFile }
 import { Test, TestingModule } from '@nestjs/testing';
 import { ApiValidationError } from '@shared/common';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { PreviewProducer } from '@shared/infra/preview-generator';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { cleanupCollections, mapUserToCurrentUser, roleFactory, schoolFactory, userFactory } from '@shared/testing';

apps/server/src/modules/files-storage/controller/api-test/files-storage-restore-files.api.spec.ts

@@ -4,7 +4,7 @@ import { ExecutionContext, INestApplication } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ApiValidationError } from '@shared/common';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import {
 	cleanupCollections,

apps/server/src/modules/files-storage/controller/dto/file-storage.params.ts

@@ -1,6 +1,7 @@
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 import { StringToBoolean } from '@shared/controller';
 import { EntityId } from '@shared/domain';
+import { ScanResult } from '@shared/infra/antivirus';
 import { Allow, IsBoolean, IsEnum, IsMongoId, IsNotEmpty, IsOptional, IsString, ValidateNested } from 'class-validator';
 import { FileRecordParentType } from '../../entity';
 import { PreviewOutputMimeTypes, PreviewWidth } from '../../interface';
@@ -51,7 +52,7 @@ export class DownloadFileParams {
 	fileName!: string;
 }
 
-export class ScanResultParams {
+export class ScanResultParams implements ScanResult {
 	@ApiProperty()
 	@Allow()
 	virus_detected?: boolean;

apps/server/src/modules/files-storage/entity/filerecord.entity.ts

@@ -254,6 +254,12 @@ export class FileRecord extends BaseEntityWithTimestamps {
 		return isVerified;
 	}
 
+	public isPreviewPossible(): boolean {
+		const isPreviewPossible = Object.values<string>(PreviewInputMimeTypes).includes(this.mimeType);
+
+		return isPreviewPossible;
+	}
+
 	public getParentInfo(): IParentInfo {
 		const { parentId, parentType, schoolId } = this;
 
@@ -269,7 +275,7 @@ export class FileRecord extends BaseEntityWithTimestamps {
 			return PreviewStatus.PREVIEW_NOT_POSSIBLE_SCAN_STATUS_BLOCKED;
 		}
 
-		if (!Object.values<string>(PreviewInputMimeTypes).includes(this.mimeType)) {
+		if (!this.isPreviewPossible()) {
 			return PreviewStatus.PREVIEW_NOT_POSSIBLE_WRONG_MIME_TYPE;
 		}
 

apps/server/src/modules/files-storage/files-storage.config.ts

@@ -6,14 +6,16 @@ export const FILES_STORAGE_S3_CONNECTION = 'FILES_STORAGE_S3_CONNECTION';
 export interface IFileStorageConfig extends ICoreModuleConfig {
 	MAX_FILE_SIZE: number;
 	MAX_SECURITY_CHECK_FILE_SIZE: number;
+	USE_STREAM_TO_ANTIVIRUS: boolean;
 }
 
 const fileStorageConfig: IFileStorageConfig = {
 	INCOMING_REQUEST_TIMEOUT: Configuration.get('FILES_STORAGE__INCOMING_REQUEST_TIMEOUT') as number,
 	INCOMING_REQUEST_TIMEOUT_COPY_API: Configuration.get('INCOMING_REQUEST_TIMEOUT_COPY_API') as number,
 	MAX_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number,
-	MAX_SECURITY_CHECK_FILE_SIZE: Configuration.get('FILE_SECURITY_CHECK_MAX_FILE_SIZE') as number,
+	MAX_SECURITY_CHECK_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number,
 	NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string,
+	USE_STREAM_TO_ANTIVIRUS: Configuration.get('FILES_STORAGE__USE_STREAM_TO_ANTIVIRUS') as boolean,
 };
 
 // The configurations lookup

apps/server/src/modules/files-storage/files-storage.module.ts

@@ -24,6 +24,8 @@ const imports = [
 		filesServiceBaseUrl: Configuration.get('FILES_STORAGE__SERVICE_BASE_URL') as string,
 		exchange: Configuration.get('ANTIVIRUS_EXCHANGE') as string,
 		routingKey: Configuration.get('ANTIVIRUS_ROUTING_KEY') as string,
+		hostname: Configuration.get('CLAMAV__SERVICE_HOSTNAME') as string,
+		port: Configuration.get('CLAMAV__SERVICE_PORT') as number,
 	}),
 	S3ClientModule.register([s3Config]),
 	PreviewGeneratorProducerModule,

apps/server/src/modules/files-storage/service/files-storage-copy.service.spec.ts

@@ -2,7 +2,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { InternalServerErrorException } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-download.service.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { NotAcceptableException, NotFoundException } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { GetFile, S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-get.service.spec.ts

@@ -2,7 +2,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-restore.service.spec.ts

@@ -2,7 +2,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-update.service.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { ConflictException, NotFoundException } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/service/files-storage-upload.service.spec.ts

@@ -3,14 +3,14 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { BadRequestException } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { readableStreamWithFileTypeFactory } from '@shared/testing/factory/readable-stream-with-file-type.factory';
 import { LegacyLogger } from '@src/core/logger';
 import { MimeType } from 'file-type';
 import FileType from 'file-type-cjs/file-type-cjs-index';
-import { Readable } from 'stream';
+import { PassThrough, Readable } from 'stream';
 import { FileRecordParams } from '../controller/dto';
 import { FileDto } from '../dto';
 import { FileRecord, FileRecordParentType } from '../entity';
@@ -122,6 +122,12 @@ describe('FilesStorageService upload methods', () => {
 
 			const readableStreamWithFileType = readableStreamWithFileTypeFactory.build();
 
+			antivirusService.checkStream.mockResolvedValueOnce({
+				virus_detected: undefined,
+				virus_signature: undefined,
+				error: undefined,
+			});
+
 			return {
 				params,
 				file,
@@ -170,7 +176,7 @@ describe('FilesStorageService upload methods', () => {
 
 				await service.uploadFile(userId, params, file);
 
-				expect(getMimeTypeSpy).toHaveBeenCalledWith(file.data);
+				expect(getMimeTypeSpy).toHaveBeenCalledWith(expect.any(PassThrough));
 			});
 
 			it('should call getFileRecordsOfParent with correct params', async () => {
@@ -199,14 +205,6 @@ describe('FilesStorageService upload methods', () => {
 				);
 			});
 
-			it('should call antivirusService.send with fileRecord', async () => {
-				const { params, file, userId } = setup();
-
-				const fileRecord = await service.uploadFile(userId, params, file);
-
-				expect(antivirusService.send).toHaveBeenCalledWith(fileRecord.securityCheck.requestToken);
-			});
-
 			it('should call storageClient.create with correct params', async () => {
 				const { params, file, userId } = setup();
 
@@ -223,6 +221,29 @@ describe('FilesStorageService upload methods', () => {
 
 				expect(result).toBeInstanceOf(FileRecord);
 			});
+
+			describe('Antivirus handling by upload ', () => {
+				describe('when useStreamToAntivirus is true', () => {
+					it('should call antivirusService.send with fileRecord', async () => {
+						const { params, file, userId } = setup();
+						configService.get.mockReturnValueOnce(true);
+						await service.uploadFile(userId, params, file);
+
+						expect(antivirusService.checkStream).toHaveBeenCalledWith(file);
+					});
+				});
+
+				describe('when useStreamToAntivirus is false', () => {
+					it('should call antivirusService.send with fileRecord', async () => {
+						const { params, file, userId } = setup();
+						configService.get.mockReturnValueOnce(false);
+
+						const fileRecord = await service.uploadFile(userId, params, file);
+
+						expect(antivirusService.send).toHaveBeenCalledWith(fileRecord.securityCheck.requestToken);
+					});
+				});
+			});
 		});
 
 		describe('WHEN file record repo throws error', () => {
@@ -294,6 +315,7 @@ describe('FilesStorageService upload methods', () => {
 					}
 				});
 
+				configService.get.mockReturnValueOnce(true);
 				configService.get.mockReturnValueOnce(2);
 				const error = new BadRequestException(ErrorType.FILE_TOO_BIG);
 
@@ -315,6 +337,9 @@ describe('FilesStorageService upload methods', () => {
 
 				jest.spyOn(service, 'getFileRecordsOfParent').mockResolvedValue([[fileRecord], 1]);
 
+				// Mock for useStreamToAntivirus
+				configService.get.mockReturnValueOnce(false);
+
 				// Mock for max file size
 				configService.get.mockReturnValueOnce(10);
 
@@ -364,6 +389,8 @@ describe('FilesStorageService upload methods', () => {
 
 				jest.spyOn(FileType, 'fileTypeStream').mockResolvedValueOnce(readableStreamWithFileType);
 
+				configService.get.mockReturnValueOnce(false);
+
 				// The fileRecord._id must be set by fileRecordRepo.save. Otherwise createPath fails.
 				// eslint-disable-next-line @typescript-eslint/require-await
 				fileRecordRepo.save.mockImplementation(async (fr) => {

apps/server/src/modules/files-storage/service/files-storage.service.ts

@@ -8,11 +8,11 @@ import {
 } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { Counted, EntityId } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { LegacyLogger } from '@src/core/logger';
 import FileType from 'file-type-cjs/file-type-cjs-index';
-import { Readable } from 'stream';
+import { PassThrough, Readable } from 'stream';
 import {
 	CopyFileResponse,
 	CopyFilesOfParentParams,
@@ -104,7 +104,8 @@ export class FilesStorageService {
 
 	private async detectMimeType(file: FileDto): Promise<{ mimeType: string; stream: Readable }> {
 		if (this.isStreamMimeTypeDetectionPossible(file.mimeType)) {
-			const { stream, mime: detectedMimeType } = await this.detectMimeTypeByStream(file.data);
+			const source = file.data.pipe(new PassThrough());
+			const { stream, mime: detectedMimeType } = await this.detectMimeTypeByStream(source);
 
 			const mimeType = detectedMimeType ?? file.mimeType;
 
@@ -151,18 +152,32 @@ export class FilesStorageService {
 		file: FileDto
 	): Promise<void> {
 		const filePath = createPath(params.schoolId, fileRecord.id);
+		const useStreamToAntivirus = this.configService.get<boolean>('USE_STREAM_TO_ANTIVIRUS');
 
 		try {
 			const fileSizePromise = this.countFileSize(file);
 
-			await this.storageClient.create(filePath, file);
+			if (useStreamToAntivirus && fileRecord.isPreviewPossible()) {
+				const streamToAntivirus = file.data.pipe(new PassThrough());
+
+				const [, antivirusClientResponse] = await Promise.all([
+					this.storageClient.create(filePath, file),
+					this.antivirusService.checkStream(streamToAntivirus),
+				]);
+				const { status, reason } = FileRecordMapper.mapScanResultParamsToDto(antivirusClientResponse);
+				fileRecord.updateSecurityCheckStatus(status, reason);
+			} else {
+				await this.storageClient.create(filePath, file);
+			}
 
 			// The actual file size is set here because it is known only after the whole file is streamed.
 			fileRecord.size = await fileSizePromise;
 			this.throwErrorIfFileIsTooBig(fileRecord.size);
 			await this.fileRecordRepo.save(fileRecord);
 
-			await this.sendToAntivirus(fileRecord);
+			if (!useStreamToAntivirus || !fileRecord.isPreviewPossible()) {
+				await this.sendToAntivirus(fileRecord);
+			}
 		} catch (error) {
 			await this.storageClient.delete([filePath]);
 			await this.fileRecordRepo.delete(fileRecord);

apps/server/src/modules/files-storage/uc/files-storage-copy.uc.spec.ts

@@ -4,7 +4,7 @@ import { HttpService } from '@nestjs/axios';
 import { ForbiddenException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { EntityId, Permission } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts

@@ -4,7 +4,7 @@ import { HttpService } from '@nestjs/axios';
 import { ForbiddenException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { Counted, EntityId } from '@shared/domain';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { HttpService } from '@nestjs/axios';
 import { ForbiddenException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-download.uc.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { HttpService } from '@nestjs/axios';
 import { ForbiddenException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-get.uc.spec.ts

@@ -2,7 +2,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { HttpService } from '@nestjs/axios';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-restore.uc.spec.ts

@@ -3,7 +3,7 @@ import { ObjectId } from '@mikro-orm/mongodb';
 import { HttpService } from '@nestjs/axios';
 import { ForbiddenException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';

apps/server/src/modules/files-storage/uc/files-storage-update.uc.spec.ts

@@ -2,7 +2,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { HttpService } from '@nestjs/axios';
 import { Test, TestingModule } from '@nestjs/testing';
-import { AntivirusService } from '@shared/infra/antivirus/antivirus.service';
+import { AntivirusService } from '@shared/infra/antivirus';
 import { S3ClientAdapter } from '@shared/infra/s3-client';
 import { fileRecordFactory, setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';