From a7273a1394ddb02f51cc4b11d89af4b9b2281d2e Mon Sep 17 00:00:00 2001 From: Max Bischof <106820326+bischofmax@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:08:04 +0100 Subject: [PATCH 1/2] BC-5727 - Use etag to use preview cache (#4537) --- .../files-storage-preview.api.spec.ts | 111 +++++++++++++----- .../controller/files-storage.controller.ts | 14 ++- 2 files changed, 95 insertions(+), 30 deletions(-) diff --git a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts index f63eef6ac68..6ccc6d0d5ea 100644 --- a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts +++ b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts @@ -1,4 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { AntivirusService } from '@infra/antivirus'; +import { PreviewProducer } from '@infra/preview-generator'; +import { S3ClientAdapter } from '@infra/s3-client'; import { EntityManager, ObjectId } from '@mikro-orm/mongodb'; import { ICurrentUser } from '@modules/authentication'; import { JwtAuthGuard } from '@modules/authentication/guard/jwt-auth.guard'; @@ -6,9 +9,6 @@ import { ExecutionContext, INestApplication, NotFoundException, StreamableFile } import { Test, TestingModule } from '@nestjs/testing'; import { ApiValidationError } from '@shared/common'; import { EntityId, Permission } from '@shared/domain'; -import { AntivirusService } from '@infra/antivirus'; -import { PreviewProducer } from '@infra/preview-generator'; -import { S3ClientAdapter } from '@infra/s3-client'; import { cleanupCollections, mapUserToCurrentUser, roleFactory, schoolFactory, userFactory } from '@shared/testing'; import NodeClam from 'clamscan'; import { Request } from 'express'; @@ -63,6 +63,19 @@ class API { }; } + async getPreviewWithEtag(routeName: string, etag: string, query?: string | Record) { + const response = await request(this.app.getHttpServer()) + .get(routeName) + .query(query || {}) + .set('If-None-Match', etag); + + return { + result: response.body as StreamableFile, + error: response.body as ApiValidationError, + status: response.status, + }; + } + async getPreviewBytesRange(routeName: string, bytesRange: string, query?: string | Record) { const response = await request(this.app.getHttpServer()) .get(routeName) @@ -299,34 +312,75 @@ describe('File Controller (API) - preview', () => { return { uploadedFile }; }; - it('should return status 200 for successful download', async () => { - const { uploadedFile } = await setup(); - const query = { - ...defaultQueryParameters, - forceUpdate: false, - }; - - const response = await api.getPreview(`/file/preview/${uploadedFile.id}/${uploadedFile.name}`, query); - - expect(response.status).toEqual(200); + describe('WHEN header contains no etag', () => { + it('should return status 200 for successful download', async () => { + const { uploadedFile } = await setup(); + const query = { + ...defaultQueryParameters, + forceUpdate: false, + }; + + const response = await api.getPreview(`/file/preview/${uploadedFile.id}/${uploadedFile.name}`, query); + + expect(response.status).toEqual(200); + }); + + it('should return status 206 and required headers for the successful partial file stream download', async () => { + const { uploadedFile } = await setup(); + const query = { + ...defaultQueryParameters, + forceUpdate: false, + }; + + const response = await api.getPreviewBytesRange( + `/file/preview/${uploadedFile.id}/${uploadedFile.name}`, + 'bytes=0-', + query + ); + + expect(response.status).toEqual(206); + expect(response.headers['accept-ranges']).toMatch('bytes'); + expect(response.headers['content-range']).toMatch('bytes 0-3/4'); + expect(response.headers.etag).toMatch('testTag'); + }); }); - it('should return status 206 and required headers for the successful partial file stream download', async () => { - const { uploadedFile } = await setup(); - const query = { - ...defaultQueryParameters, - forceUpdate: false, - }; - - const response = await api.getPreviewBytesRange( - `/file/preview/${uploadedFile.id}/${uploadedFile.name}`, - 'bytes=0-', - query - ); + describe('WHEN header contains not matching etag', () => { + it('should return status 200 for successful download', async () => { + const { uploadedFile } = await setup(); + const query = { + ...defaultQueryParameters, + forceUpdate: false, + }; + const etag = 'otherTag'; + + const response = await api.getPreviewWithEtag( + `/file/preview/${uploadedFile.id}/${uploadedFile.name}`, + etag, + query + ); + + expect(response.status).toEqual(200); + }); + }); - expect(response.status).toEqual(206); - expect(response.headers['accept-ranges']).toMatch('bytes'); - expect(response.headers['content-range']).toMatch('bytes 0-3/4'); + describe('WHEN header contains matching etag', () => { + it('should return status 304', async () => { + const { uploadedFile } = await setup(); + const query = { + ...defaultQueryParameters, + forceUpdate: false, + }; + const etag = 'testTag'; + + const response = await api.getPreviewWithEtag( + `/file/preview/${uploadedFile.id}/${uploadedFile.name}`, + etag, + query + ); + + expect(response.status).toEqual(304); + }); }); }); @@ -369,6 +423,7 @@ describe('File Controller (API) - preview', () => { expect(response.status).toEqual(206); expect(response.headers['accept-ranges']).toMatch('bytes'); expect(response.headers['content-range']).toMatch('bytes 0-3/4'); + expect(response.headers.etag).toMatch('testTag'); }); }); }); diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 736d69e3e29..7269336c44c 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -120,6 +120,7 @@ export class FilesStorageController { @ApiOperation({ summary: 'Streamable download of a preview file.' }) @ApiResponse({ status: 200, type: StreamableFile }) @ApiResponse({ status: 206, type: StreamableFile }) + @ApiResponse({ status: 304, description: 'Not Modified' }) @ApiResponse({ status: 400, type: ApiValidationError }) @ApiResponse({ status: 403, type: ForbiddenException }) @ApiResponse({ status: 404, type: NotFoundException }) @@ -134,8 +135,9 @@ export class FilesStorageController { @Query() previewParams: PreviewParams, @Req() req: Request, @Res({ passthrough: true }) response: Response, - @Headers('Range') bytesRange?: string - ): Promise { + @Headers('Range') bytesRange?: string, + @Headers('If-None-Match') etag?: string + ): Promise { const fileResponse = await this.filesStorageUC.downloadPreview( currentUser.userId, params, @@ -143,6 +145,14 @@ export class FilesStorageController { bytesRange ); + response.set({ ETag: fileResponse.etag }); + + if (etag === fileResponse.etag) { + response.status(HttpStatus.NOT_MODIFIED); + + return undefined; + } + const streamableFile = this.streamFileToClient(req, fileResponse, response, bytesRange); return streamableFile; From f81f023ddff2194b5154e58a511f271022df33a4 Mon Sep 17 00:00:00 2001 From: sszafGCA <116172610+sszafGCA@users.noreply.github.com> Date: Thu, 9 Nov 2023 16:52:31 +0100 Subject: [PATCH 2/2] BC-4895-stop sending emails to non existent thr @schul-cloud.org addresses (#4476) * impl of mail addresses whitelist logic --- .../src/infra/mail/interfaces/mail-config.ts | 3 ++ apps/server/src/infra/mail/mail.module.ts | 3 ++ .../src/infra/mail/mail.service.spec.ts | 49 ++++++++++++++++--- apps/server/src/infra/mail/mail.service.ts | 42 ++++++++++++++-- .../src/modules/server/server.config.ts | 7 ++- config/default.schema.json | 1 + 6 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 apps/server/src/infra/mail/interfaces/mail-config.ts diff --git a/apps/server/src/infra/mail/interfaces/mail-config.ts b/apps/server/src/infra/mail/interfaces/mail-config.ts new file mode 100644 index 00000000000..6dbb0c7864d --- /dev/null +++ b/apps/server/src/infra/mail/interfaces/mail-config.ts @@ -0,0 +1,3 @@ +export interface IMailConfig { + ADDITIONAL_BLACKLISTED_EMAIL_DOMAINS: string[]; +} diff --git a/apps/server/src/infra/mail/mail.module.ts b/apps/server/src/infra/mail/mail.module.ts index 1ca0630c44f..ee6d50d59e7 100644 --- a/apps/server/src/infra/mail/mail.module.ts +++ b/apps/server/src/infra/mail/mail.module.ts @@ -1,5 +1,7 @@ import { Module, DynamicModule } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { MailService } from './mail.service'; +import { IMailConfig } from './interfaces/mail-config'; interface MailModuleOptions { exchange: string; @@ -17,6 +19,7 @@ export class MailModule { provide: 'MAIL_SERVICE_OPTIONS', useValue: { exchange: options.exchange, routingKey: options.routingKey }, }, + ConfigService, ], exports: [MailService], }; diff --git a/apps/server/src/infra/mail/mail.service.spec.ts b/apps/server/src/infra/mail/mail.service.spec.ts index 58c0ce9336a..ebc77030252 100644 --- a/apps/server/src/infra/mail/mail.service.spec.ts +++ b/apps/server/src/infra/mail/mail.service.spec.ts @@ -1,7 +1,10 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; import { Test, TestingModule } from '@nestjs/testing'; +import { createMock } from '@golevelup/ts-jest'; +import { ConfigService } from '@nestjs/config'; import { Mail } from './mail.interface'; import { MailService } from './mail.service'; +import { IMailConfig } from './interfaces/mail-config'; describe('MailService', () => { let module: TestingModule; @@ -19,6 +22,10 @@ describe('MailService', () => { MailService, { provide: AmqpConnection, useValue: { publish: () => {} } }, { provide: 'MAIL_SERVICE_OPTIONS', useValue: mailServiceOptions }, + { + provide: ConfigService, + useValue: createMock>({ get: () => ['schul-cloud.org', 'example.com'] }), + }, ], }).compile(); @@ -34,13 +41,43 @@ describe('MailService', () => { expect(service).toBeDefined(); }); - it('should send given data to queue', async () => { - const data: Mail = { mail: { plainTextContent: 'content', subject: 'Test' }, recipients: ['test@example.com'] }; - const amqpConnectionSpy = jest.spyOn(amqpConnection, 'publish'); + describe('send', () => { + describe('when recipients array is empty', () => { + it('should not send email', async () => { + const data: Mail = { + mail: { plainTextContent: 'content', subject: 'Test' }, + recipients: ['test@schul-cloud.org'], + }; - await service.send(data); + const amqpConnectionSpy = jest.spyOn(amqpConnection, 'publish'); - const expectedParams = [mailServiceOptions.exchange, mailServiceOptions.routingKey, data, { persistent: true }]; - expect(amqpConnectionSpy).toHaveBeenCalledWith(...expectedParams); + await service.send(data); + + expect(amqpConnectionSpy).toHaveBeenCalledTimes(0); + }); + }); + describe('when sending email', () => { + it('should remove email address that have blacklisted domain and send given data to queue', async () => { + const data: Mail = { + mail: { plainTextContent: 'content', subject: 'Test' }, + recipients: ['test@schul-cloud.org', 'test@example1.com', 'test2@schul-cloud.org', 'test3@schul-cloud.org'], + cc: ['test@example.com', 'test5@schul-cloud.org', 'test6@schul-cloud.org'], + bcc: ['test7@schul-cloud.org', 'test@example2.com', 'test8@schul-cloud.org'], + replyTo: ['test@example3.com', 'test9@schul-cloud.org', 'test10@schul-cloud.org'], + }; + + const amqpConnectionSpy = jest.spyOn(amqpConnection, 'publish'); + + await service.send(data); + + expect(data.recipients).toEqual(['test@example1.com']); + expect(data.cc).toEqual([]); + expect(data.bcc).toEqual(['test@example2.com']); + expect(data.replyTo).toEqual(['test@example3.com']); + + const expectedParams = [mailServiceOptions.exchange, mailServiceOptions.routingKey, data, { persistent: true }]; + expect(amqpConnectionSpy).toHaveBeenCalledWith(...expectedParams); + }); + }); }); }); diff --git a/apps/server/src/infra/mail/mail.service.ts b/apps/server/src/infra/mail/mail.service.ts index aaf9cfacb9d..432f0746934 100644 --- a/apps/server/src/infra/mail/mail.service.ts +++ b/apps/server/src/infra/mail/mail.service.ts @@ -1,7 +1,8 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; import { Inject, Injectable } from '@nestjs/common'; - +import { ConfigService } from '@nestjs/config'; import { Mail } from './mail.interface'; +import { IMailConfig } from './interfaces/mail-config'; interface MailServiceOptions { exchange: string; @@ -10,12 +11,47 @@ interface MailServiceOptions { @Injectable() export class MailService { + private readonly domainBlacklist: string[]; + constructor( private readonly amqpConnection: AmqpConnection, - @Inject('MAIL_SERVICE_OPTIONS') private readonly options: MailServiceOptions - ) {} + @Inject('MAIL_SERVICE_OPTIONS') private readonly options: MailServiceOptions, + private readonly configService: ConfigService + ) { + this.domainBlacklist = this.configService.get('ADDITIONAL_BLACKLISTED_EMAIL_DOMAINS'); + } public async send(data: Mail): Promise { + if (this.domainBlacklist.length > 0) { + data.recipients = this.filterEmailAdresses(data.recipients) as string[]; + data.cc = this.filterEmailAdresses(data.cc); + data.bcc = this.filterEmailAdresses(data.bcc); + data.replyTo = this.filterEmailAdresses(data.replyTo); + } + + if (data.recipients.length === 0) { + return; + } + await this.amqpConnection.publish(this.options.exchange, this.options.routingKey, data, { persistent: true }); } + + private filterEmailAdresses(mails: string[] | undefined): string[] | undefined { + if (mails === undefined || mails === null) { + return mails; + } + const mailWhitelist: string[] = []; + + for (const mail of mails) { + const mailDomain = this.getMailDomain(mail); + if (mailDomain && !this.domainBlacklist.includes(mailDomain)) { + mailWhitelist.push(mail); + } + } + return mailWhitelist; + } + + private getMailDomain(mail: string): string { + return mail.split('@')[1]; + } } diff --git a/apps/server/src/modules/server/server.config.ts b/apps/server/src/modules/server/server.config.ts index becc7ec78fc..5d1ea95cc3b 100644 --- a/apps/server/src/modules/server/server.config.ts +++ b/apps/server/src/modules/server/server.config.ts @@ -5,6 +5,7 @@ import type { IAccountConfig } from '@modules/account'; import type { IFilesStorageClientConfig } from '@modules/files-storage-client'; import type { IUserConfig } from '@modules/user'; import type { ICommonCartridgeConfig } from '@modules/learnroom/common-cartridge'; +import { IMailConfig } from '@src/infra/mail/interfaces/mail-config'; export enum NodeEnvType { TEST = 'test', @@ -19,7 +20,8 @@ export interface IServerConfig IFilesStorageClientConfig, IAccountConfig, IIdentityManagementConfig, - ICommonCartridgeConfig { + ICommonCartridgeConfig, + IMailConfig { NODE_ENV: string; SC_DOMAIN: string; } @@ -39,6 +41,9 @@ const config: IServerConfig = { FEATURE_IDENTITY_MANAGEMENT_ENABLED: Configuration.get('FEATURE_IDENTITY_MANAGEMENT_ENABLED') as boolean, FEATURE_IDENTITY_MANAGEMENT_STORE_ENABLED: Configuration.get('FEATURE_IDENTITY_MANAGEMENT_STORE_ENABLED') as boolean, FEATURE_IDENTITY_MANAGEMENT_LOGIN_ENABLED: Configuration.get('FEATURE_IDENTITY_MANAGEMENT_LOGIN_ENABLED') as boolean, + ADDITIONAL_BLACKLISTED_EMAIL_DOMAINS: (Configuration.get('ADDITIONAL_BLACKLISTED_EMAIL_DOMAINS') as string) + .split(',') + .map((domain) => domain.trim()), }; export const serverConfig = () => config; diff --git a/config/default.schema.json b/config/default.schema.json index a34d8e899ad..bd637b719e9 100644 --- a/config/default.schema.json +++ b/config/default.schema.json @@ -177,6 +177,7 @@ }, "ADDITIONAL_BLACKLISTED_EMAIL_DOMAINS": { "type": "string", + "default":"", "description": "Add custom domain to the list of blocked domains (comma separated list)." }, "FEATURE_TSP_AUTO_CONSENT_ENABLED": {