From 84c44c1bf99f92b16df1c32792b5670dbb2b4d72 Mon Sep 17 00:00:00 2001 From: Sergej Hoffmann <97111299+SevenWaysDP@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:56:47 +0200 Subject: [PATCH 1/4] Bc 5170 no preview for larger files (#4457) --- .github/workflows/push.yml | 18 +- .github/workflows/tag.yml | 8 +- ...file.filestorage => Dockerfile.filepreview | 0 .../schulcloud-server-core/tasks/main.yml | 29 + .../templates/api-files-deployment.yml.j2 | 2 +- .../preview-generator-configmap.yml.j2 | 8 + .../preview-generator-deployment.yml.j2 | 46 + .../preview-generator-onepassword.yml.j2 | 9 + .../preview-generator-scaled-object.yml.j2 | 23 + .../src/apps/files-storage-consumer.app.ts | 2 +- apps/server/src/apps/files-storage.app.ts | 3 +- .../apps/preview-generator-consumer.app.ts | 17 + .../files-storage-client/mapper/index.ts | 3 +- .../service/files-storage.producer.spec.ts | 3 +- .../service/files-storage.producer.ts | 53 +- .../files-storage-preview.api.spec.ts | 15 +- .../controller/files-storage.consumer.spec.ts | 21 +- .../controller/files-storage.consumer.ts | 8 +- .../controller/files-storage.controller.ts | 6 +- .../modules/files-storage/controller/index.ts | 2 +- .../src/modules/files-storage/dto/file.dto.ts | 3 +- .../entity/filerecord.entity.spec.ts | 50 ++ .../files-storage/entity/filerecord.entity.ts | 7 + .../files-preview-amqp.module.ts | 8 + .../files-storage/files-storage.config.ts | 8 +- .../files-storage/files-storage.module.ts | 2 + .../files-storage/helper/file-record.spec.ts | 44 +- .../files-storage/helper/file-record.ts | 25 + .../server/src/modules/files-storage/index.ts | 7 +- .../files-storage/interface/interfaces.ts | 7 +- .../src/modules/files-storage/mapper/index.ts | 1 + .../mapper/preview.builder.spec.ts | 61 ++ .../files-storage/mapper/preview.builder.ts | 47 ++ .../files-storage-delete.service.spec.ts | 62 +- .../service/files-storage.service.ts | 10 +- .../service/preview.service.spec.ts | 787 +++++++----------- .../files-storage/service/preview.service.ts | 110 +-- .../uc/files-storage-delete.uc.spec.ts | 12 +- .../files-storage-download-preview.uc.spec.ts | 11 +- .../files-storage/uc/files-storage.uc.ts | 13 +- .../shared/infra/preview-generator/index.ts | 4 + .../preview-generator/interface/index.ts | 1 + .../interface/preview-consumer-config.ts | 11 + .../preview-generator/interface/preview.ts | 15 + .../loggable/preview-actions.loggable.spec.ts | 37 + .../loggable/preview-actions.loggable.ts | 19 + .../preview-generator-consumer.module.ts | 36 + .../preview-generator-producer.module.ts | 11 + .../preview-generator.builder.spec.ts | 28 + .../preview-generator.builder.ts | 16 + .../preview-generator.consumer.spec.ts | 80 ++ .../preview-generator.consumer.ts | 27 + .../preview-generator.service.spec.ts | 151 ++++ .../preview-generator.service.ts | 56 ++ .../preview.producer.spec.ts | 128 +++ .../preview-generator/preview.producer.ts | 31 + .../infra/rabbitmq}/error.mapper.spec.ts | 0 .../infra/rabbitmq}/error.mapper.ts | 0 .../infra/rabbitmq/exchange/files-preview.ts | 7 + .../shared/infra/rabbitmq/exchange/index.ts | 1 + .../server/src/shared/infra/rabbitmq/index.ts | 2 + .../shared/infra/rabbitmq/rabbitmq.module.ts | 6 +- .../rabbitmq/rpc-message-producer.spec.ts | 130 +++ .../infra/rabbitmq/rpc-message-producer.ts | 37 + .../src/shared/infra/rabbitmq/rpc-message.ts | 2 +- .../shared/infra/s3-client/interface/index.ts | 1 - config/default.schema.json | 9 +- nest-cli.json | 9 + package.json | 7 +- 69 files changed, 1654 insertions(+), 759 deletions(-) rename Dockerfile.filestorage => Dockerfile.filepreview (100%) create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 create mode 100644 apps/server/src/apps/preview-generator-consumer.app.ts create mode 100644 apps/server/src/modules/files-storage/files-preview-amqp.module.ts create mode 100644 apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts create mode 100644 apps/server/src/modules/files-storage/mapper/preview.builder.ts create mode 100644 apps/server/src/shared/infra/preview-generator/index.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/index.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/preview.ts create mode 100644 apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.service.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview.producer.ts rename apps/server/src/{modules/files-storage-client/mapper => shared/infra/rabbitmq}/error.mapper.spec.ts (100%) rename apps/server/src/{modules/files-storage-client/mapper => shared/infra/rabbitmq}/error.mapper.ts (100%) create mode 100644 apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts create mode 100644 apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts create mode 100644 apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 94b3514822f..c042be2c2a9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -68,36 +68,36 @@ jobs: tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }} labels: ${{ steps.docker_meta_img.outputs.labels }} - - name: Docker meta Service Name (file storage) + - name: Docker meta Service Name (file preview) id: docker_meta_img_file_storage uses: docker/metadata-action@v4 with: images: ghcr.io/${{ github.repository }} tags: | - type=ref,event=branch,enable=false,priority=600,prefix=file-storage- - type=sha,enable=true,priority=600,prefix=file-storage- + type=ref,event=branch,enable=false,priority=600,prefix=file-preview- + type=sha,enable=true,priority=600,prefix=file-preview- labels: | org.opencontainers.image.title=schulcloud-file-storage - - name: test image exists (file storage) + - name: test image exists (file preview) run: | - echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV + echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV - - name: Set up Docker Buildx (file storage) + - name: Set up Docker Buildx (file preview) if: ${{ env.IMAGE_EXISTS == 0 }} uses: docker/setup-buildx-action@v2 - - name: Build and push ${{ github.repository }} (file storage) + - name: Build and push ${{ github.repository }} (file preview) if: ${{ env.IMAGE_EXISTS == 0 }} uses: docker/build-push-action@v4 with: build-args: | BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }} context: . - file: ./Dockerfile.filestorage + file: ./Dockerfile.filepreview platforms: linux/amd64 push: true - tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} + tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} labels: | ${{ steps.docker_meta_img_file_storage.outputs.labels }} diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index fa62df4b0ab..8f484116ca8 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -48,14 +48,14 @@ jobs: tags: ${{ steps.docker_meta_img_hub.outputs.tags }} labels: ${{ steps.docker_meta_img_hub.outputs.labels }} - - name: Docker meta Service Name for docker hub (file storage) + - name: Docker meta Service Name for docker hub (file preview) id: docker_meta_img_hub_file_storage uses: docker/metadata-action@v4 with: images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server tags: | - type=semver,pattern={{version}},prefix=file-storage-,onlatest=false - type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false + type=semver,pattern={{version}},prefix=file-preview-,onlatest=false + type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false labels: | org.opencontainers.image.title=schulcloud-file-storage - name: Build and push ${{ github.repository }} (file-storage) @@ -64,7 +64,7 @@ jobs: build-args: | BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }} context: . - file: ./Dockerfile.filestorage + file: ./Dockerfile.filepreview platforms: linux/amd64 push: true tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }} diff --git a/Dockerfile.filestorage b/Dockerfile.filepreview similarity index 100% rename from Dockerfile.filestorage rename to Dockerfile.filepreview diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml index fff5b10197a..7f1bbeeecfe 100644 --- a/ansible/roles/schulcloud-server-core/tasks/main.yml +++ b/ansible/roles/schulcloud-server-core/tasks/main.yml @@ -83,3 +83,32 @@ kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" template: amqp-files-deployment.yml.j2 + + - name: Preview Generator Deployment + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-deployment.yml.j2 + + - name: preview generator configmap + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-configmap.yml.j2 + apply: yes + + - name: preview generator Secret by 1Password + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-onepassword.yml.j2 + when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool + + - name: preview generator scaled object + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-scaled-object.yml.j2 + when: + - KEDA_ENABLED is defined and KEDA_ENABLED|bool + - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 index 7f5a5b7e50b..727b5a9f4f0 100644 --- a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 @@ -29,7 +29,7 @@ spec: runAsNonRoot: true containers: - name: api-files - image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }} imagePullPolicy: IfNotPresent ports: - containerPort: 4444 diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 new file mode 100644 index 00000000000..457a5e47364 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: preview-generator-configmap + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +data: diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 new file mode 100644 index 00000000000..51d87b88755 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: preview-generator-deployment + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + #maxUnavailable: 1 + revisionHistoryLimit: 4 + paused: false + selector: + matchLabels: + app: preview-generator + template: + metadata: + labels: + app: preview-generator + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + runAsNonRoot: true + containers: + - name: preview-generator + image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: preview-generator-configmap + - secretRef: + name: preview-generator-secret + command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod'] + resources: + limits: + cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }} + memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }} + requests: + cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }} + memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }} diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 new file mode 100644 index 00000000000..51d979e27e0 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: preview-generator-secret + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator" diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 new file mode 100644 index 00000000000..2f8f9091b8e --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: preview-generator-rabbitmq-scaledobject + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + scaleTargetRef: + name: preview-generator-deployment + idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }} + minReplicaCount: {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }} + maxReplicaCount: {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }} + triggers: + - type: rabbitmq + metadata: + protocol: amqp + queueName: generate-preview + mode: QueueLength + value: "1" + authenticationRef: + name: rabbitmq-trigger-auth diff --git a/apps/server/src/apps/files-storage-consumer.app.ts b/apps/server/src/apps/files-storage-consumer.app.ts index a18b5f4604b..cb62c9c76f9 100644 --- a/apps/server/src/apps/files-storage-consumer.app.ts +++ b/apps/server/src/apps/files-storage-consumer.app.ts @@ -3,7 +3,7 @@ import { NestFactory } from '@nestjs/core'; // register source-map-support for debugging -import { FilesStorageAMQPModule } from '@modules/files-storage'; +import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module'; import { install as sourceMapInstall } from 'source-map-support'; async function bootstrap() { diff --git a/apps/server/src/apps/files-storage.app.ts b/apps/server/src/apps/files-storage.app.ts index 2d2f9343ac2..adbaac9d90c 100644 --- a/apps/server/src/apps/files-storage.app.ts +++ b/apps/server/src/apps/files-storage.app.ts @@ -8,9 +8,10 @@ import express from 'express'; import { install as sourceMapInstall } from 'source-map-support'; // application imports +import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module'; +import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const'; import { SwaggerDocumentOptions } from '@nestjs/swagger'; import { LegacyLogger } from '@src/core/logger'; -import { API_VERSION_PATH, FilesStorageApiModule } from '@modules/files-storage'; import { enableOpenApiDocs } from '@src/shared/controller/swagger'; async function bootstrap() { diff --git a/apps/server/src/apps/preview-generator-consumer.app.ts b/apps/server/src/apps/preview-generator-consumer.app.ts new file mode 100644 index 00000000000..1c2be631233 --- /dev/null +++ b/apps/server/src/apps/preview-generator-consumer.app.ts @@ -0,0 +1,17 @@ +/* istanbul ignore file */ +/* eslint-disable no-console */ +import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module'; +import { NestFactory } from '@nestjs/core'; +import { install as sourceMapInstall } from 'source-map-support'; + +async function bootstrap() { + sourceMapInstall(); + + const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule); + await nestApp.init(); + + console.log('#############################################'); + console.log(`### Start Preview Generator AMQP Consumer ###`); + console.log('#############################################'); +} +void bootstrap(); diff --git a/apps/server/src/modules/files-storage-client/mapper/index.ts b/apps/server/src/modules/files-storage-client/mapper/index.ts index 06f40e707e1..006cf64b627 100644 --- a/apps/server/src/modules/files-storage-client/mapper/index.ts +++ b/apps/server/src/modules/files-storage-client/mapper/index.ts @@ -1,4 +1,3 @@ -export * from './error.mapper'; +export * from './copy-files-of-parent-param.builder'; export * from './files-storage-client.mapper'; export * from './files-storage-param.builder'; -export * from './copy-files-of-parent-param.builder'; diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts index 926f860c736..7e4ed5a1c83 100644 --- a/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts +++ b/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts @@ -3,10 +3,9 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; -import { FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq'; +import { ErrorMapper, FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq'; import { setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { ErrorMapper } from '../mapper'; import { FilesStorageProducer } from './files-storage.producer'; describe('FilesStorageProducer', () => { diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts index afd4f6365e1..ea049442df4 100644 --- a/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts +++ b/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts @@ -2,7 +2,6 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { EntityId } from '@shared/domain'; -import { RpcMessage } from '@shared/infra/rabbitmq/rpc-message'; import { LegacyLogger } from '@src/core/logger'; import { FilesStorageEvents, @@ -11,75 +10,45 @@ import { ICopyFilesOfParentParams, IFileDO, IFileRecordParams, + RpcMessageProducer, } from '@src/shared/infra/rabbitmq'; import { IFilesStorageClientConfig } from '../interfaces'; -import { ErrorMapper } from '../mapper/error.mapper'; @Injectable() -export class FilesStorageProducer { - private readonly timeout = 0; - +export class FilesStorageProducer extends RpcMessageProducer { constructor( + protected readonly amqpConnection: AmqpConnection, private readonly logger: LegacyLogger, - private readonly amqpConnection: AmqpConnection, - private readonly configService: ConfigService + protected readonly configService: ConfigService ) { + super(amqpConnection, FilesStorageExchange, configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API')); this.logger.setContext(FilesStorageProducer.name); - this.timeout = this.configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API'); } async copyFilesOfParent(payload: ICopyFilesOfParentParams): Promise { this.logger.debug({ action: 'copyFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.COPY_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.COPY_FILES_OF_PARENT, payload); this.logger.debug({ action: 'copyFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; + return response; } async listFilesOfParent(payload: IFileRecordParams): Promise { this.logger.debug({ action: 'listFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.LIST_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.LIST_FILES_OF_PARENT, payload); this.logger.debug({ action: 'listFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; + return response; } async deleteFilesOfParent(payload: EntityId): Promise { this.logger.debug({ action: 'deleteFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload); this.logger.debug({ action: 'deleteFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; - } - - // need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984 - // mapRpcErrorResponseToDomainError should also removed with this ticket - private checkError(response: RpcMessage) { - const { error } = response; - if (error) { - const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error); - throw domainError; - } - } - - private createRequest(event: FilesStorageEvents, payload: IFileRecordParams | ICopyFilesOfParentParams | EntityId) { - return { - exchange: FilesStorageExchange, - routingKey: event, - payload, - timeout: this.timeout, - }; + return response; } } diff --git a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts index 22749024b31..e87aa5ddbe6 100644 --- a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts +++ b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts @@ -7,6 +7,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ApiValidationError } from '@shared/common'; import { EntityId, Permission } from '@shared/domain'; import { AntivirusService } from '@shared/infra/antivirus'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { cleanupCollections, mapUserToCurrentUser, roleFactory, schoolFactory, userFactory } from '@shared/testing'; import NodeClam from 'clamscan'; @@ -89,6 +90,7 @@ describe('File Controller (API) - preview', () => { let app: INestApplication; let em: EntityManager; let s3ClientAdapter: DeepMocked; + let antivirusService: DeepMocked; let currentUser: ICurrentUser; let api: API; let schoolId: EntityId; @@ -103,6 +105,8 @@ describe('File Controller (API) - preview', () => { }) .overrideProvider(AntivirusService) .useValue(createMock()) + .overrideProvider(PreviewProducer) + .useValue(createMock()) .overrideProvider(FILES_STORAGE_S3_CONNECTION) .useValue(createMock()) .overrideGuard(JwtAuthGuard) @@ -123,6 +127,7 @@ describe('File Controller (API) - preview', () => { em = module.get(EntityManager); s3ClientAdapter = module.get(FILES_STORAGE_S3_CONNECTION); + antivirusService = module.get(AntivirusService); api = new API(app); }); @@ -147,6 +152,7 @@ describe('File Controller (API) - preview', () => { uploadPath = `/file/upload/${schoolId}/schools/${schoolId}`; jest.spyOn(FileType, 'fileTypeStream').mockImplementation((readable) => Promise.resolve(readable)); + antivirusService.checkStream.mockResolvedValueOnce({ virus_detected: false }); }); const setScanStatus = async (fileRecordId: EntityId, status: ScanStatus) => { @@ -329,9 +335,8 @@ describe('File Controller (API) - preview', () => { const { result: uploadedFile } = await api.postUploadFile(uploadPath); await setScanStatus(uploadedFile.id, ScanStatus.VERIFIED); - const originalFile = TestHelper.createFile(); const previewFile = TestHelper.createFile('bytes 0-3/4'); - s3ClientAdapter.get.mockResolvedValueOnce(originalFile).mockResolvedValueOnce(previewFile); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); return { uploadedFile }; }; @@ -374,12 +379,8 @@ describe('File Controller (API) - preview', () => { await setScanStatus(uploadedFile.id, ScanStatus.VERIFIED); const error = new NotFoundException(); - const originalFile = TestHelper.createFile(); const previewFile = TestHelper.createFile('bytes 0-3/4'); - s3ClientAdapter.get - .mockRejectedValueOnce(error) - .mockResolvedValueOnce(originalFile) - .mockResolvedValueOnce(previewFile); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); return { uploadedFile }; }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts b/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts index 124f486dc68..81fd2884347 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts @@ -7,6 +7,7 @@ import { courseFactory, fileRecordFactory, setupEntities } from '@shared/testing import { LegacyLogger } from '@src/core/logger'; import { FileRecord, FileRecordParentType } from '../entity'; import { FilesStorageService } from '../service/files-storage.service'; +import { PreviewService } from '../service/preview.service'; import { FileRecordResponse } from './dto'; import { FilesStorageConsumer } from './files-storage.consumer'; @@ -33,6 +34,10 @@ describe('FilesStorageConsumer', () => { provide: FilesStorageService, useValue: createMock(), }, + { + provide: PreviewService, + useValue: createMock(), + }, { provide: LegacyLogger, useValue: createMock(), @@ -165,9 +170,9 @@ describe('FilesStorageConsumer', () => { const parentId = new ObjectId().toHexString(); const fileRecords = fileRecordFactory.buildList(3); - filesStorageService.deleteFilesOfParent.mockResolvedValue([fileRecords, fileRecords.length]); + filesStorageService.getFileRecordsOfParent.mockResolvedValue([fileRecords, fileRecords.length]); - return { parentId }; + return { parentId, fileRecords }; }; it('should call filesStorageService.deleteFilesOfParent with params', async () => { @@ -175,7 +180,15 @@ describe('FilesStorageConsumer', () => { await service.deleteFilesOfParent(parentId); - expect(filesStorageService.deleteFilesOfParent).toBeCalledWith(parentId); + expect(filesStorageService.getFileRecordsOfParent).toBeCalledWith(parentId); + }); + + it('should call filesStorageService.deleteFilesOfParent with params', async () => { + const { parentId, fileRecords } = setup(); + + await service.deleteFilesOfParent(parentId); + + expect(filesStorageService.deleteFilesOfParent).toBeCalledWith(fileRecords); }); it('should return array instances of FileRecordResponse', async () => { @@ -191,7 +204,7 @@ describe('FilesStorageConsumer', () => { const setup = () => { const parentId = new ObjectId().toHexString(); - filesStorageService.deleteFilesOfParent.mockResolvedValue([[], 0]); + filesStorageService.getFileRecordsOfParent.mockResolvedValue([[], 0]); return { parentId }; }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts b/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts index e7dee3c6b0d..aabefa60f16 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts @@ -7,12 +7,14 @@ import { LegacyLogger } from '@src/core/logger'; import { FilesStorageEvents, FilesStorageExchange, ICopyFileDO, IFileDO } from '@src/shared/infra/rabbitmq'; import { FilesStorageMapper } from '../mapper'; import { FilesStorageService } from '../service/files-storage.service'; +import { PreviewService } from '../service/preview.service'; import { CopyFilesOfParentPayload, FileRecordParams } from './dto'; @Injectable() export class FilesStorageConsumer { constructor( private readonly filesStorageService: FilesStorageService, + private readonly previewService: PreviewService, private logger: LegacyLogger, // eslint-disable-next-line @typescript-eslint/no-unused-vars private readonly orm: MikroORM // don't remove it, we need it for @UseRequestContext @@ -61,7 +63,11 @@ export class FilesStorageConsumer { public async deleteFilesOfParent(@RabbitPayload() payload: EntityId): Promise> { this.logger.debug({ action: 'deleteFilesOfParent', payload }); - const [fileRecords, total] = await this.filesStorageService.deleteFilesOfParent(payload); + const [fileRecords, total] = await this.filesStorageService.getFileRecordsOfParent(payload); + + await this.previewService.deletePreviews(fileRecords); + await this.filesStorageService.deleteFilesOfParent(fileRecords); + const response = FilesStorageMapper.mapToFileRecordListResponse(fileRecords, total); return { message: response.data }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 564919670e4..736d69e3e29 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -1,3 +1,4 @@ +import { Authenticate, CurrentUser, ICurrentUser } from '@modules/authentication'; import { BadRequestException, Body, @@ -22,10 +23,10 @@ import { UseInterceptors, } from '@nestjs/common'; import { ApiConsumes, ApiHeader, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; -import { ApiValidationError, RequestLoggingInterceptor } from '@shared/common'; +import { ApiValidationError, RequestLoggingInterceptor, RequestTimeout } from '@shared/common'; import { PaginationParams } from '@shared/controller'; -import { ICurrentUser, Authenticate, CurrentUser } from '@modules/authentication'; import { Request, Response } from 'express'; +import { config } from '../files-storage.config'; import { GetFileResponse } from '../interface'; import { FilesStorageMapper } from '../mapper'; import { FileRecordMapper } from '../mapper/file-record.mapper'; @@ -126,6 +127,7 @@ export class FilesStorageController { @ApiResponse({ status: 500, type: InternalServerErrorException }) @ApiHeader({ name: 'Range', required: false }) @Get('/preview/:fileRecordId/:fileName') + @RequestTimeout(config().INCOMING_REQUEST_TIMEOUT) async downloadPreview( @Param() params: DownloadFileParams, @CurrentUser() currentUser: ICurrentUser, diff --git a/apps/server/src/modules/files-storage/controller/index.ts b/apps/server/src/modules/files-storage/controller/index.ts index ffd18a5db36..7aa61d2c93b 100644 --- a/apps/server/src/modules/files-storage/controller/index.ts +++ b/apps/server/src/modules/files-storage/controller/index.ts @@ -1,3 +1,3 @@ export * from './file-security.controller'; -export * from './files-storage.controller'; export * from './files-storage.consumer'; +export * from './files-storage.controller'; diff --git a/apps/server/src/modules/files-storage/dto/file.dto.ts b/apps/server/src/modules/files-storage/dto/file.dto.ts index 540e35bb7e6..9668ac3af72 100644 --- a/apps/server/src/modules/files-storage/dto/file.dto.ts +++ b/apps/server/src/modules/files-storage/dto/file.dto.ts @@ -1,6 +1,7 @@ +import { File } from '@shared/infra/s3-client'; import { Readable } from 'stream'; -export class FileDto { +export class FileDto implements File { constructor(file: FileDto) { this.name = file.name; this.data = file.data; diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts index f07d88d85fd..f497ff39a6c 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts @@ -783,4 +783,54 @@ describe('FileRecord Entity', () => { }); }); }); + + describe('fileNameWithoutExtension is called', () => { + describe('WHEN file name has extension', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: 'file-name.jpg' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('file-name'); + }); + }); + + describe('WHEN file name has not extension', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: 'file-name' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('file-name'); + }); + }); + + describe('WHEN file name starts with dot', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: '.bild.123.jpg' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('.bild.123'); + }); + }); + }); }); diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts index a87789d30fc..26f2807924c 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts @@ -2,6 +2,7 @@ import { Embeddable, Embedded, Entity, Enum, Index, Property } from '@mikro-orm/ import { ObjectId } from '@mikro-orm/mongodb'; import { BadRequestException } from '@nestjs/common'; import { BaseEntityWithTimestamps, EntityId } from '@shared/domain'; +import path from 'path'; import { v4 as uuid } from 'uuid'; import { ErrorType } from '../error'; import { PreviewInputMimeTypes } from '../interface/preview-input-mime-types.enum'; @@ -293,4 +294,10 @@ export class FileRecord extends BaseEntityWithTimestamps { return PreviewStatus.PREVIEW_NOT_POSSIBLE_SCAN_STATUS_ERROR; } + + public get fileNameWithoutExtension(): string { + const filenameObj = path.parse(this.name); + + return filenameObj.name; + } } diff --git a/apps/server/src/modules/files-storage/files-preview-amqp.module.ts b/apps/server/src/modules/files-storage/files-preview-amqp.module.ts new file mode 100644 index 00000000000..411a26e76d6 --- /dev/null +++ b/apps/server/src/modules/files-storage/files-preview-amqp.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { PreviewGeneratorConsumerModule } from '@shared/infra/preview-generator'; +import { defaultConfig, s3Config } from './files-storage.config'; + +@Module({ + imports: [PreviewGeneratorConsumerModule.register({ storageConfig: s3Config, serverConfig: defaultConfig })], +}) +export class PreviewGeneratorAMQPModule {} diff --git a/apps/server/src/modules/files-storage/files-storage.config.ts b/apps/server/src/modules/files-storage/files-storage.config.ts index 7dc01e9f4e1..7fac8ded763 100644 --- a/apps/server/src/modules/files-storage/files-storage.config.ts +++ b/apps/server/src/modules/files-storage/files-storage.config.ts @@ -9,13 +9,17 @@ export interface IFileStorageConfig extends ICoreModuleConfig { USE_STREAM_TO_ANTIVIRUS: boolean; } -const fileStorageConfig: IFileStorageConfig = { +export const defaultConfig = { + NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string, INCOMING_REQUEST_TIMEOUT: Configuration.get('FILES_STORAGE__INCOMING_REQUEST_TIMEOUT') as number, +}; + +const fileStorageConfig: IFileStorageConfig = { INCOMING_REQUEST_TIMEOUT_COPY_API: Configuration.get('INCOMING_REQUEST_TIMEOUT_COPY_API') as number, MAX_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number, MAX_SECURITY_CHECK_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number, - NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string, USE_STREAM_TO_ANTIVIRUS: Configuration.get('FILES_STORAGE__USE_STREAM_TO_ANTIVIRUS') as boolean, + ...defaultConfig, }; // The configurations lookup diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index 248654218ef..ccdaeb7f9fa 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -5,6 +5,7 @@ import { Module, NotFoundException } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { ALL_ENTITIES } from '@shared/domain'; import { AntivirusModule } from '@shared/infra/antivirus/antivirus.module'; +import { PreviewGeneratorProducerModule } from '@shared/infra/preview-generator'; import { RabbitMQWrapperModule } from '@shared/infra/rabbitmq/rabbitmq.module'; import { S3ClientModule } from '@shared/infra/s3-client'; import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config'; @@ -27,6 +28,7 @@ const imports = [ port: Configuration.get('CLAMAV__SERVICE_PORT') as number, }), S3ClientModule.register([s3Config]), + PreviewGeneratorProducerModule, ]; const providers = [FilesStorageService, PreviewService, FileRecordRepo]; diff --git a/apps/server/src/modules/files-storage/helper/file-record.spec.ts b/apps/server/src/modules/files-storage/helper/file-record.spec.ts index 6d975ebe21b..a999b599936 100644 --- a/apps/server/src/modules/files-storage/helper/file-record.spec.ts +++ b/apps/server/src/modules/files-storage/helper/file-record.spec.ts @@ -1,8 +1,9 @@ import { EntityId } from '@shared/domain'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { ObjectId } from 'bson'; -import { createFileRecord, markForDelete, unmarkForDelete } from '.'; +import { createFileRecord, getFormat, getPreviewName, markForDelete, unmarkForDelete } from '.'; import { FileRecord } from '../entity'; +import { PreviewOutputMimeTypes } from '../interface'; describe('File Record Helper', () => { const setupFileRecords = () => { @@ -88,4 +89,45 @@ describe('File Record Helper', () => { expect(newFileRecord).toEqual(expect.any(FileRecord)); }); }); + + describe('getFormat is called', () => { + it('should return format', () => { + const mimeType = 'image/jpeg'; + + const result = getFormat(mimeType); + + expect(result).toEqual('jpeg'); + }); + + it('should throw error', () => { + const mimeType = 'image'; + + expect(() => getFormat(mimeType)).toThrowError(`could not get format from mime type: ${mimeType}`); + }); + }); + + describe('getPreviewName is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const outputFormat = PreviewOutputMimeTypes.IMAGE_WEBP; + + return { fileRecord, outputFormat }; + }; + + it('should return origin file name', () => { + const { fileRecord } = setup(); + + const result = getPreviewName(fileRecord, undefined); + + expect(result).toEqual(fileRecord.name); + }); + + it('should return preview name with format', () => { + const { fileRecord, outputFormat } = setup(); + + const result = getPreviewName(fileRecord, outputFormat); + + expect(result).toEqual(`${fileRecord.name.split('.')[0]}.webp`); + }); + }); }); diff --git a/apps/server/src/modules/files-storage/helper/file-record.ts b/apps/server/src/modules/files-storage/helper/file-record.ts index c0984063420..ed291661735 100644 --- a/apps/server/src/modules/files-storage/helper/file-record.ts +++ b/apps/server/src/modules/files-storage/helper/file-record.ts @@ -1,5 +1,7 @@ +import { InternalServerErrorException } from '@nestjs/common'; import { FileRecordParams } from '../controller/dto'; import { FileRecord } from '../entity'; +import { PreviewOutputMimeTypes } from '../interface'; export function markForDelete(fileRecords: FileRecord[]): FileRecord[] { const markedFileRecords = fileRecords.map((fileRecord) => { @@ -38,3 +40,26 @@ export function createFileRecord( return entity; } + +export function getFormat(mimeType: string): string { + const format = mimeType.split('/')[1]; + + if (!format) { + throw new InternalServerErrorException(`could not get format from mime type: ${mimeType}`); + } + + return format; +} + +export function getPreviewName(fileRecord: FileRecord, outputFormat?: PreviewOutputMimeTypes): string { + const { fileNameWithoutExtension, name } = fileRecord; + + if (!outputFormat) { + return name; + } + + const format = getFormat(outputFormat); + const previewFileName = `${fileNameWithoutExtension}.${format}`; + + return previewFileName; +} diff --git a/apps/server/src/modules/files-storage/index.ts b/apps/server/src/modules/files-storage/index.ts index c22e4c2be98..6ee0883938f 100644 --- a/apps/server/src/modules/files-storage/index.ts +++ b/apps/server/src/modules/files-storage/index.ts @@ -1,5 +1,2 @@ -export * from './files-storage-amqp.module'; -export * from './files-storage-api.module'; -export * from './files-storage-test.module'; // @deprecated remove after move api tests to modules -export * from './files-storage.config'; -export * from './files-storage.const'; +// this module has no exports +// it is an isolated module, it cannot be used in other modules diff --git a/apps/server/src/modules/files-storage/interface/interfaces.ts b/apps/server/src/modules/files-storage/interface/interfaces.ts index 047c943e55a..2f288f9133b 100644 --- a/apps/server/src/modules/files-storage/interface/interfaces.ts +++ b/apps/server/src/modules/files-storage/interface/interfaces.ts @@ -1,5 +1,5 @@ import { Readable } from 'stream'; -import type { DownloadFileParams, PreviewParams } from '../controller/dto'; +import type { PreviewParams } from '../controller/dto'; import { FileRecord } from '../entity'; export interface GetFileResponse { @@ -13,9 +13,10 @@ export interface GetFileResponse { export interface PreviewFileParams { fileRecord: FileRecord; - downloadParams: DownloadFileParams; previewParams: PreviewParams; hash: string; - filePath: string; + originFilePath: string; + previewFilePath: string; + format: string; bytesRange?: string; } diff --git a/apps/server/src/modules/files-storage/mapper/index.ts b/apps/server/src/modules/files-storage/mapper/index.ts index 556e7508929..bc8af7f7f05 100644 --- a/apps/server/src/modules/files-storage/mapper/index.ts +++ b/apps/server/src/modules/files-storage/mapper/index.ts @@ -3,3 +3,4 @@ export * from './file-dto.builder'; export * from './file-record.mapper'; export * from './file-response.builder'; export * from './files-storage.mapper'; +export * from './preview.builder'; diff --git a/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts b/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts new file mode 100644 index 00000000000..1a3cc843f86 --- /dev/null +++ b/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts @@ -0,0 +1,61 @@ +import { fileRecordFactory } from '@shared/testing'; +import { PreviewOutputMimeTypes } from '../interface'; +import { PreviewBuilder } from './preview.builder'; + +describe('PreviewBuilder', () => { + describe('buildParams is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const previewParams = { outputFormat: PreviewOutputMimeTypes.IMAGE_WEBP }; + const bytesRange = 'bytes=0-100'; + + const expectedResponse = { + fileRecord, + previewParams, + hash: expect.any(String), + previewFilePath: expect.any(String), + originFilePath: expect.any(String), + format: expect.any(String), + bytesRange, + }; + + return { fileRecord, previewParams, bytesRange, expectedResponse }; + }; + + it('should return preview file params', () => { + const { fileRecord, previewParams, bytesRange, expectedResponse } = setup(); + + const result = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); + + expect(result).toEqual(expectedResponse); + }); + }); + + describe('buildPayload is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const previewParams = { outputFormat: PreviewOutputMimeTypes.IMAGE_WEBP }; + const bytesRange = 'bytes=0-100'; + const previewFileParams = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); + + const expectedResponse = { + originFilePath: previewFileParams.originFilePath, + previewFilePath: previewFileParams.previewFilePath, + previewOptions: { + format: previewFileParams.format, + width: previewFileParams.previewParams.width, + }, + }; + + return { previewFileParams, expectedResponse }; + }; + + it('should return preview payload', () => { + const { previewFileParams, expectedResponse } = setup(); + + const result = PreviewBuilder.buildPayload(previewFileParams); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/modules/files-storage/mapper/preview.builder.ts b/apps/server/src/modules/files-storage/mapper/preview.builder.ts new file mode 100644 index 00000000000..83a16448a98 --- /dev/null +++ b/apps/server/src/modules/files-storage/mapper/preview.builder.ts @@ -0,0 +1,47 @@ +import { PreviewFileOptions } from '@shared/infra/preview-generator'; +import { PreviewParams } from '../controller/dto'; +import { FileRecord } from '../entity'; +import { createPath, createPreviewFilePath, createPreviewNameHash, getFormat } from '../helper'; +import { PreviewFileParams } from '../interface'; + +export class PreviewBuilder { + public static buildParams( + fileRecord: FileRecord, + previewParams: PreviewParams, + bytesRange: string | undefined + ): PreviewFileParams { + const { schoolId, id, mimeType } = fileRecord; + const originFilePath = createPath(schoolId, id); + const format = getFormat(previewParams.outputFormat ?? mimeType); + + const hash = createPreviewNameHash(id, previewParams); + const previewFilePath = createPreviewFilePath(schoolId, hash, id); + + const previewFileParams = { + fileRecord, + previewParams, + hash, + previewFilePath, + originFilePath, + format, + bytesRange, + }; + + return previewFileParams; + } + + public static buildPayload(params: PreviewFileParams): PreviewFileOptions { + const { originFilePath, previewFilePath, previewParams, format } = params; + + const payload = { + originFilePath, + previewFilePath, + previewOptions: { + format, + width: previewParams.width, + }, + }; + + return payload; + } +} diff --git a/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts b/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts index cd76b564b31..3705f93b51c 100644 --- a/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts @@ -171,35 +171,13 @@ describe('FilesStorageService delete methods', () => { return { parentId, fileRecords }; }; - it('should call findBySchoolIdAndParentId once with correct params', async () => { - const { parentId } = setup(); - - await service.deleteFilesOfParent(parentId); - - expect(fileRecordRepo.findByParentId).toHaveBeenNthCalledWith(1, parentId); - }); - it('should call delete with correct params', async () => { - const { parentId, fileRecords } = setup(); + const { fileRecords } = setup(); - await service.deleteFilesOfParent(parentId); + await service.deleteFilesOfParent(fileRecords); expect(service.delete).toHaveBeenCalledWith(fileRecords); }); - - it('should return file records and count', async () => { - const { parentId, fileRecords } = setup(); - - const responseData = await service.deleteFilesOfParent(parentId); - expect(responseData[0]).toEqual( - expect.arrayContaining([ - expect.objectContaining({ ...fileRecords[0] }), - expect.objectContaining({ ...fileRecords[1] }), - expect.objectContaining({ ...fileRecords[2] }), - ]) - ); - expect(responseData[1]).toEqual(fileRecords.length); - }); }); describe('WHEN no files exists', () => { @@ -215,43 +193,17 @@ describe('FilesStorageService delete methods', () => { const { parentId } = params; spy = jest.spyOn(service, 'delete'); - fileRecordRepo.findByParentId.mockResolvedValueOnce([fileRecords, fileRecords.length]); - return { parentId }; + return { parentId, fileRecords }; }; it('should not call delete', async () => { - const { parentId } = setup(); + const { fileRecords } = setup(); - await service.deleteFilesOfParent(parentId); + await service.deleteFilesOfParent(fileRecords); expect(service.delete).toHaveBeenCalledTimes(0); }); - - it('should return empty counted type', async () => { - const { parentId } = setup(); - - const result = await service.deleteFilesOfParent(parentId); - - expect(result).toEqual([[], 0]); - }); - }); - - describe('WHEN repository throw an error', () => { - const setup = () => { - const { params } = buildFileRecordsWithParams(); - const { parentId } = params; - - fileRecordRepo.findByParentId.mockRejectedValueOnce(new Error('bla')); - - return { parentId }; - }; - - it('should pass the error', async () => { - const { parentId } = setup(); - - await expect(service.deleteFilesOfParent(parentId)).rejects.toThrow(new Error('bla')); - }); }); describe('WHEN service.delete throw an error', () => { @@ -272,9 +224,9 @@ describe('FilesStorageService delete methods', () => { }; it('should pass the error', async () => { - const { parentId } = setup(); + const { fileRecords } = setup(); - await expect(service.deleteFilesOfParent(parentId)).rejects.toThrow(new Error('bla')); + await expect(service.deleteFilesOfParent(fileRecords)).rejects.toThrow(new Error('bla')); }); }); }); diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 8c0c85630de..209f1804d3e 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -244,7 +244,7 @@ export class FilesStorageService { } // download - private checkFileName(fileRecord: FileRecord, params: DownloadFileParams): void | NotFoundException { + public checkFileName(fileRecord: FileRecord, params: DownloadFileParams): void | NotFoundException { if (!fileRecord.hasName(params.fileName)) { this.logger.debug(`could not find file with id: ${fileRecord.id} by filename`); throw new NotFoundException(ErrorType.FILE_NOT_FOUND); @@ -304,14 +304,10 @@ export class FilesStorageService { await this.deleteWithRollbackByError(fileRecords); } - public async deleteFilesOfParent(parentId: EntityId): Promise> { - const [fileRecords, count] = await this.getFileRecordsOfParent(parentId); - - if (count > 0) { + public async deleteFilesOfParent(fileRecords: FileRecord[]): Promise { + if (fileRecords.length > 0) { await this.delete(fileRecords); } - - return [fileRecords, count]; } // restore diff --git a/apps/server/src/modules/files-storage/service/preview.service.spec.ts b/apps/server/src/modules/files-storage/service/preview.service.spec.ts index ed4592b97da..f02f48aee21 100644 --- a/apps/server/src/modules/files-storage/service/preview.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/preview.service.spec.ts @@ -2,33 +2,22 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; import { NotFoundException, UnprocessableEntityException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { Readable } from 'stream'; import { FileRecordParams } from '../controller/dto'; import { FileRecord, FileRecordParentType, ScanStatus } from '../entity'; import { ErrorType } from '../error'; import { FILES_STORAGE_S3_CONNECTION } from '../files-storage.config'; -import { createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; +import { createPath, createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; import { TestHelper } from '../helper/test-helper'; import { PreviewWidth } from '../interface'; import { PreviewOutputMimeTypes } from '../interface/preview-output-mime-types.enum'; -import { FileDtoBuilder, FileResponseBuilder } from '../mapper'; +import { FileResponseBuilder } from '../mapper'; import { FilesStorageService } from './files-storage.service'; import { PreviewService } from './preview.service'; -const streamMock = jest.fn(); -const resizeMock = jest.fn(); -const imageMagickMock = () => { - return { stream: streamMock, resize: resizeMock, data: Readable.from('text') }; -}; -jest.mock('gm', () => { - return { - subClass: () => imageMagickMock, - }; -}); - const buildFileRecordWithParams = (mimeType: string, scanStatus?: ScanStatus) => { const parentId = new ObjectId().toHexString(); const parentSchoolId = new ObjectId().toHexString(); @@ -62,8 +51,8 @@ const defaultPreviewParamsWithWidth = { describe('PreviewService', () => { let module: TestingModule; let previewService: PreviewService; - let fileStorageService: DeepMocked; let s3ClientAdapter: DeepMocked; + let previewProducer: DeepMocked; beforeAll(async () => { await setupEntities([FileRecord]); @@ -83,514 +72,369 @@ describe('PreviewService', () => { provide: LegacyLogger, useValue: createMock(), }, + { provide: PreviewProducer, useValue: createMock() }, ], }).compile(); previewService = module.get(PreviewService); - fileStorageService = module.get(FilesStorageService); s3ClientAdapter = module.get(FILES_STORAGE_S3_CONNECTION); - }); - - beforeEach(() => { - jest.resetAllMocks(); + previewProducer = module.get(PreviewProducer); }); afterAll(async () => { await module.close(); }); - describe('getPreview is called', () => { + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('download is called', () => { describe('WHEN preview is possbile', () => { describe('WHEN forceUpdate is true', () => { - describe('WHEN width and outputformat are not set', () => { - describe('WHEN download of original and preview file is successfull', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const orignalMimeType = 'image/png'; - const format = orignalMimeType.split('/')[1]; - const { fileRecord } = buildFileRecordWithParams(orignalMimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, {}); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, orignalMimeType); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file is successfull', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; - - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); - }); - - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(response).toEqual(previewFileResponse); - }); - }); - - describe('WHEN download of original file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const error = new Error('testError'); - fileStorageService.download.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); - - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); - }); - - describe('WHEN create of preview file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const error = new Error('testError'); - s3ClientAdapter.create.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; - }; + it('calls previewProducer.generate with correct params', async () => { + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = setup(); - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + await previewService.download(fileRecord, previewParams, bytesRange); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, }); }); - describe('WHEN get of preview file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; + it('calls S3ClientAdapters get method', async () => { + const { fileRecord, previewParams, previewPath } = setup(); - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); + await previewService.download(fileRecord, previewParams); - const error = new Error('testError'); - s3ClientAdapter.get.mockRejectedValueOnce(error); + expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + }); - return { fileRecord, downloadParams, previewParams, error }; - }; + it('returns preview file response', async () => { + const { fileRecord, previewParams, previewFileResponse } = setup(); - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + const response = await previewService.download(fileRecord, previewParams); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); + expect(response).toEqual(previewFileResponse); }); }); - describe('WHEN width and outputFormat are set', () => { - describe('WHEN download of original and preview file is successfull', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - forceUpdate: true, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, previewParams.outputFormat); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - resizeMock.mockClear(); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file throws error and second is successfull', () => { + const setup = (error: Error | NotFoundException) => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks resize method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(resizeMock).toHaveBeenCalledWith(previewParams.width, undefined, '>'); - expect(resizeMock).toHaveBeenCalledTimes(1); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); + describe('WHEN error is a NotFoundException', () => { + it('calls previewProducer.generate with correct params', async () => { + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = + setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, + }); + expect(previewProducer.generate).toHaveBeenCalledTimes(2); }); it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, previewPath } = setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(2); }); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + describe('WHEN error is other error', () => { + it('should pass error', async () => { + const error = new Error('testError'); + const { fileRecord, previewParams } = setup(error); - expect(response).toEqual(previewFileResponse); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(error); }); }); + }); - describe('WHEN download of original file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - - const error = new Error('testError'); - fileStorageService.download.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; + describe('WHEN both gets of preview file throw error', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + const notFoundException = new NotFoundException(); + s3ClientAdapter.get.mockRejectedValueOnce(notFoundException).mockRejectedValueOnce(notFoundException); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + it('should pass error', async () => { + const { fileRecord, previewParams } = setup(); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(); }); }); }); describe('WHEN forceUpdate is false', () => { - describe('WHEN width and outputFormat are set', () => { - describe('WHEN S3ClientAdapter get returns already stored preview file', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - resizeMock.mockClear(); - streamMock.mockClear(); - - return { - fileRecord, - downloadParams, - previewParams, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file is successfull', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: false, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + it('calls S3ClientAdapters get method', async () => { + const { fileRecord, previewParams, previewPath } = setup(); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); - expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); - }); + expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); + it('returns preview file response', async () => { + const { fileRecord, previewParams, previewFileResponse } = setup(); - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + const response = await previewService.download(fileRecord, previewParams); - expect(response).toEqual(previewFileResponse); - }); + expect(response).toEqual(previewFileResponse); + }); - it('does not call image magicks resize and stream method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); + it('does not call generate', async () => { + const { fileRecord, previewParams, bytesRange } = setup(); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(resizeMock).not.toHaveBeenCalled(); - expect(streamMock).not.toHaveBeenCalled(); - }); + expect(previewProducer.generate).toHaveBeenCalledTimes(0); }); + }); - describe('WHEN S3ClientAdapter get throws NotFoundException', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const error = new NotFoundException(); - s3ClientAdapter.get.mockRejectedValueOnce(error); - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, previewParams.outputFormat); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - resizeMock.mockClear(); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file throws error and second is successfull', () => { + const setup = (error: Error | NotFoundException) => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: false, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks resize method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(resizeMock).toHaveBeenCalledWith(previewParams.width, undefined, '>'); - expect(resizeMock).toHaveBeenCalledTimes(1); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); + describe('WHEN error is a NotFoundException', () => { + it('calls previewProducer.generate with correct params', async () => { + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = + setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, + }); + expect(previewProducer.generate).toHaveBeenCalledTimes(1); }); it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, previewPath } = setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); expect(s3ClientAdapter.get).toHaveBeenCalledTimes(2); }); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + describe('WHEN error is other error', () => { + it('should pass error', async () => { + const error = new Error('testError'); + const { fileRecord, previewParams } = setup(error); - expect(response).toEqual(previewFileResponse); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(error); }); }); + }); - describe('WHEN S3ClientAdapter get throws other than NotFoundException', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const error = new Error('testError'); - s3ClientAdapter.get.mockRejectedValueOnce(error); - - return { - fileRecord, - downloadParams, - previewParams, - format, - error, - }; + describe('WHEN both gets of preview file throw error', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + const notFoundException = new NotFoundException(); + s3ClientAdapter.get.mockRejectedValueOnce(notFoundException).mockRejectedValueOnce(notFoundException); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + it('should pass error', async () => { + const { fileRecord, previewParams } = setup(); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrow(error); - }); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(); }); }); }); @@ -603,33 +447,23 @@ describe('PreviewService', () => { const mimeType = 'application/zip'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, }; }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + it('passes error', async () => { + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -639,33 +473,23 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.PENDING); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, }; }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + it('passes error', async () => { + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -675,21 +499,14 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.ERROR); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); + const previewParams = { ...defaultPreviewParams, forceUpdate: true }; const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, @@ -697,11 +514,9 @@ describe('PreviewService', () => { }; it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -711,21 +526,13 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.BLOCKED); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, @@ -733,11 +540,9 @@ describe('PreviewService', () => { }; it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); }); @@ -789,10 +594,10 @@ describe('PreviewService', () => { }; }; - it('does not pass error', async () => { - const { fileRecord } = setup(); + it('should throw error', async () => { + const { fileRecord, error } = setup(); - await previewService.deletePreviews([fileRecord]); + await expect(previewService.deletePreviews([fileRecord])).rejects.toThrowError(error); }); }); }); diff --git a/apps/server/src/modules/files-storage/service/preview.service.ts b/apps/server/src/modules/files-storage/service/preview.service.ts index 5ca2093351b..e27fbc0645a 100644 --- a/apps/server/src/modules/files-storage/service/preview.service.ts +++ b/apps/server/src/modules/files-storage/service/preview.service.ts @@ -1,64 +1,45 @@ import { Inject, Injectable, NotFoundException, UnprocessableEntityException } from '@nestjs/common'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { LegacyLogger } from '@src/core/logger'; -import { subClass } from 'gm'; -import { PassThrough } from 'stream'; -import { DownloadFileParams, PreviewParams } from '../controller/dto'; +import { PreviewParams } from '../controller/dto'; import { FileRecord, PreviewStatus } from '../entity'; import { ErrorType } from '../error'; import { FILES_STORAGE_S3_CONNECTION } from '../files-storage.config'; -import { createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; +import { createPreviewDirectoryPath, getPreviewName } from '../helper'; import { GetFileResponse, PreviewFileParams } from '../interface'; -import { PreviewOutputMimeTypes } from '../interface/preview-output-mime-types.enum'; -import { FileDtoBuilder, FileResponseBuilder } from '../mapper'; -import { FilesStorageService } from './files-storage.service'; +import { FileResponseBuilder, PreviewBuilder } from '../mapper'; @Injectable() export class PreviewService { constructor( @Inject(FILES_STORAGE_S3_CONNECTION) private readonly storageClient: S3ClientAdapter, - private readonly fileStorageService: FilesStorageService, - private logger: LegacyLogger + private logger: LegacyLogger, + private readonly previewProducer: PreviewProducer ) { this.logger.setContext(PreviewService.name); } - public async getPreview( + public async download( fileRecord: FileRecord, - downloadParams: DownloadFileParams, previewParams: PreviewParams, bytesRange?: string ): Promise { this.checkIfPreviewPossible(fileRecord); - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const filePath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const previewFileParams = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); - let response: GetFileResponse; - - const previewFileParams = { fileRecord, downloadParams, previewParams, hash, filePath, bytesRange }; - - if (previewParams.forceUpdate) { - response = await this.generatePreview(previewFileParams); - } else { - response = await this.tryGetPreviewOrGenerate(previewFileParams); - } + const response = await this.tryGetPreviewOrGenerate(previewFileParams); return response; } public async deletePreviews(fileRecords: FileRecord[]): Promise { - try { - const paths = fileRecords.map((fileRecord) => - createPreviewDirectoryPath(fileRecord.getSchoolId(), fileRecord.id) - ); + const paths = fileRecords.map((fileRecord) => createPreviewDirectoryPath(fileRecord.getSchoolId(), fileRecord.id)); - const promises = paths.map((path) => this.storageClient.deleteDirectory(path)); + const promises = paths.map((path) => this.storageClient.deleteDirectory(path)); - await Promise.all(promises); - } catch (error) { - this.logger.warn(error); - } + await Promise.all(promises); } private checkIfPreviewPossible(fileRecord: FileRecord): void | UnprocessableEntityException { @@ -72,79 +53,36 @@ export class PreviewService { let file: GetFileResponse; try { + if (params.previewParams.forceUpdate) { + await this.generatePreview(params); + } + file = await this.getPreviewFile(params); } catch (error) { if (!(error instanceof NotFoundException)) { throw error; } - file = await this.generatePreview(params); + await this.generatePreview(params); + file = await this.getPreviewFile(params); } return file; } private async getPreviewFile(params: PreviewFileParams): Promise { - const { fileRecord, filePath, bytesRange, previewParams } = params; - const name = this.getPreviewName(fileRecord, previewParams.outputFormat); - const file = await this.storageClient.get(filePath, bytesRange); + const { fileRecord, previewFilePath, bytesRange, previewParams } = params; + const name = getPreviewName(fileRecord, previewParams.outputFormat); + const file = await this.storageClient.get(previewFilePath, bytesRange); const response = FileResponseBuilder.build(file, name); return response; } - private async generatePreview(params: PreviewFileParams): Promise { - const { fileRecord, downloadParams, previewParams, hash, filePath, bytesRange } = params; - - const original = await this.fileStorageService.download(fileRecord, downloadParams, bytesRange); - const preview = this.resizeAndConvert(original, fileRecord, previewParams); - - const format = previewParams.outputFormat ?? fileRecord.mimeType; - const fileDto = FileDtoBuilder.build(hash, preview, format); - await this.storageClient.create(filePath, fileDto); - - const response = await this.getPreviewFile(params); - - return response; - } - - private resizeAndConvert( - original: GetFileResponse, - fileRecord: FileRecord, - previewParams: PreviewParams - ): PassThrough { - const mimeType = previewParams.outputFormat ?? fileRecord.mimeType; - const format = this.getFormat(mimeType); - const im = subClass({ imageMagick: '7+' }); - - const preview = im(original.data, fileRecord.name); - const { width } = previewParams; - - if (width) { - preview.resize(width, undefined, '>'); - } - - const result = preview.stream(format); - - return result; - } - - private getFormat(mimeType: string): string { - const format = mimeType.split('/')[1]; - - return format; - } - - private getPreviewName(fileRecord: FileRecord, outputFormat?: PreviewOutputMimeTypes): string { - if (!outputFormat) { - return fileRecord.name; - } - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const format = this.getFormat(outputFormat); - const name = `${fileNameWithoutExtension}.${format}`; + private async generatePreview(params: PreviewFileParams): Promise { + const payload = PreviewBuilder.buildPayload(params); - return name; + await this.previewProducer.generate(payload); } } diff --git a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts index a1aaf0342ee..b12006367aa 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts @@ -1,5 +1,6 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { ForbiddenException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; @@ -8,7 +9,6 @@ import { AntivirusService } from '@shared/infra/antivirus'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { FileRecordParams } from '../controller/dto'; import { FileRecord, FileRecordParentType } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -123,7 +123,7 @@ describe('FilesStorageUC delete methods', () => { const mockedResult = [[fileRecord], 0] as Counted; authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); - filesStorageService.deleteFilesOfParent.mockResolvedValueOnce(mockedResult); + filesStorageService.getFileRecordsOfParent.mockResolvedValueOnce(mockedResult); return { params, userId, mockedResult, requestParams, fileRecord }; }; @@ -143,11 +143,11 @@ describe('FilesStorageUC delete methods', () => { }); it('should call service with correct params', async () => { - const { requestParams, userId } = setup(); + const { requestParams, userId, fileRecord } = setup(); await filesStorageUC.deleteFilesOfParent(userId, requestParams); - expect(filesStorageService.deleteFilesOfParent).toHaveBeenCalledWith(requestParams.parentId); + expect(filesStorageService.deleteFilesOfParent).toHaveBeenCalledWith([fileRecord]); }); it('should call deletePreviews', async () => { @@ -189,10 +189,14 @@ describe('FilesStorageUC delete methods', () => { describe('WHEN service throws error', () => { const setup = () => { + const { fileRecords } = buildFileRecordsWithParams(); + + authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); const { requestParams, userId } = createParams(); const error = new Error('test'); authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); + filesStorageService.getFileRecordsOfParent.mockResolvedValueOnce([fileRecords, fileRecords.length]); filesStorageService.deleteFilesOfParent.mockRejectedValueOnce(error); return { requestParams, userId, error }; diff --git a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts index 81b54553d1e..f0aa9dcc25a 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts @@ -111,7 +111,7 @@ describe('FilesStorageUC', () => { const previewFileResponse = TestHelper.createFileResponse(); filesStorageService.getFileRecord.mockResolvedValueOnce(fileRecord); - previewService.getPreview.mockResolvedValueOnce(previewFileResponse); + previewService.download.mockResolvedValueOnce(previewFileResponse); return { fileDownloadParams, previewParams, userId, fileRecord, singleFileParams, previewFileResponse }; }; @@ -129,12 +129,7 @@ describe('FilesStorageUC', () => { await filesStorageUC.downloadPreview(userId, fileDownloadParams, previewParams); - expect(previewService.getPreview).toHaveBeenCalledWith( - fileRecord, - fileDownloadParams, - previewParams, - undefined - ); + expect(previewService.download).toHaveBeenCalledWith(fileRecord, previewParams, undefined); }); it('should call checkPermission with correct params', async () => { @@ -211,7 +206,7 @@ describe('FilesStorageUC', () => { filesStorageService.getFileRecord.mockResolvedValueOnce(fileRecord); const error = new Error('test'); - previewService.getPreview.mockRejectedValueOnce(error); + previewService.download.mockRejectedValueOnce(error); return { fileDownloadParams, previewParams, userId, error }; }; diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 833d7575bdf..47eb7c79104 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -1,9 +1,9 @@ +import { AuthorizationContext } from '@modules/authorization'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { Injectable, NotFoundException } from '@nestjs/common'; import { Counted, EntityId } from '@shared/domain'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationContext } from '@modules/authorization'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { AxiosRequestConfig, AxiosResponse } from 'axios'; import busboy from 'busboy'; import { Request } from 'express'; @@ -154,7 +154,9 @@ export class FilesStorageUC { await this.checkPermission(userId, parentType, parentId, FileStorageAuthorizationContext.read); - const result = this.previewService.getPreview(fileRecord, params, previewParams, bytesRange); + this.filesStorageService.checkFileName(fileRecord, params); + + const result = this.previewService.download(fileRecord, previewParams, bytesRange); return result; } @@ -162,8 +164,9 @@ export class FilesStorageUC { // delete public async deleteFilesOfParent(userId: EntityId, params: FileRecordParams): Promise> { await this.checkPermission(userId, params.parentType, params.parentId, FileStorageAuthorizationContext.delete); - const [fileRecords, count] = await this.filesStorageService.deleteFilesOfParent(params.parentId); + const [fileRecords, count] = await this.filesStorageService.getFileRecordsOfParent(params.parentId); await this.previewService.deletePreviews(fileRecords); + await this.filesStorageService.deleteFilesOfParent(fileRecords); return [fileRecords, count]; } @@ -173,8 +176,8 @@ export class FilesStorageUC { const { parentType, parentId } = fileRecord.getParentInfo(); await this.checkPermission(userId, parentType, parentId, FileStorageAuthorizationContext.delete); - await this.filesStorageService.delete([fileRecord]); await this.previewService.deletePreviews([fileRecord]); + await this.filesStorageService.delete([fileRecord]); return fileRecord; } diff --git a/apps/server/src/shared/infra/preview-generator/index.ts b/apps/server/src/shared/infra/preview-generator/index.ts new file mode 100644 index 00000000000..570b38cc9bd --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/index.ts @@ -0,0 +1,4 @@ +export * from './interface'; +export * from './preview-generator-consumer.module'; +export * from './preview-generator-producer.module'; +export * from './preview.producer'; diff --git a/apps/server/src/shared/infra/preview-generator/interface/index.ts b/apps/server/src/shared/infra/preview-generator/interface/index.ts new file mode 100644 index 00000000000..37aae418ee2 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/index.ts @@ -0,0 +1 @@ +export * from './preview'; diff --git a/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts b/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts new file mode 100644 index 00000000000..2924fc945bc --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts @@ -0,0 +1,11 @@ +import { S3Config } from '@shared/infra/s3-client'; + +export interface PreviewModuleConfig { + NEST_LOG_LEVEL: string; + INCOMING_REQUEST_TIMEOUT: number; +} + +export interface PreviewConfig { + storageConfig: S3Config; + serverConfig: PreviewModuleConfig; +} diff --git a/apps/server/src/shared/infra/preview-generator/interface/preview.ts b/apps/server/src/shared/infra/preview-generator/interface/preview.ts new file mode 100644 index 00000000000..92ab2808151 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/preview.ts @@ -0,0 +1,15 @@ +export interface PreviewOptions { + format: string; + width?: number; +} + +export interface PreviewFileOptions { + originFilePath: string; + previewFilePath: string; + previewOptions: PreviewOptions; +} + +export interface PreviewResponseMessage { + previewFilePath: string; + status: boolean; +} diff --git a/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts new file mode 100644 index 00000000000..04d56a991f1 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts @@ -0,0 +1,37 @@ +import { PreviewActionsLoggable } from './preview-actions.loggable'; + +describe('PreviewActionsLoggable', () => { + describe('getLogMessage is called', () => { + const setup = () => { + const message = 'message'; + const payload = { + originFilePath: 'originFilePath', + previewFilePath: 'previewFilePath', + previewOptions: { + format: 'webp', + width: 100, + }, + }; + + const expectedResponse = { + message, + data: { + originFilePath: payload.originFilePath, + previewFilePath: payload.previewFilePath, + format: payload.previewOptions.format, + width: payload.previewOptions.width, + }, + }; + + return { message, payload, expectedResponse }; + }; + + it('should return log message', () => { + const { message, payload, expectedResponse } = setup(); + + const result = new PreviewActionsLoggable(message, payload).getLogMessage(); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts new file mode 100644 index 00000000000..e98f21d09be --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts @@ -0,0 +1,19 @@ +import { LogMessage, Loggable } from '@src/core/logger'; +import { PreviewFileOptions } from '../interface'; + +export class PreviewActionsLoggable implements Loggable { + constructor(private readonly message: string, private readonly payload: PreviewFileOptions) {} + + getLogMessage(): LogMessage { + const { originFilePath, previewFilePath, previewOptions } = this.payload; + return { + message: this.message, + data: { + originFilePath, + previewFilePath, + format: previewOptions.format, + width: previewOptions.width, + }, + }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts b/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts new file mode 100644 index 00000000000..9d352b81d9d --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts @@ -0,0 +1,36 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { RabbitMQWrapperModule } from '@shared/infra/rabbitmq'; +import { S3ClientAdapter, S3ClientModule } from '@shared/infra/s3-client'; +import { createConfigModuleOptions } from '@src/config'; +import { Logger, LoggerModule } from '@src/core/logger'; +import { PreviewConfig } from './interface/preview-consumer-config'; +import { PreviewGeneratorConsumer } from './preview-generator.consumer'; +import { PreviewGeneratorService } from './preview-generator.service'; + +@Module({}) +export class PreviewGeneratorConsumerModule { + static register(config: PreviewConfig): DynamicModule { + const { storageConfig, serverConfig } = config; + const providers = [ + { + provide: PreviewGeneratorService, + useFactory: (logger: Logger, storageClient: S3ClientAdapter) => + new PreviewGeneratorService(storageClient, logger), + inject: [Logger, storageConfig.connectionName], + }, + PreviewGeneratorConsumer, + ]; + + return { + module: PreviewGeneratorConsumerModule, + imports: [ + LoggerModule, + S3ClientModule.register([storageConfig]), + RabbitMQWrapperModule, + ConfigModule.forRoot(createConfigModuleOptions(() => serverConfig)), + ], + providers, + }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts b/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts new file mode 100644 index 00000000000..d3f65299657 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { LoggerModule } from '@src/core/logger'; +import { RabbitMQWrapperModule } from '../rabbitmq'; +import { PreviewProducer } from './preview.producer'; + +@Module({ + imports: [LoggerModule, RabbitMQWrapperModule], + providers: [PreviewProducer], + exports: [PreviewProducer], +}) +export class PreviewGeneratorProducerModule {} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts new file mode 100644 index 00000000000..2caaad7abe8 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts @@ -0,0 +1,28 @@ +import { PassThrough } from 'stream'; +import { PreviewGeneratorBuilder } from './preview-generator.builder'; + +describe('PreviewGeneratorBuilder', () => { + describe('buildFile is called', () => { + const setup = () => { + const preview = new PassThrough(); + const previewOptions = { + format: 'webp', + }; + + const expectedResponse = { + data: preview, + mimeType: previewOptions.format, + }; + + return { preview, previewOptions, expectedResponse }; + }; + + it('should return preview file', () => { + const { preview, previewOptions, expectedResponse } = setup(); + + const result = PreviewGeneratorBuilder.buildFile(preview, previewOptions); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts new file mode 100644 index 00000000000..4c5561ed089 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts @@ -0,0 +1,16 @@ +import { File } from '@shared/infra/s3-client'; +import { PassThrough } from 'stream'; +import { PreviewOptions } from './interface'; + +export class PreviewGeneratorBuilder { + public static buildFile(preview: PassThrough, previewOptions: PreviewOptions): File { + const { format } = previewOptions; + + const file = { + data: preview, + mimeType: format, + }; + + return file; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts new file mode 100644 index 00000000000..b1a24a30a57 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts @@ -0,0 +1,80 @@ +import { DeepMocked, createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { Logger } from '@src/core/logger'; +import { PreviewFileOptions, PreviewResponseMessage } from './interface'; +import { PreviewGeneratorConsumer } from './preview-generator.consumer'; +import { PreviewGeneratorService } from './preview-generator.service'; + +describe('PreviewGeneratorConsumer', () => { + let module: TestingModule; + let previewGeneratorService: DeepMocked; + let service: PreviewGeneratorConsumer; + + beforeAll(async () => { + module = await Test.createTestingModule({ + providers: [ + PreviewGeneratorConsumer, + { + provide: PreviewGeneratorService, + useValue: createMock(), + }, + { + provide: Logger, + useValue: createMock(), + }, + ], + }).compile(); + + previewGeneratorService = module.get(PreviewGeneratorService); + service = module.get(PreviewGeneratorConsumer); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + afterAll(async () => { + await module.close(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generatePreview()', () => { + const setup = () => { + const payload: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const response: PreviewResponseMessage = { + previewFilePath: payload.previewFilePath, + status: true, + }; + previewGeneratorService.generatePreview.mockResolvedValueOnce(response); + + return { payload, response }; + }; + + it('should call previewGeneratorService.generatePreview with payload', async () => { + const { payload } = setup(); + + await service.generatePreview(payload); + + expect(previewGeneratorService.generatePreview).toBeCalledWith(payload); + }); + + it('should return expected value', async () => { + const { payload, response } = setup(); + + const result = await service.generatePreview(payload); + + expect(result).toEqual({ message: response }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts new file mode 100644 index 00000000000..8fc08d261f3 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts @@ -0,0 +1,27 @@ +import { RabbitPayload, RabbitRPC } from '@golevelup/nestjs-rabbitmq'; +import { Injectable } from '@nestjs/common'; +import { Logger } from '@src/core/logger'; +import { FilesPreviewEvents, FilesPreviewExchange } from '@src/shared/infra/rabbitmq'; +import { PreviewFileOptions } from './interface'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; +import { PreviewGeneratorService } from './preview-generator.service'; + +@Injectable() +export class PreviewGeneratorConsumer { + constructor(private readonly previewGeneratorService: PreviewGeneratorService, private logger: Logger) { + this.logger.setContext(PreviewGeneratorConsumer.name); + } + + @RabbitRPC({ + exchange: FilesPreviewExchange, + routingKey: FilesPreviewEvents.GENERATE_PREVIEW, + queue: FilesPreviewEvents.GENERATE_PREVIEW, + }) + public async generatePreview(@RabbitPayload() payload: PreviewFileOptions) { + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorConsumer.generatePreview', payload)); + + const response = await this.previewGeneratorService.generatePreview(payload); + + return { message: response }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts new file mode 100644 index 00000000000..b8eeea612f5 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts @@ -0,0 +1,151 @@ +import { DeepMocked, createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { GetFile, S3ClientAdapter } from '@shared/infra/s3-client'; +import { Logger } from '@src/core/logger'; +import { Readable } from 'node:stream'; +import { PreviewGeneratorService } from './preview-generator.service'; + +const streamMock = jest.fn(); +const resizeMock = jest.fn(); +const imageMagickMock = () => { + return { stream: streamMock, resize: resizeMock, data: Readable.from('text') }; +}; +jest.mock('gm', () => { + return { + subClass: () => imageMagickMock, + }; +}); + +const createFile = (contentRange?: string): GetFile => { + const text = 'testText'; + const readable = Readable.from(text); + + const fileResponse = { + data: readable, + contentType: 'image/jpeg', + contentLength: text.length, + contentRange, + etag: 'testTag', + }; + + return fileResponse; +}; + +describe('PreviewGeneratorService', () => { + let module: TestingModule; + let service: PreviewGeneratorService; + let s3ClientAdapter: DeepMocked; + + beforeAll(async () => { + module = await Test.createTestingModule({ + providers: [ + PreviewGeneratorService, + { + provide: S3ClientAdapter, + useValue: createMock(), + }, + { + provide: Logger, + useValue: createMock(), + }, + ], + }).compile(); + + service = module.get(PreviewGeneratorService); + s3ClientAdapter = module.get(S3ClientAdapter); + }); + + afterAll(async () => { + await module.close(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generatePreview', () => { + const setup = (width = 500) => { + const params = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width, + }, + }; + const originFile = createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(originFile); + + const data = Readable.from('text'); + streamMock.mockReturnValueOnce(data); + + const expectedFileData = { + data, + mimeType: params.previewOptions.format, + }; + + return { params, originFile, expectedFileData }; + }; + + describe('WHEN download of original and preview file is successful', () => { + it('should call storageClient get method with originFilePath', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(s3ClientAdapter.get).toBeCalledWith(params.originFilePath); + }); + + it('should call imagemagicks resize method', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(resizeMock).toHaveBeenCalledWith(params.previewOptions.width, undefined, '>'); + expect(resizeMock).toHaveBeenCalledTimes(1); + }); + + it('should call imagemagicks stream method', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(streamMock).toHaveBeenCalledWith(params.previewOptions.format); + expect(streamMock).toHaveBeenCalledTimes(1); + }); + + it('should call S3ClientAdapters create method', async () => { + const { params, expectedFileData } = setup(); + + await service.generatePreview(params); + + expect(s3ClientAdapter.create).toHaveBeenCalledWith(params.previewFilePath, expectedFileData); + expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + }); + + it('should should return values', async () => { + const { params } = setup(); + const expectedValue = { previewFilePath: params.previewFilePath, status: true }; + + const result = await service.generatePreview(params); + + expect(result).toEqual(expectedValue); + }); + }); + + describe('WHEN previewParams.width not set', () => { + it('should not call imagemagicks resize method', async () => { + const { params } = setup(0); + + await service.generatePreview(params); + + expect(resizeMock).not.toHaveBeenCalledWith(params.previewOptions.width, undefined, '>'); + expect(resizeMock).not.toHaveBeenCalledTimes(1); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts new file mode 100644 index 00000000000..72dac25f076 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts @@ -0,0 +1,56 @@ +import { Injectable } from '@nestjs/common'; +import { GetFile, S3ClientAdapter } from '@shared/infra/s3-client'; +import { Logger } from '@src/core/logger'; +import { subClass } from 'gm'; +import { PassThrough } from 'stream'; +import { PreviewFileOptions, PreviewOptions, PreviewResponseMessage } from './interface'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; +import { PreviewGeneratorBuilder } from './preview-generator.builder'; + +@Injectable() +export class PreviewGeneratorService { + private imageMagick = subClass({ imageMagick: '7+' }); + + constructor(private readonly storageClient: S3ClientAdapter, private logger: Logger) { + this.logger.setContext(PreviewGeneratorService.name); + } + + public async generatePreview(params: PreviewFileOptions): Promise { + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorService.generatePreview:start', params)); + const { originFilePath, previewFilePath, previewOptions } = params; + + const original = await this.downloadOriginFile(originFilePath); + const preview = this.resizeAndConvert(original, previewOptions); + + const file = PreviewGeneratorBuilder.buildFile(preview, params.previewOptions); + + await this.storageClient.create(previewFilePath, file); + + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorService.generatePreview:end', params)); + + return { + previewFilePath, + status: true, + }; + } + + private async downloadOriginFile(pathToFile: string): Promise { + const file = await this.storageClient.get(pathToFile); + + return file; + } + + private resizeAndConvert(original: GetFile, previewParams: PreviewOptions): PassThrough { + const { format, width } = previewParams; + + const preview = this.imageMagick(original.data); + + if (width) { + preview.resize(width, undefined, '>'); + } + + const result = preview.stream(format); + + return result; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts b/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts new file mode 100644 index 00000000000..47adea158a6 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts @@ -0,0 +1,128 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { InternalServerErrorException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; +import { setupEntities } from '@shared/testing'; +import { Logger } from '@src/core/logger'; +import { ErrorMapper, FilesPreviewEvents, FilesPreviewExchange } from '../rabbitmq'; +import { PreviewFileOptions } from './interface'; +import { PreviewProducer } from './preview.producer'; + +describe('PreviewProducer', () => { + let module: TestingModule; + let service: PreviewProducer; + let configService: DeepMocked; + let amqpConnection: DeepMocked; + + beforeAll(async () => { + await setupEntities(); + module = await Test.createTestingModule({ + providers: [ + PreviewProducer, + { + provide: Logger, + useValue: createMock(), + }, + { + provide: AmqpConnection, + useValue: createMock(), + }, + { + provide: ConfigService, + useValue: createMock(), + }, + ], + }).compile(); + + service = module.get(PreviewProducer); + amqpConnection = module.get(AmqpConnection); + configService = module.get(ConfigService); + }); + + afterAll(async () => { + await module.close(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generate', () => { + describe('when valid params are passed and amqp connection return with a message', () => { + const setup = () => { + const timeout = 10000; + + const params: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const message = []; + amqpConnection.request.mockResolvedValueOnce({ message }); + configService.get.mockReturnValue(timeout); + + const expectedParams = { + exchange: FilesPreviewExchange, + routingKey: FilesPreviewEvents.GENERATE_PREVIEW, + payload: params, + timeout, + }; + + return { params, expectedParams, message }; + }; + + it('should call the ampqConnection.', async () => { + const { params, expectedParams } = setup(); + + await service.generate(params); + + expect(amqpConnection.request).toHaveBeenCalledWith(expectedParams); + }); + + it('should return the response message.', async () => { + const { params, message } = setup(); + + const res = await service.generate(params); + + expect(res).toEqual(message); + }); + }); + + describe('when amqpConnection return with error in response', () => { + const setup = () => { + const params: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockResolvedValueOnce({ error }); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.generate(params)).rejects.toThrowError( + new InternalServerErrorException(null, { cause: error }) + ); + expect(spy).toBeCalled(); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview.producer.ts b/apps/server/src/shared/infra/preview-generator/preview.producer.ts new file mode 100644 index 00000000000..602e2503185 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview.producer.ts @@ -0,0 +1,31 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { FilesPreviewEvents, FilesPreviewExchange, RpcMessageProducer } from '@shared/infra/rabbitmq'; +import { Logger } from '@src/core/logger'; +import { PreviewFileOptions, PreviewResponseMessage } from './interface'; +import { PreviewModuleConfig } from './interface/preview-consumer-config'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; + +@Injectable() +export class PreviewProducer extends RpcMessageProducer { + constructor( + protected readonly amqpConnection: AmqpConnection, + private readonly logger: Logger, + protected readonly configService: ConfigService + ) { + const timeout = configService.get('INCOMING_REQUEST_TIMEOUT'); + + super(amqpConnection, FilesPreviewExchange, timeout); + this.logger.setContext(PreviewProducer.name); + } + + async generate(payload: PreviewFileOptions): Promise { + this.logger.debug(new PreviewActionsLoggable('PreviewProducer.generate:started', payload)); + const response = await this.request(FilesPreviewEvents.GENERATE_PREVIEW, payload); + + this.logger.debug(new PreviewActionsLoggable('PreviewProducer.generate:finished', payload)); + + return response; + } +} diff --git a/apps/server/src/modules/files-storage-client/mapper/error.mapper.spec.ts b/apps/server/src/shared/infra/rabbitmq/error.mapper.spec.ts similarity index 100% rename from apps/server/src/modules/files-storage-client/mapper/error.mapper.spec.ts rename to apps/server/src/shared/infra/rabbitmq/error.mapper.spec.ts diff --git a/apps/server/src/modules/files-storage-client/mapper/error.mapper.ts b/apps/server/src/shared/infra/rabbitmq/error.mapper.ts similarity index 100% rename from apps/server/src/modules/files-storage-client/mapper/error.mapper.ts rename to apps/server/src/shared/infra/rabbitmq/error.mapper.ts diff --git a/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts b/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts new file mode 100644 index 00000000000..0bab7491e07 --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts @@ -0,0 +1,7 @@ +import { Configuration } from '@hpi-schul-cloud/commons/lib'; + +export const FilesPreviewExchange = Configuration.get('FILES_STORAGE__EXCHANGE') as string; + +export enum FilesPreviewEvents { + 'GENERATE_PREVIEW' = 'generate-preview', +} diff --git a/apps/server/src/shared/infra/rabbitmq/exchange/index.ts b/apps/server/src/shared/infra/rabbitmq/exchange/index.ts index f0d84c9e1a1..0cf6bd00d13 100644 --- a/apps/server/src/shared/infra/rabbitmq/exchange/index.ts +++ b/apps/server/src/shared/infra/rabbitmq/exchange/index.ts @@ -1 +1,2 @@ +export * from './files-preview'; export * from './files-storage'; diff --git a/apps/server/src/shared/infra/rabbitmq/index.ts b/apps/server/src/shared/infra/rabbitmq/index.ts index 0183c37b284..99f5887b9a8 100644 --- a/apps/server/src/shared/infra/rabbitmq/index.ts +++ b/apps/server/src/shared/infra/rabbitmq/index.ts @@ -1,3 +1,5 @@ +export * from './error.mapper'; export * from './exchange'; export * from './rabbitmq.module'; export * from './rpc-message'; +export * from './rpc-message-producer'; diff --git a/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts b/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts index 2d73162e6e3..946b642e779 100644 --- a/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts +++ b/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts @@ -1,7 +1,7 @@ import { AmqpConnectionManager, RabbitMQModule } from '@golevelup/nestjs-rabbitmq'; import { Configuration } from '@hpi-schul-cloud/commons'; import { Global, Module, OnModuleDestroy } from '@nestjs/common'; -import { FilesStorageExchange } from './exchange'; +import { FilesPreviewExchange, FilesStorageExchange } from './exchange'; /** * https://www.npmjs.com/package/@golevelup/nestjs-rabbitmq#usage @@ -28,6 +28,10 @@ const imports = [ name: FilesStorageExchange, type: 'direct', }, + { + name: FilesPreviewExchange, + type: 'direct', + }, ], uri: Configuration.get('RABBITMQ_URI') as string, }), diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts new file mode 100644 index 00000000000..b2e94ea676b --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts @@ -0,0 +1,130 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { ErrorMapper, RpcMessageProducer } from '.'; + +interface TestPayload { + value: boolean; +} + +interface TestResponse { + value: boolean; +} + +const TestEvent = 'test-event'; +const TestExchange = 'test-exchange'; +const timeout = 1000; + +class RpcMessageProducerImp extends RpcMessageProducer { + constructor(protected readonly amqpConnection: AmqpConnection) { + super(amqpConnection, TestExchange, timeout); + } + + async testRequest(payload: TestPayload): Promise { + const response = await this.request(TestEvent, payload); + + return response; + } +} + +describe('RpcMessageProducer', () => { + let service: RpcMessageProducerImp; + let amqpConnection: DeepMocked; + + beforeAll(() => { + amqpConnection = createMock(); + + service = new RpcMessageProducerImp(amqpConnection); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generate', () => { + describe('when valid params are passed and amqp connection return with a message', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const message = []; + amqpConnection.request.mockResolvedValueOnce({ message }); + + const expectedParams = { + exchange: TestExchange, + routingKey: TestEvent, + payload: params, + timeout, + }; + + return { params, expectedParams, message }; + }; + + it('should call the ampqConnection.', async () => { + const { params, expectedParams } = setup(); + + await service.testRequest(params); + + expect(amqpConnection.request).toHaveBeenCalledWith(expectedParams); + }); + + it('should return the response message.', async () => { + const { params, message } = setup(); + + const res = await service.testRequest(params); + + expect(res).toEqual(message); + }); + }); + + describe('when amqpConnection return with error in response', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockResolvedValueOnce({ error }); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.testRequest(params)).rejects.toThrowError( + ErrorMapper.mapRpcErrorResponseToDomainError(error) + ); + expect(spy).toBeCalled(); + }); + }); + + describe('when amqpConnection throw an error', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockRejectedValueOnce(error); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.testRequest(params)).rejects.toThrowError(error); + expect(spy).not.toBeCalled(); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts new file mode 100644 index 00000000000..8a239b2cbcd --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts @@ -0,0 +1,37 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { ErrorMapper } from './error.mapper'; +import { RpcMessage } from './rpc-message'; + +export abstract class RpcMessageProducer { + constructor( + protected readonly amqpConnection: AmqpConnection, + protected readonly exchange: string, + protected readonly timeout: number + ) {} + + protected async request(event: string, payload: unknown) { + const response = await this.amqpConnection.request>(this.createRequest(event, payload)); + + this.checkError(response); + return response.message; + } + + // need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984 + // mapRpcErrorResponseToDomainError should also removed with this ticket + protected checkError(response: RpcMessage) { + const { error } = response; + if (error) { + const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error); + throw domainError; + } + } + + protected createRequest(event: string, payload: unknown) { + return { + exchange: this.exchange, + routingKey: event, + payload, + timeout: this.timeout, + }; + } +} diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message.ts index 0d512e73b4a..c7e0e7de41f 100644 --- a/apps/server/src/shared/infra/rabbitmq/rpc-message.ts +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message.ts @@ -1,6 +1,6 @@ export interface IError extends Error { status?: number; - message: never; + message: string; } export interface RpcMessage { message: T; diff --git a/apps/server/src/shared/infra/s3-client/interface/index.ts b/apps/server/src/shared/infra/s3-client/interface/index.ts index dc4b76ad922..ad6ed9c81da 100644 --- a/apps/server/src/shared/infra/s3-client/interface/index.ts +++ b/apps/server/src/shared/infra/s3-client/interface/index.ts @@ -24,6 +24,5 @@ export interface CopyFiles { export interface File { data: Readable; - name: string; mimeType: string; } diff --git a/config/default.schema.json b/config/default.schema.json index ef92d3f1db5..a34d8e899ad 100644 --- a/config/default.schema.json +++ b/config/default.schema.json @@ -1344,14 +1344,7 @@ } } }, - "required": [ - "TEACHER_VISIBILITY_FOR_EXTERNAL_TEAM_INVITATION", - "STUDENT_TEAM_CREATION", - "BLOCK_DISPOSABLE_EMAIL_DOMAINS", - "HOST", - "ACTIVATION_LINK_PERIOD_OF_VALIDITY_SECONDS", - "AES_KEY" - ], + "required": [], "allOf": [ { "$ref": "#/definitions/FEATURE_ES_MERLIN_ENABLED", diff --git a/nest-cli.json b/nest-cli.json index c92fcacf1cf..73dea03c093 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -63,6 +63,15 @@ "tsConfigPath": "apps/server/tsconfig.app.json" } }, + "preview-generator-amqp": { + "type": "application", + "root": "apps/server", + "entryFile": "apps/preview-generator-consumer.app", + "sourceRoot": "apps/server/src", + "compilerOptions": { + "tsConfigPath": "apps/server/tsconfig.app.json" + } + }, "fwu-learning-contents": { "type": "application", "root": "apps/server", diff --git a/package.json b/package.json index 3c5a73df7d1..a82df72904f 100644 --- a/package.json +++ b/package.json @@ -61,11 +61,12 @@ "nest:start:management:dev": "nest start management --watch", "nest:start:management:debug": "nest start management --debug --watch", "nest:start:management:prod": "node dist/apps/server/apps/management.app", - "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp", - "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch", - "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch", + "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", + "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", + "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", + "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", "nest:start:fwu-learning-contents": "nest start fwu-learning-contents", "nest:start:fwu-learning-contents:debug": "nest start fwu-learning-contents --debug --watch", "nest:start:fwu-learning-contents:prod": "node dist/apps/server/apps/fwu-learning-contents.app", From 0a32c99a52ed29178afe2ac0bed22d0ec485c11b Mon Sep 17 00:00:00 2001 From: Martin Schuhmacher <55735359+MartinSchuhmacher@users.noreply.github.com> Date: Fri, 27 Oct 2023 14:59:05 +0200 Subject: [PATCH 2/4] Bump crypto-js from 4.1.1 to 4.2.0 (#4508) updated-dependencies: - dependency-name: crypto-js (from 4.1.1 to 4.2.0.) - dependency-type: direct:production --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e36ecdd9873..cd0606b9fde 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,7 +68,7 @@ "connect-redis": "^6.1.3", "cors": "^2.8.1", "cross-env": "^7.0.0", - "crypto-js": "^4.0.0", + "crypto-js": "^4.2.0", "disposable-email-domains": "^1.0.56", "es6-promisify": "^7.0.0", "express": "^4.14.0", @@ -9069,9 +9069,9 @@ } }, "node_modules/crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" }, "node_modules/css-select": { "version": "5.1.0", @@ -31721,9 +31721,9 @@ "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" }, "crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" }, "css-select": { "version": "5.1.0", diff --git a/package.json b/package.json index a82df72904f..45e150f6668 100644 --- a/package.json +++ b/package.json @@ -151,7 +151,7 @@ "connect-redis": "^6.1.3", "cors": "^2.8.1", "cross-env": "^7.0.0", - "crypto-js": "^4.0.0", + "crypto-js": "^4.2.0", "disposable-email-domains": "^1.0.56", "es6-promisify": "^7.0.0", "express": "^4.14.0", From 0cb9d54e58142979c9f7dae4bd72ea1e240eb7a6 Mon Sep 17 00:00:00 2001 From: Sergej Hoffmann <97111299+SevenWaysDP@users.noreply.github.com> Date: Mon, 30 Oct 2023 11:28:30 +0100 Subject: [PATCH 3/4] BC-5489 - For loggables it should possible to pass unknown cause error (#4501) --- .../error/filter/global-error.filter.spec.ts | 104 +++++++++++++++++- .../core/error/filter/global-error.filter.ts | 4 +- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/apps/server/src/core/error/filter/global-error.filter.spec.ts b/apps/server/src/core/error/filter/global-error.filter.spec.ts index ca40620515f..c45c13e4bff 100644 --- a/apps/server/src/core/error/filter/global-error.filter.spec.ts +++ b/apps/server/src/core/error/filter/global-error.filter.spec.ts @@ -1,7 +1,7 @@ /* eslint-disable promise/valid-params */ import { NotFound } from '@feathersjs/errors'; import { createMock, DeepMocked } from '@golevelup/ts-jest'; -import { ArgumentsHost, BadRequestException, HttpStatus } from '@nestjs/common'; +import { ArgumentsHost, BadRequestException, HttpStatus, InternalServerErrorException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { BusinessError } from '@shared/common'; import { ErrorLogger, ErrorLogMessage, Loggable, LogMessage, ValidationErrorLogMessage } from '@src/core/logger'; @@ -9,6 +9,7 @@ import { Response } from 'express'; import util from 'util'; import { ErrorResponse } from '../dto'; import { ErrorLoggable } from '../loggable/error.loggable'; +import { ErrorUtils } from '../utils'; import { GlobalErrorFilter } from './global-error.filter'; class SampleBusinessError extends BusinessError { @@ -42,6 +43,24 @@ class SampleLoggableException extends BadRequestException implements Loggable { } } +class SampleLoggableExceptionWithCause extends InternalServerErrorException implements Loggable { + constructor(private readonly testValue: string, error?: unknown) { + super(ErrorUtils.createHttpExceptionOptions(error)); + } + + getLogMessage(): ErrorLogMessage { + const message: ErrorLogMessage = { + type: 'WITH_CAUSE', + stack: this.stack, + data: { + testValue: this.testValue, + }, + }; + + return message; + } +} + describe('GlobalErrorFilter', () => { let module: TestingModule; let service: GlobalErrorFilter; @@ -304,24 +323,101 @@ describe('GlobalErrorFilter', () => { ).toBeCalledWith(expectedResponse); }); }); + + describe('when error has a cause error', () => { + const setup = () => { + const causeError = new Error('Cause error'); + const error = new SampleLoggableExceptionWithCause('test', causeError); + const expectedResponse = new ErrorResponse( + 'SAMPLE_WITH_CAUSE', + 'Sample With Cause', + 'Sample Loggable Exception With Cause', + HttpStatus.INTERNAL_SERVER_ERROR + ); + + const argumentsHost = setupHttpArgumentsHost(); + + return { error, argumentsHost, expectedResponse }; + }; + + it('should set response status appropriately', () => { + const { error, argumentsHost } = setup(); + + service.catch(error, argumentsHost); + + expect(argumentsHost.switchToHttp().getResponse().status).toBeCalledWith( + HttpStatus.INTERNAL_SERVER_ERROR + ); + }); + + it('should send appropriate error response', () => { + const { error, argumentsHost, expectedResponse } = setup(); + + service.catch(error, argumentsHost); + + expect( + argumentsHost.switchToHttp().getResponse().status(HttpStatus.INTERNAL_SERVER_ERROR).json + ).toBeCalledWith(expectedResponse); + }); + }); }); describe('when context is rmq', () => { + describe('when error is unknown error', () => { + const setup = () => { + const argumentsHost = createMock(); + argumentsHost.getType.mockReturnValueOnce('rmq'); + + const error = new Error(); + + return { error, argumentsHost }; + }; + + it('should return an RpcMessage with the error', () => { + const { error, argumentsHost } = setup(); + + const result = service.catch(error, argumentsHost); + + expect(result).toEqual({ message: undefined, error }); + }); + }); + + describe('when error is a LoggableError', () => { + const setup = () => { + const causeError = new Error('Cause error'); + const error = new SampleLoggableExceptionWithCause('test', causeError); + const argumentsHost = createMock(); + argumentsHost.getType.mockReturnValueOnce('rmq'); + + return { error, argumentsHost }; + }; + + it('should return appropriate error', () => { + const { error, argumentsHost } = setup(); + + const result = service.catch(error, argumentsHost); + + expect(result).toEqual({ message: undefined, error }); + }); + }); + }); + + describe('when context is other than rmq and http', () => { const setup = () => { const argumentsHost = createMock(); - argumentsHost.getType.mockReturnValueOnce('rmq'); + argumentsHost.getType.mockReturnValueOnce('other'); const error = new Error(); return { error, argumentsHost }; }; - it('should return an RpcMessage with the error', () => { + it('should return undefined', () => { const { error, argumentsHost } = setup(); const result = service.catch(error, argumentsHost); - expect(result).toEqual({ message: undefined, error }); + expect(result).toBeUndefined(); }); }); }); diff --git a/apps/server/src/core/error/filter/global-error.filter.ts b/apps/server/src/core/error/filter/global-error.filter.ts index 7e0d1dc3c3f..56760b18dd9 100644 --- a/apps/server/src/core/error/filter/global-error.filter.ts +++ b/apps/server/src/core/error/filter/global-error.filter.ts @@ -24,7 +24,9 @@ export class GlobalErrorFilter implements Exceptio if (contextType === 'http') { this.sendHttpResponse(error, host); - } else if (contextType === 'rmq') { + } + + if (contextType === 'rmq') { return { message: undefined, error }; } } From 63f38be55227c83660bb31cff110fc7cdf9f5ab3 Mon Sep 17 00:00:00 2001 From: mamutmk5 <3045922+mamutmk5@users.noreply.github.com> Date: Thu, 2 Nov 2023 10:28:36 +0100 Subject: [PATCH 4/4] BC-5546 - Split ingress for Domains (#4495) Separate the Server Ingress to the Server Repo and for each service in an own one. Add the new ingress definitons files to the ansible roles. With the current version of nginx ingress is it possible to have more igresses with different resources for one domain. --- .../schulcloud-server-core/tasks/main.yml | 28 +++++++++++++ .../templates/api-files-ingress.yml.j2 | 41 +++++++++++++++++++ .../templates/api-fwu-ingress.yml.j2 | 41 +++++++++++++++++++ .../templates/ingress.yml.j2 | 41 +++++++++++++++++++ .../schulcloud-server-h5p/tasks/main.yml | 8 ++++ .../templates/api-h5p-ingress.yml.j2 | 41 +++++++++++++++++++ 6 files changed, 200 insertions(+) create mode 100644 ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/api-fwu-ingress.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/ingress.yml.j2 create mode 100644 ansible/roles/schulcloud-server-h5p/templates/api-h5p-ingress.yml.j2 diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml index 7f1bbeeecfe..1b58c8a5413 100644 --- a/ansible/roles/schulcloud-server-core/tasks/main.yml +++ b/ansible/roles/schulcloud-server-core/tasks/main.yml @@ -58,6 +58,13 @@ kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" template: deployment.yml.j2 + + - name: Ingress + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: ingress.yml.j2 + apply: yes - name: FileStorageDeployment kubernetes.core.k8s: @@ -65,6 +72,19 @@ namespace: "{{ NAMESPACE }}" template: api-files-deployment.yml.j2 + - name: FileStorageDeployment + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-files-deployment.yml.j2 + + - name: File Storage Ingress + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-files-ingress.yml.j2 + apply: yes + - name: FwuLearningContentsDeployment kubernetes.core.k8s: kubeconfig: ~/.kube/config @@ -72,6 +92,14 @@ template: api-fwu-deployment.yml.j2 when: FEATURE_FWU_CONTENT_ENABLED is defined and FEATURE_FWU_CONTENT_ENABLED|bool + - name: Fwu Learning Contents Ingress + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-fwu-ingress.yml.j2 + apply: yes + when: FEATURE_FWU_CONTENT_ENABLED is defined and FEATURE_FWU_CONTENT_ENABLED|bool + - name: Delete Files CronJob kubernetes.core.k8s: kubeconfig: ~/.kube/config diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 new file mode 100644 index 00000000000..a1264b52001 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/api-files-ingress.yml.j2 @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ NAMESPACE }}-api-files-ingress + namespace: {{ NAMESPACE }} + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "{{ TLS_ENABELD|default("false") }}" + nginx.ingress.kubernetes.io/proxy-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + nginx.org/client-max-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + # The following properties added with BC-3606. + # The header size of the request is too big. For e.g. state and the permanent growing jwt. + # Nginx throws away the Location header, resulting in the 502 Bad Gateway. + nginx.ingress.kubernetes.io/client-header-buffer-size: 100k + nginx.ingress.kubernetes.io/http2-max-header-size: 96k + nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k + nginx.ingress.kubernetes.io/proxy-buffer-size: 96k +{% if CLUSTER_ISSUER is defined %} + cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }} +{% endif %} + +spec: + ingressClassName: nginx +{% if CLUSTER_ISSUER is defined or (TLS_ENABELD is defined and TLS_ENABELD|bool) %} + tls: + - hosts: + - {{ DOMAIN }} +{% if CLUSTER_ISSUER is defined %} + secretName: {{ DOMAIN }}-tls +{% endif %} +{% endif %} + rules: + - host: {{ DOMAIN }} + http: + paths: + - path: /api/v3/file/ + backend: + service: + name: api-files-svc + port: + number: {{ PORT_FILE_SERVICE }} + pathType: Prefix diff --git a/ansible/roles/schulcloud-server-core/templates/api-fwu-ingress.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-fwu-ingress.yml.j2 new file mode 100644 index 00000000000..f42c322e45b --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/api-fwu-ingress.yml.j2 @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ NAMESPACE }}-api-fwu-ingress + namespace: {{ NAMESPACE }} + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "{{ TLS_ENABELD|default("false") }}" + nginx.ingress.kubernetes.io/proxy-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + nginx.org/client-max-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + # The following properties added with BC-3606. + # The header size of the request is too big. For e.g. state and the permanent growing jwt. + # Nginx throws away the Location header, resulting in the 502 Bad Gateway. + nginx.ingress.kubernetes.io/client-header-buffer-size: 100k + nginx.ingress.kubernetes.io/http2-max-header-size: 96k + nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k + nginx.ingress.kubernetes.io/proxy-buffer-size: 96k +{% if CLUSTER_ISSUER is defined %} + cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }} +{% endif %} + +spec: + ingressClassName: nginx +{% if CLUSTER_ISSUER is defined or (TLS_ENABELD is defined and TLS_ENABELD|bool) %} + tls: + - hosts: + - {{ DOMAIN }} +{% if CLUSTER_ISSUER is defined %} + secretName: {{ DOMAIN }}-tls +{% endif %} +{% endif %} + rules: + - host: {{ DOMAIN }} + http: + paths: + - path: /api/v3/fwu/ + backend: + service: + name: api-fwu-svc + port: + number: {{ PORT_FWU_LEARNING_CONTENTS }} + pathType: Prefix diff --git a/ansible/roles/schulcloud-server-core/templates/ingress.yml.j2 b/ansible/roles/schulcloud-server-core/templates/ingress.yml.j2 new file mode 100644 index 00000000000..b2dd208765f --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/ingress.yml.j2 @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ NAMESPACE }}-api-ingress + namespace: {{ NAMESPACE }} + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "{{ TLS_ENABELD|default("false") }}" + nginx.ingress.kubernetes.io/proxy-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + nginx.org/client-max-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + # The following properties added with BC-3606. + # The header size of the request is too big. For e.g. state and the permanent growing jwt. + # Nginx throws away the Location header, resulting in the 502 Bad Gateway. + nginx.ingress.kubernetes.io/client-header-buffer-size: 100k + nginx.ingress.kubernetes.io/http2-max-header-size: 96k + nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k + nginx.ingress.kubernetes.io/proxy-buffer-size: 96k +{% if CLUSTER_ISSUER is defined %} + cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }} +{% endif %} + +spec: + ingressClassName: nginx +{% if CLUSTER_ISSUER is defined or (TLS_ENABELD is defined and TLS_ENABELD|bool) %} + tls: + - hosts: + - {{ DOMAIN }} +{% if CLUSTER_ISSUER is defined %} + secretName: {{ DOMAIN }}-tls +{% endif %} +{% endif %} + rules: + - host: {{ DOMAIN }} + http: + paths: + - path: /api/v3/ + backend: + service: + name: api-svc + port: + number: {{ PORT_SERVER }} + pathType: Prefix diff --git a/ansible/roles/schulcloud-server-h5p/tasks/main.yml b/ansible/roles/schulcloud-server-h5p/tasks/main.yml index f630b1f3671..368e97a216e 100644 --- a/ansible/roles/schulcloud-server-h5p/tasks/main.yml +++ b/ansible/roles/schulcloud-server-h5p/tasks/main.yml @@ -11,4 +11,12 @@ namespace: "{{ NAMESPACE }}" template: api-h5p-deployment.yml.j2 when: WITH_H5P_EDITOR is defined and WITH_H5P_EDITOR|bool + + - name: H5p Editor Ingress + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: api-h5p-ingress.yml.j2 + apply: yes + when: WITH_H5P_EDITOR is defined and WITH_H5P_EDITOR|bool \ No newline at end of file diff --git a/ansible/roles/schulcloud-server-h5p/templates/api-h5p-ingress.yml.j2 b/ansible/roles/schulcloud-server-h5p/templates/api-h5p-ingress.yml.j2 new file mode 100644 index 00000000000..ec68641bfa2 --- /dev/null +++ b/ansible/roles/schulcloud-server-h5p/templates/api-h5p-ingress.yml.j2 @@ -0,0 +1,41 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ NAMESPACE }}-api-h5p-ingress + namespace: {{ NAMESPACE }} + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "{{ TLS_ENABELD|default("false") }}" + nginx.ingress.kubernetes.io/proxy-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + nginx.org/client-max-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m" + # The following properties added with BC-3606. + # The header size of the request is too big. For e.g. state and the permanent growing jwt. + # Nginx throws away the Location header, resulting in the 502 Bad Gateway. + nginx.ingress.kubernetes.io/client-header-buffer-size: 100k + nginx.ingress.kubernetes.io/http2-max-header-size: 96k + nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k + nginx.ingress.kubernetes.io/proxy-buffer-size: 96k +{% if CLUSTER_ISSUER is defined %} + cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }} +{% endif %} + +spec: + ingressClassName: nginx +{% if CLUSTER_ISSUER is defined or (TLS_ENABELD is defined and TLS_ENABELD|bool) %} + tls: + - hosts: + - {{ DOMAIN }} +{% if CLUSTER_ISSUER is defined %} + secretName: {{ DOMAIN }}-tls +{% endif %} +{% endif %} + rules: + - host: {{ DOMAIN }} + http: + paths: + - path: /api/v3/h5p-editor/ + backend: + service: + name: api-h5p-svc + port: + number: 4448 + pathType: Prefix