From 84c44c1bf99f92b16df1c32792b5670dbb2b4d72 Mon Sep 17 00:00:00 2001 From: Sergej Hoffmann <97111299+SevenWaysDP@users.noreply.github.com> Date: Fri, 27 Oct 2023 08:56:47 +0200 Subject: [PATCH] Bc 5170 no preview for larger files (#4457) --- .github/workflows/push.yml | 18 +- .github/workflows/tag.yml | 8 +- ...file.filestorage => Dockerfile.filepreview | 0 .../schulcloud-server-core/tasks/main.yml | 29 + .../templates/api-files-deployment.yml.j2 | 2 +- .../preview-generator-configmap.yml.j2 | 8 + .../preview-generator-deployment.yml.j2 | 46 + .../preview-generator-onepassword.yml.j2 | 9 + .../preview-generator-scaled-object.yml.j2 | 23 + .../src/apps/files-storage-consumer.app.ts | 2 +- apps/server/src/apps/files-storage.app.ts | 3 +- .../apps/preview-generator-consumer.app.ts | 17 + .../files-storage-client/mapper/index.ts | 3 +- .../service/files-storage.producer.spec.ts | 3 +- .../service/files-storage.producer.ts | 53 +- .../files-storage-preview.api.spec.ts | 15 +- .../controller/files-storage.consumer.spec.ts | 21 +- .../controller/files-storage.consumer.ts | 8 +- .../controller/files-storage.controller.ts | 6 +- .../modules/files-storage/controller/index.ts | 2 +- .../src/modules/files-storage/dto/file.dto.ts | 3 +- .../entity/filerecord.entity.spec.ts | 50 ++ .../files-storage/entity/filerecord.entity.ts | 7 + .../files-preview-amqp.module.ts | 8 + .../files-storage/files-storage.config.ts | 8 +- .../files-storage/files-storage.module.ts | 2 + .../files-storage/helper/file-record.spec.ts | 44 +- .../files-storage/helper/file-record.ts | 25 + .../server/src/modules/files-storage/index.ts | 7 +- .../files-storage/interface/interfaces.ts | 7 +- .../src/modules/files-storage/mapper/index.ts | 1 + .../mapper/preview.builder.spec.ts | 61 ++ .../files-storage/mapper/preview.builder.ts | 47 ++ .../files-storage-delete.service.spec.ts | 62 +- .../service/files-storage.service.ts | 10 +- .../service/preview.service.spec.ts | 787 +++++++----------- .../files-storage/service/preview.service.ts | 110 +-- .../uc/files-storage-delete.uc.spec.ts | 12 +- .../files-storage-download-preview.uc.spec.ts | 11 +- .../files-storage/uc/files-storage.uc.ts | 13 +- .../shared/infra/preview-generator/index.ts | 4 + .../preview-generator/interface/index.ts | 1 + .../interface/preview-consumer-config.ts | 11 + .../preview-generator/interface/preview.ts | 15 + .../loggable/preview-actions.loggable.spec.ts | 37 + .../loggable/preview-actions.loggable.ts | 19 + .../preview-generator-consumer.module.ts | 36 + .../preview-generator-producer.module.ts | 11 + .../preview-generator.builder.spec.ts | 28 + .../preview-generator.builder.ts | 16 + .../preview-generator.consumer.spec.ts | 80 ++ .../preview-generator.consumer.ts | 27 + .../preview-generator.service.spec.ts | 151 ++++ .../preview-generator.service.ts | 56 ++ .../preview.producer.spec.ts | 128 +++ .../preview-generator/preview.producer.ts | 31 + .../infra/rabbitmq}/error.mapper.spec.ts | 0 .../infra/rabbitmq}/error.mapper.ts | 0 .../infra/rabbitmq/exchange/files-preview.ts | 7 + .../shared/infra/rabbitmq/exchange/index.ts | 1 + .../server/src/shared/infra/rabbitmq/index.ts | 2 + .../shared/infra/rabbitmq/rabbitmq.module.ts | 6 +- .../rabbitmq/rpc-message-producer.spec.ts | 130 +++ .../infra/rabbitmq/rpc-message-producer.ts | 37 + .../src/shared/infra/rabbitmq/rpc-message.ts | 2 +- .../shared/infra/s3-client/interface/index.ts | 1 - config/default.schema.json | 9 +- nest-cli.json | 9 + package.json | 7 +- 69 files changed, 1654 insertions(+), 759 deletions(-) rename Dockerfile.filestorage => Dockerfile.filepreview (100%) create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 create mode 100644 ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 create mode 100644 apps/server/src/apps/preview-generator-consumer.app.ts create mode 100644 apps/server/src/modules/files-storage/files-preview-amqp.module.ts create mode 100644 apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts create mode 100644 apps/server/src/modules/files-storage/mapper/preview.builder.ts create mode 100644 apps/server/src/shared/infra/preview-generator/index.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/index.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts create mode 100644 apps/server/src/shared/infra/preview-generator/interface/preview.ts create mode 100644 apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview-generator.service.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts create mode 100644 apps/server/src/shared/infra/preview-generator/preview.producer.ts rename apps/server/src/{modules/files-storage-client/mapper => shared/infra/rabbitmq}/error.mapper.spec.ts (100%) rename apps/server/src/{modules/files-storage-client/mapper => shared/infra/rabbitmq}/error.mapper.ts (100%) create mode 100644 apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts create mode 100644 apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts create mode 100644 apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 94b3514822f..c042be2c2a9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -68,36 +68,36 @@ jobs: tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }} labels: ${{ steps.docker_meta_img.outputs.labels }} - - name: Docker meta Service Name (file storage) + - name: Docker meta Service Name (file preview) id: docker_meta_img_file_storage uses: docker/metadata-action@v4 with: images: ghcr.io/${{ github.repository }} tags: | - type=ref,event=branch,enable=false,priority=600,prefix=file-storage- - type=sha,enable=true,priority=600,prefix=file-storage- + type=ref,event=branch,enable=false,priority=600,prefix=file-preview- + type=sha,enable=true,priority=600,prefix=file-preview- labels: | org.opencontainers.image.title=schulcloud-file-storage - - name: test image exists (file storage) + - name: test image exists (file preview) run: | - echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV + echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV - - name: Set up Docker Buildx (file storage) + - name: Set up Docker Buildx (file preview) if: ${{ env.IMAGE_EXISTS == 0 }} uses: docker/setup-buildx-action@v2 - - name: Build and push ${{ github.repository }} (file storage) + - name: Build and push ${{ github.repository }} (file preview) if: ${{ env.IMAGE_EXISTS == 0 }} uses: docker/build-push-action@v4 with: build-args: | BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }} context: . - file: ./Dockerfile.filestorage + file: ./Dockerfile.filepreview platforms: linux/amd64 push: true - tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} + tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} labels: | ${{ steps.docker_meta_img_file_storage.outputs.labels }} diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index fa62df4b0ab..8f484116ca8 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -48,14 +48,14 @@ jobs: tags: ${{ steps.docker_meta_img_hub.outputs.tags }} labels: ${{ steps.docker_meta_img_hub.outputs.labels }} - - name: Docker meta Service Name for docker hub (file storage) + - name: Docker meta Service Name for docker hub (file preview) id: docker_meta_img_hub_file_storage uses: docker/metadata-action@v4 with: images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server tags: | - type=semver,pattern={{version}},prefix=file-storage-,onlatest=false - type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false + type=semver,pattern={{version}},prefix=file-preview-,onlatest=false + type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false labels: | org.opencontainers.image.title=schulcloud-file-storage - name: Build and push ${{ github.repository }} (file-storage) @@ -64,7 +64,7 @@ jobs: build-args: | BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }} context: . - file: ./Dockerfile.filestorage + file: ./Dockerfile.filepreview platforms: linux/amd64 push: true tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }} diff --git a/Dockerfile.filestorage b/Dockerfile.filepreview similarity index 100% rename from Dockerfile.filestorage rename to Dockerfile.filepreview diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml index fff5b10197a..7f1bbeeecfe 100644 --- a/ansible/roles/schulcloud-server-core/tasks/main.yml +++ b/ansible/roles/schulcloud-server-core/tasks/main.yml @@ -83,3 +83,32 @@ kubeconfig: ~/.kube/config namespace: "{{ NAMESPACE }}" template: amqp-files-deployment.yml.j2 + + - name: Preview Generator Deployment + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-deployment.yml.j2 + + - name: preview generator configmap + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-configmap.yml.j2 + apply: yes + + - name: preview generator Secret by 1Password + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-onepassword.yml.j2 + when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool + + - name: preview generator scaled object + kubernetes.core.k8s: + kubeconfig: ~/.kube/config + namespace: "{{ NAMESPACE }}" + template: preview-generator-scaled-object.yml.j2 + when: + - KEDA_ENABLED is defined and KEDA_ENABLED|bool + - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 index 7f5a5b7e50b..727b5a9f4f0 100644 --- a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 +++ b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 @@ -29,7 +29,7 @@ spec: runAsNonRoot: true containers: - name: api-files - image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }} imagePullPolicy: IfNotPresent ports: - containerPort: 4444 diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 new file mode 100644 index 00000000000..457a5e47364 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: preview-generator-configmap + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +data: diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 new file mode 100644 index 00000000000..51d87b88755 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: preview-generator-deployment + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + #maxUnavailable: 1 + revisionHistoryLimit: 4 + paused: false + selector: + matchLabels: + app: preview-generator + template: + metadata: + labels: + app: preview-generator + spec: + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + runAsNonRoot: true + containers: + - name: preview-generator + image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }} + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: preview-generator-configmap + - secretRef: + name: preview-generator-secret + command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod'] + resources: + limits: + cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }} + memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }} + requests: + cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }} + memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }} diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 new file mode 100644 index 00000000000..51d979e27e0 --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 @@ -0,0 +1,9 @@ +apiVersion: onepassword.com/v1 +kind: OnePasswordItem +metadata: + name: preview-generator-secret + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator" diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 new file mode 100644 index 00000000000..2f8f9091b8e --- /dev/null +++ b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 @@ -0,0 +1,23 @@ +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: preview-generator-rabbitmq-scaledobject + namespace: {{ NAMESPACE }} + labels: + app: preview-generator +spec: + scaleTargetRef: + name: preview-generator-deployment + idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }} + minReplicaCount: {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }} + maxReplicaCount: {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }} + triggers: + - type: rabbitmq + metadata: + protocol: amqp + queueName: generate-preview + mode: QueueLength + value: "1" + authenticationRef: + name: rabbitmq-trigger-auth diff --git a/apps/server/src/apps/files-storage-consumer.app.ts b/apps/server/src/apps/files-storage-consumer.app.ts index a18b5f4604b..cb62c9c76f9 100644 --- a/apps/server/src/apps/files-storage-consumer.app.ts +++ b/apps/server/src/apps/files-storage-consumer.app.ts @@ -3,7 +3,7 @@ import { NestFactory } from '@nestjs/core'; // register source-map-support for debugging -import { FilesStorageAMQPModule } from '@modules/files-storage'; +import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module'; import { install as sourceMapInstall } from 'source-map-support'; async function bootstrap() { diff --git a/apps/server/src/apps/files-storage.app.ts b/apps/server/src/apps/files-storage.app.ts index 2d2f9343ac2..adbaac9d90c 100644 --- a/apps/server/src/apps/files-storage.app.ts +++ b/apps/server/src/apps/files-storage.app.ts @@ -8,9 +8,10 @@ import express from 'express'; import { install as sourceMapInstall } from 'source-map-support'; // application imports +import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module'; +import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const'; import { SwaggerDocumentOptions } from '@nestjs/swagger'; import { LegacyLogger } from '@src/core/logger'; -import { API_VERSION_PATH, FilesStorageApiModule } from '@modules/files-storage'; import { enableOpenApiDocs } from '@src/shared/controller/swagger'; async function bootstrap() { diff --git a/apps/server/src/apps/preview-generator-consumer.app.ts b/apps/server/src/apps/preview-generator-consumer.app.ts new file mode 100644 index 00000000000..1c2be631233 --- /dev/null +++ b/apps/server/src/apps/preview-generator-consumer.app.ts @@ -0,0 +1,17 @@ +/* istanbul ignore file */ +/* eslint-disable no-console */ +import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module'; +import { NestFactory } from '@nestjs/core'; +import { install as sourceMapInstall } from 'source-map-support'; + +async function bootstrap() { + sourceMapInstall(); + + const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule); + await nestApp.init(); + + console.log('#############################################'); + console.log(`### Start Preview Generator AMQP Consumer ###`); + console.log('#############################################'); +} +void bootstrap(); diff --git a/apps/server/src/modules/files-storage-client/mapper/index.ts b/apps/server/src/modules/files-storage-client/mapper/index.ts index 06f40e707e1..006cf64b627 100644 --- a/apps/server/src/modules/files-storage-client/mapper/index.ts +++ b/apps/server/src/modules/files-storage-client/mapper/index.ts @@ -1,4 +1,3 @@ -export * from './error.mapper'; +export * from './copy-files-of-parent-param.builder'; export * from './files-storage-client.mapper'; export * from './files-storage-param.builder'; -export * from './copy-files-of-parent-param.builder'; diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts index 926f860c736..7e4ed5a1c83 100644 --- a/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts +++ b/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts @@ -3,10 +3,9 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; -import { FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq'; +import { ErrorMapper, FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq'; import { setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { ErrorMapper } from '../mapper'; import { FilesStorageProducer } from './files-storage.producer'; describe('FilesStorageProducer', () => { diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts index afd4f6365e1..ea049442df4 100644 --- a/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts +++ b/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts @@ -2,7 +2,6 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { EntityId } from '@shared/domain'; -import { RpcMessage } from '@shared/infra/rabbitmq/rpc-message'; import { LegacyLogger } from '@src/core/logger'; import { FilesStorageEvents, @@ -11,75 +10,45 @@ import { ICopyFilesOfParentParams, IFileDO, IFileRecordParams, + RpcMessageProducer, } from '@src/shared/infra/rabbitmq'; import { IFilesStorageClientConfig } from '../interfaces'; -import { ErrorMapper } from '../mapper/error.mapper'; @Injectable() -export class FilesStorageProducer { - private readonly timeout = 0; - +export class FilesStorageProducer extends RpcMessageProducer { constructor( + protected readonly amqpConnection: AmqpConnection, private readonly logger: LegacyLogger, - private readonly amqpConnection: AmqpConnection, - private readonly configService: ConfigService + protected readonly configService: ConfigService ) { + super(amqpConnection, FilesStorageExchange, configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API')); this.logger.setContext(FilesStorageProducer.name); - this.timeout = this.configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API'); } async copyFilesOfParent(payload: ICopyFilesOfParentParams): Promise { this.logger.debug({ action: 'copyFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.COPY_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.COPY_FILES_OF_PARENT, payload); this.logger.debug({ action: 'copyFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; + return response; } async listFilesOfParent(payload: IFileRecordParams): Promise { this.logger.debug({ action: 'listFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.LIST_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.LIST_FILES_OF_PARENT, payload); this.logger.debug({ action: 'listFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; + return response; } async deleteFilesOfParent(payload: EntityId): Promise { this.logger.debug({ action: 'deleteFilesOfParent:started', payload }); - const response = await this.amqpConnection.request>( - this.createRequest(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload) - ); + const response = await this.request(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload); this.logger.debug({ action: 'deleteFilesOfParent:finished', payload }); - this.checkError(response); - return response.message || []; - } - - // need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984 - // mapRpcErrorResponseToDomainError should also removed with this ticket - private checkError(response: RpcMessage) { - const { error } = response; - if (error) { - const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error); - throw domainError; - } - } - - private createRequest(event: FilesStorageEvents, payload: IFileRecordParams | ICopyFilesOfParentParams | EntityId) { - return { - exchange: FilesStorageExchange, - routingKey: event, - payload, - timeout: this.timeout, - }; + return response; } } diff --git a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts index 22749024b31..e87aa5ddbe6 100644 --- a/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts +++ b/apps/server/src/modules/files-storage/controller/api-test/files-storage-preview.api.spec.ts @@ -7,6 +7,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ApiValidationError } from '@shared/common'; import { EntityId, Permission } from '@shared/domain'; import { AntivirusService } from '@shared/infra/antivirus'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { cleanupCollections, mapUserToCurrentUser, roleFactory, schoolFactory, userFactory } from '@shared/testing'; import NodeClam from 'clamscan'; @@ -89,6 +90,7 @@ describe('File Controller (API) - preview', () => { let app: INestApplication; let em: EntityManager; let s3ClientAdapter: DeepMocked; + let antivirusService: DeepMocked; let currentUser: ICurrentUser; let api: API; let schoolId: EntityId; @@ -103,6 +105,8 @@ describe('File Controller (API) - preview', () => { }) .overrideProvider(AntivirusService) .useValue(createMock()) + .overrideProvider(PreviewProducer) + .useValue(createMock()) .overrideProvider(FILES_STORAGE_S3_CONNECTION) .useValue(createMock()) .overrideGuard(JwtAuthGuard) @@ -123,6 +127,7 @@ describe('File Controller (API) - preview', () => { em = module.get(EntityManager); s3ClientAdapter = module.get(FILES_STORAGE_S3_CONNECTION); + antivirusService = module.get(AntivirusService); api = new API(app); }); @@ -147,6 +152,7 @@ describe('File Controller (API) - preview', () => { uploadPath = `/file/upload/${schoolId}/schools/${schoolId}`; jest.spyOn(FileType, 'fileTypeStream').mockImplementation((readable) => Promise.resolve(readable)); + antivirusService.checkStream.mockResolvedValueOnce({ virus_detected: false }); }); const setScanStatus = async (fileRecordId: EntityId, status: ScanStatus) => { @@ -329,9 +335,8 @@ describe('File Controller (API) - preview', () => { const { result: uploadedFile } = await api.postUploadFile(uploadPath); await setScanStatus(uploadedFile.id, ScanStatus.VERIFIED); - const originalFile = TestHelper.createFile(); const previewFile = TestHelper.createFile('bytes 0-3/4'); - s3ClientAdapter.get.mockResolvedValueOnce(originalFile).mockResolvedValueOnce(previewFile); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); return { uploadedFile }; }; @@ -374,12 +379,8 @@ describe('File Controller (API) - preview', () => { await setScanStatus(uploadedFile.id, ScanStatus.VERIFIED); const error = new NotFoundException(); - const originalFile = TestHelper.createFile(); const previewFile = TestHelper.createFile('bytes 0-3/4'); - s3ClientAdapter.get - .mockRejectedValueOnce(error) - .mockResolvedValueOnce(originalFile) - .mockResolvedValueOnce(previewFile); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); return { uploadedFile }; }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts b/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts index 124f486dc68..81fd2884347 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.consumer.spec.ts @@ -7,6 +7,7 @@ import { courseFactory, fileRecordFactory, setupEntities } from '@shared/testing import { LegacyLogger } from '@src/core/logger'; import { FileRecord, FileRecordParentType } from '../entity'; import { FilesStorageService } from '../service/files-storage.service'; +import { PreviewService } from '../service/preview.service'; import { FileRecordResponse } from './dto'; import { FilesStorageConsumer } from './files-storage.consumer'; @@ -33,6 +34,10 @@ describe('FilesStorageConsumer', () => { provide: FilesStorageService, useValue: createMock(), }, + { + provide: PreviewService, + useValue: createMock(), + }, { provide: LegacyLogger, useValue: createMock(), @@ -165,9 +170,9 @@ describe('FilesStorageConsumer', () => { const parentId = new ObjectId().toHexString(); const fileRecords = fileRecordFactory.buildList(3); - filesStorageService.deleteFilesOfParent.mockResolvedValue([fileRecords, fileRecords.length]); + filesStorageService.getFileRecordsOfParent.mockResolvedValue([fileRecords, fileRecords.length]); - return { parentId }; + return { parentId, fileRecords }; }; it('should call filesStorageService.deleteFilesOfParent with params', async () => { @@ -175,7 +180,15 @@ describe('FilesStorageConsumer', () => { await service.deleteFilesOfParent(parentId); - expect(filesStorageService.deleteFilesOfParent).toBeCalledWith(parentId); + expect(filesStorageService.getFileRecordsOfParent).toBeCalledWith(parentId); + }); + + it('should call filesStorageService.deleteFilesOfParent with params', async () => { + const { parentId, fileRecords } = setup(); + + await service.deleteFilesOfParent(parentId); + + expect(filesStorageService.deleteFilesOfParent).toBeCalledWith(fileRecords); }); it('should return array instances of FileRecordResponse', async () => { @@ -191,7 +204,7 @@ describe('FilesStorageConsumer', () => { const setup = () => { const parentId = new ObjectId().toHexString(); - filesStorageService.deleteFilesOfParent.mockResolvedValue([[], 0]); + filesStorageService.getFileRecordsOfParent.mockResolvedValue([[], 0]); return { parentId }; }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts b/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts index e7dee3c6b0d..aabefa60f16 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.consumer.ts @@ -7,12 +7,14 @@ import { LegacyLogger } from '@src/core/logger'; import { FilesStorageEvents, FilesStorageExchange, ICopyFileDO, IFileDO } from '@src/shared/infra/rabbitmq'; import { FilesStorageMapper } from '../mapper'; import { FilesStorageService } from '../service/files-storage.service'; +import { PreviewService } from '../service/preview.service'; import { CopyFilesOfParentPayload, FileRecordParams } from './dto'; @Injectable() export class FilesStorageConsumer { constructor( private readonly filesStorageService: FilesStorageService, + private readonly previewService: PreviewService, private logger: LegacyLogger, // eslint-disable-next-line @typescript-eslint/no-unused-vars private readonly orm: MikroORM // don't remove it, we need it for @UseRequestContext @@ -61,7 +63,11 @@ export class FilesStorageConsumer { public async deleteFilesOfParent(@RabbitPayload() payload: EntityId): Promise> { this.logger.debug({ action: 'deleteFilesOfParent', payload }); - const [fileRecords, total] = await this.filesStorageService.deleteFilesOfParent(payload); + const [fileRecords, total] = await this.filesStorageService.getFileRecordsOfParent(payload); + + await this.previewService.deletePreviews(fileRecords); + await this.filesStorageService.deleteFilesOfParent(fileRecords); + const response = FilesStorageMapper.mapToFileRecordListResponse(fileRecords, total); return { message: response.data }; diff --git a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts index 564919670e4..736d69e3e29 100644 --- a/apps/server/src/modules/files-storage/controller/files-storage.controller.ts +++ b/apps/server/src/modules/files-storage/controller/files-storage.controller.ts @@ -1,3 +1,4 @@ +import { Authenticate, CurrentUser, ICurrentUser } from '@modules/authentication'; import { BadRequestException, Body, @@ -22,10 +23,10 @@ import { UseInterceptors, } from '@nestjs/common'; import { ApiConsumes, ApiHeader, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; -import { ApiValidationError, RequestLoggingInterceptor } from '@shared/common'; +import { ApiValidationError, RequestLoggingInterceptor, RequestTimeout } from '@shared/common'; import { PaginationParams } from '@shared/controller'; -import { ICurrentUser, Authenticate, CurrentUser } from '@modules/authentication'; import { Request, Response } from 'express'; +import { config } from '../files-storage.config'; import { GetFileResponse } from '../interface'; import { FilesStorageMapper } from '../mapper'; import { FileRecordMapper } from '../mapper/file-record.mapper'; @@ -126,6 +127,7 @@ export class FilesStorageController { @ApiResponse({ status: 500, type: InternalServerErrorException }) @ApiHeader({ name: 'Range', required: false }) @Get('/preview/:fileRecordId/:fileName') + @RequestTimeout(config().INCOMING_REQUEST_TIMEOUT) async downloadPreview( @Param() params: DownloadFileParams, @CurrentUser() currentUser: ICurrentUser, diff --git a/apps/server/src/modules/files-storage/controller/index.ts b/apps/server/src/modules/files-storage/controller/index.ts index ffd18a5db36..7aa61d2c93b 100644 --- a/apps/server/src/modules/files-storage/controller/index.ts +++ b/apps/server/src/modules/files-storage/controller/index.ts @@ -1,3 +1,3 @@ export * from './file-security.controller'; -export * from './files-storage.controller'; export * from './files-storage.consumer'; +export * from './files-storage.controller'; diff --git a/apps/server/src/modules/files-storage/dto/file.dto.ts b/apps/server/src/modules/files-storage/dto/file.dto.ts index 540e35bb7e6..9668ac3af72 100644 --- a/apps/server/src/modules/files-storage/dto/file.dto.ts +++ b/apps/server/src/modules/files-storage/dto/file.dto.ts @@ -1,6 +1,7 @@ +import { File } from '@shared/infra/s3-client'; import { Readable } from 'stream'; -export class FileDto { +export class FileDto implements File { constructor(file: FileDto) { this.name = file.name; this.data = file.data; diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts index f07d88d85fd..f497ff39a6c 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.spec.ts @@ -783,4 +783,54 @@ describe('FileRecord Entity', () => { }); }); }); + + describe('fileNameWithoutExtension is called', () => { + describe('WHEN file name has extension', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: 'file-name.jpg' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('file-name'); + }); + }); + + describe('WHEN file name has not extension', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: 'file-name' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('file-name'); + }); + }); + + describe('WHEN file name starts with dot', () => { + const setup = () => { + const fileRecord = fileRecordFactory.build({ name: '.bild.123.jpg' }); + + return { fileRecord }; + }; + + it('should return the correct file name without extension', () => { + const { fileRecord } = setup(); + + const result = fileRecord.fileNameWithoutExtension; + + expect(result).toEqual('.bild.123'); + }); + }); + }); }); diff --git a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts index a87789d30fc..26f2807924c 100644 --- a/apps/server/src/modules/files-storage/entity/filerecord.entity.ts +++ b/apps/server/src/modules/files-storage/entity/filerecord.entity.ts @@ -2,6 +2,7 @@ import { Embeddable, Embedded, Entity, Enum, Index, Property } from '@mikro-orm/ import { ObjectId } from '@mikro-orm/mongodb'; import { BadRequestException } from '@nestjs/common'; import { BaseEntityWithTimestamps, EntityId } from '@shared/domain'; +import path from 'path'; import { v4 as uuid } from 'uuid'; import { ErrorType } from '../error'; import { PreviewInputMimeTypes } from '../interface/preview-input-mime-types.enum'; @@ -293,4 +294,10 @@ export class FileRecord extends BaseEntityWithTimestamps { return PreviewStatus.PREVIEW_NOT_POSSIBLE_SCAN_STATUS_ERROR; } + + public get fileNameWithoutExtension(): string { + const filenameObj = path.parse(this.name); + + return filenameObj.name; + } } diff --git a/apps/server/src/modules/files-storage/files-preview-amqp.module.ts b/apps/server/src/modules/files-storage/files-preview-amqp.module.ts new file mode 100644 index 00000000000..411a26e76d6 --- /dev/null +++ b/apps/server/src/modules/files-storage/files-preview-amqp.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { PreviewGeneratorConsumerModule } from '@shared/infra/preview-generator'; +import { defaultConfig, s3Config } from './files-storage.config'; + +@Module({ + imports: [PreviewGeneratorConsumerModule.register({ storageConfig: s3Config, serverConfig: defaultConfig })], +}) +export class PreviewGeneratorAMQPModule {} diff --git a/apps/server/src/modules/files-storage/files-storage.config.ts b/apps/server/src/modules/files-storage/files-storage.config.ts index 7dc01e9f4e1..7fac8ded763 100644 --- a/apps/server/src/modules/files-storage/files-storage.config.ts +++ b/apps/server/src/modules/files-storage/files-storage.config.ts @@ -9,13 +9,17 @@ export interface IFileStorageConfig extends ICoreModuleConfig { USE_STREAM_TO_ANTIVIRUS: boolean; } -const fileStorageConfig: IFileStorageConfig = { +export const defaultConfig = { + NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string, INCOMING_REQUEST_TIMEOUT: Configuration.get('FILES_STORAGE__INCOMING_REQUEST_TIMEOUT') as number, +}; + +const fileStorageConfig: IFileStorageConfig = { INCOMING_REQUEST_TIMEOUT_COPY_API: Configuration.get('INCOMING_REQUEST_TIMEOUT_COPY_API') as number, MAX_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number, MAX_SECURITY_CHECK_FILE_SIZE: Configuration.get('FILES_STORAGE__MAX_FILE_SIZE') as number, - NEST_LOG_LEVEL: Configuration.get('NEST_LOG_LEVEL') as string, USE_STREAM_TO_ANTIVIRUS: Configuration.get('FILES_STORAGE__USE_STREAM_TO_ANTIVIRUS') as boolean, + ...defaultConfig, }; // The configurations lookup diff --git a/apps/server/src/modules/files-storage/files-storage.module.ts b/apps/server/src/modules/files-storage/files-storage.module.ts index 248654218ef..ccdaeb7f9fa 100644 --- a/apps/server/src/modules/files-storage/files-storage.module.ts +++ b/apps/server/src/modules/files-storage/files-storage.module.ts @@ -5,6 +5,7 @@ import { Module, NotFoundException } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { ALL_ENTITIES } from '@shared/domain'; import { AntivirusModule } from '@shared/infra/antivirus/antivirus.module'; +import { PreviewGeneratorProducerModule } from '@shared/infra/preview-generator'; import { RabbitMQWrapperModule } from '@shared/infra/rabbitmq/rabbitmq.module'; import { S3ClientModule } from '@shared/infra/s3-client'; import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config'; @@ -27,6 +28,7 @@ const imports = [ port: Configuration.get('CLAMAV__SERVICE_PORT') as number, }), S3ClientModule.register([s3Config]), + PreviewGeneratorProducerModule, ]; const providers = [FilesStorageService, PreviewService, FileRecordRepo]; diff --git a/apps/server/src/modules/files-storage/helper/file-record.spec.ts b/apps/server/src/modules/files-storage/helper/file-record.spec.ts index 6d975ebe21b..a999b599936 100644 --- a/apps/server/src/modules/files-storage/helper/file-record.spec.ts +++ b/apps/server/src/modules/files-storage/helper/file-record.spec.ts @@ -1,8 +1,9 @@ import { EntityId } from '@shared/domain'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { ObjectId } from 'bson'; -import { createFileRecord, markForDelete, unmarkForDelete } from '.'; +import { createFileRecord, getFormat, getPreviewName, markForDelete, unmarkForDelete } from '.'; import { FileRecord } from '../entity'; +import { PreviewOutputMimeTypes } from '../interface'; describe('File Record Helper', () => { const setupFileRecords = () => { @@ -88,4 +89,45 @@ describe('File Record Helper', () => { expect(newFileRecord).toEqual(expect.any(FileRecord)); }); }); + + describe('getFormat is called', () => { + it('should return format', () => { + const mimeType = 'image/jpeg'; + + const result = getFormat(mimeType); + + expect(result).toEqual('jpeg'); + }); + + it('should throw error', () => { + const mimeType = 'image'; + + expect(() => getFormat(mimeType)).toThrowError(`could not get format from mime type: ${mimeType}`); + }); + }); + + describe('getPreviewName is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const outputFormat = PreviewOutputMimeTypes.IMAGE_WEBP; + + return { fileRecord, outputFormat }; + }; + + it('should return origin file name', () => { + const { fileRecord } = setup(); + + const result = getPreviewName(fileRecord, undefined); + + expect(result).toEqual(fileRecord.name); + }); + + it('should return preview name with format', () => { + const { fileRecord, outputFormat } = setup(); + + const result = getPreviewName(fileRecord, outputFormat); + + expect(result).toEqual(`${fileRecord.name.split('.')[0]}.webp`); + }); + }); }); diff --git a/apps/server/src/modules/files-storage/helper/file-record.ts b/apps/server/src/modules/files-storage/helper/file-record.ts index c0984063420..ed291661735 100644 --- a/apps/server/src/modules/files-storage/helper/file-record.ts +++ b/apps/server/src/modules/files-storage/helper/file-record.ts @@ -1,5 +1,7 @@ +import { InternalServerErrorException } from '@nestjs/common'; import { FileRecordParams } from '../controller/dto'; import { FileRecord } from '../entity'; +import { PreviewOutputMimeTypes } from '../interface'; export function markForDelete(fileRecords: FileRecord[]): FileRecord[] { const markedFileRecords = fileRecords.map((fileRecord) => { @@ -38,3 +40,26 @@ export function createFileRecord( return entity; } + +export function getFormat(mimeType: string): string { + const format = mimeType.split('/')[1]; + + if (!format) { + throw new InternalServerErrorException(`could not get format from mime type: ${mimeType}`); + } + + return format; +} + +export function getPreviewName(fileRecord: FileRecord, outputFormat?: PreviewOutputMimeTypes): string { + const { fileNameWithoutExtension, name } = fileRecord; + + if (!outputFormat) { + return name; + } + + const format = getFormat(outputFormat); + const previewFileName = `${fileNameWithoutExtension}.${format}`; + + return previewFileName; +} diff --git a/apps/server/src/modules/files-storage/index.ts b/apps/server/src/modules/files-storage/index.ts index c22e4c2be98..6ee0883938f 100644 --- a/apps/server/src/modules/files-storage/index.ts +++ b/apps/server/src/modules/files-storage/index.ts @@ -1,5 +1,2 @@ -export * from './files-storage-amqp.module'; -export * from './files-storage-api.module'; -export * from './files-storage-test.module'; // @deprecated remove after move api tests to modules -export * from './files-storage.config'; -export * from './files-storage.const'; +// this module has no exports +// it is an isolated module, it cannot be used in other modules diff --git a/apps/server/src/modules/files-storage/interface/interfaces.ts b/apps/server/src/modules/files-storage/interface/interfaces.ts index 047c943e55a..2f288f9133b 100644 --- a/apps/server/src/modules/files-storage/interface/interfaces.ts +++ b/apps/server/src/modules/files-storage/interface/interfaces.ts @@ -1,5 +1,5 @@ import { Readable } from 'stream'; -import type { DownloadFileParams, PreviewParams } from '../controller/dto'; +import type { PreviewParams } from '../controller/dto'; import { FileRecord } from '../entity'; export interface GetFileResponse { @@ -13,9 +13,10 @@ export interface GetFileResponse { export interface PreviewFileParams { fileRecord: FileRecord; - downloadParams: DownloadFileParams; previewParams: PreviewParams; hash: string; - filePath: string; + originFilePath: string; + previewFilePath: string; + format: string; bytesRange?: string; } diff --git a/apps/server/src/modules/files-storage/mapper/index.ts b/apps/server/src/modules/files-storage/mapper/index.ts index 556e7508929..bc8af7f7f05 100644 --- a/apps/server/src/modules/files-storage/mapper/index.ts +++ b/apps/server/src/modules/files-storage/mapper/index.ts @@ -3,3 +3,4 @@ export * from './file-dto.builder'; export * from './file-record.mapper'; export * from './file-response.builder'; export * from './files-storage.mapper'; +export * from './preview.builder'; diff --git a/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts b/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts new file mode 100644 index 00000000000..1a3cc843f86 --- /dev/null +++ b/apps/server/src/modules/files-storage/mapper/preview.builder.spec.ts @@ -0,0 +1,61 @@ +import { fileRecordFactory } from '@shared/testing'; +import { PreviewOutputMimeTypes } from '../interface'; +import { PreviewBuilder } from './preview.builder'; + +describe('PreviewBuilder', () => { + describe('buildParams is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const previewParams = { outputFormat: PreviewOutputMimeTypes.IMAGE_WEBP }; + const bytesRange = 'bytes=0-100'; + + const expectedResponse = { + fileRecord, + previewParams, + hash: expect.any(String), + previewFilePath: expect.any(String), + originFilePath: expect.any(String), + format: expect.any(String), + bytesRange, + }; + + return { fileRecord, previewParams, bytesRange, expectedResponse }; + }; + + it('should return preview file params', () => { + const { fileRecord, previewParams, bytesRange, expectedResponse } = setup(); + + const result = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); + + expect(result).toEqual(expectedResponse); + }); + }); + + describe('buildPayload is called', () => { + const setup = () => { + const fileRecord = fileRecordFactory.buildWithId(); + const previewParams = { outputFormat: PreviewOutputMimeTypes.IMAGE_WEBP }; + const bytesRange = 'bytes=0-100'; + const previewFileParams = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); + + const expectedResponse = { + originFilePath: previewFileParams.originFilePath, + previewFilePath: previewFileParams.previewFilePath, + previewOptions: { + format: previewFileParams.format, + width: previewFileParams.previewParams.width, + }, + }; + + return { previewFileParams, expectedResponse }; + }; + + it('should return preview payload', () => { + const { previewFileParams, expectedResponse } = setup(); + + const result = PreviewBuilder.buildPayload(previewFileParams); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/modules/files-storage/mapper/preview.builder.ts b/apps/server/src/modules/files-storage/mapper/preview.builder.ts new file mode 100644 index 00000000000..83a16448a98 --- /dev/null +++ b/apps/server/src/modules/files-storage/mapper/preview.builder.ts @@ -0,0 +1,47 @@ +import { PreviewFileOptions } from '@shared/infra/preview-generator'; +import { PreviewParams } from '../controller/dto'; +import { FileRecord } from '../entity'; +import { createPath, createPreviewFilePath, createPreviewNameHash, getFormat } from '../helper'; +import { PreviewFileParams } from '../interface'; + +export class PreviewBuilder { + public static buildParams( + fileRecord: FileRecord, + previewParams: PreviewParams, + bytesRange: string | undefined + ): PreviewFileParams { + const { schoolId, id, mimeType } = fileRecord; + const originFilePath = createPath(schoolId, id); + const format = getFormat(previewParams.outputFormat ?? mimeType); + + const hash = createPreviewNameHash(id, previewParams); + const previewFilePath = createPreviewFilePath(schoolId, hash, id); + + const previewFileParams = { + fileRecord, + previewParams, + hash, + previewFilePath, + originFilePath, + format, + bytesRange, + }; + + return previewFileParams; + } + + public static buildPayload(params: PreviewFileParams): PreviewFileOptions { + const { originFilePath, previewFilePath, previewParams, format } = params; + + const payload = { + originFilePath, + previewFilePath, + previewOptions: { + format, + width: previewParams.width, + }, + }; + + return payload; + } +} diff --git a/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts b/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts index cd76b564b31..3705f93b51c 100644 --- a/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/files-storage-delete.service.spec.ts @@ -171,35 +171,13 @@ describe('FilesStorageService delete methods', () => { return { parentId, fileRecords }; }; - it('should call findBySchoolIdAndParentId once with correct params', async () => { - const { parentId } = setup(); - - await service.deleteFilesOfParent(parentId); - - expect(fileRecordRepo.findByParentId).toHaveBeenNthCalledWith(1, parentId); - }); - it('should call delete with correct params', async () => { - const { parentId, fileRecords } = setup(); + const { fileRecords } = setup(); - await service.deleteFilesOfParent(parentId); + await service.deleteFilesOfParent(fileRecords); expect(service.delete).toHaveBeenCalledWith(fileRecords); }); - - it('should return file records and count', async () => { - const { parentId, fileRecords } = setup(); - - const responseData = await service.deleteFilesOfParent(parentId); - expect(responseData[0]).toEqual( - expect.arrayContaining([ - expect.objectContaining({ ...fileRecords[0] }), - expect.objectContaining({ ...fileRecords[1] }), - expect.objectContaining({ ...fileRecords[2] }), - ]) - ); - expect(responseData[1]).toEqual(fileRecords.length); - }); }); describe('WHEN no files exists', () => { @@ -215,43 +193,17 @@ describe('FilesStorageService delete methods', () => { const { parentId } = params; spy = jest.spyOn(service, 'delete'); - fileRecordRepo.findByParentId.mockResolvedValueOnce([fileRecords, fileRecords.length]); - return { parentId }; + return { parentId, fileRecords }; }; it('should not call delete', async () => { - const { parentId } = setup(); + const { fileRecords } = setup(); - await service.deleteFilesOfParent(parentId); + await service.deleteFilesOfParent(fileRecords); expect(service.delete).toHaveBeenCalledTimes(0); }); - - it('should return empty counted type', async () => { - const { parentId } = setup(); - - const result = await service.deleteFilesOfParent(parentId); - - expect(result).toEqual([[], 0]); - }); - }); - - describe('WHEN repository throw an error', () => { - const setup = () => { - const { params } = buildFileRecordsWithParams(); - const { parentId } = params; - - fileRecordRepo.findByParentId.mockRejectedValueOnce(new Error('bla')); - - return { parentId }; - }; - - it('should pass the error', async () => { - const { parentId } = setup(); - - await expect(service.deleteFilesOfParent(parentId)).rejects.toThrow(new Error('bla')); - }); }); describe('WHEN service.delete throw an error', () => { @@ -272,9 +224,9 @@ describe('FilesStorageService delete methods', () => { }; it('should pass the error', async () => { - const { parentId } = setup(); + const { fileRecords } = setup(); - await expect(service.deleteFilesOfParent(parentId)).rejects.toThrow(new Error('bla')); + await expect(service.deleteFilesOfParent(fileRecords)).rejects.toThrow(new Error('bla')); }); }); }); diff --git a/apps/server/src/modules/files-storage/service/files-storage.service.ts b/apps/server/src/modules/files-storage/service/files-storage.service.ts index 8c0c85630de..209f1804d3e 100644 --- a/apps/server/src/modules/files-storage/service/files-storage.service.ts +++ b/apps/server/src/modules/files-storage/service/files-storage.service.ts @@ -244,7 +244,7 @@ export class FilesStorageService { } // download - private checkFileName(fileRecord: FileRecord, params: DownloadFileParams): void | NotFoundException { + public checkFileName(fileRecord: FileRecord, params: DownloadFileParams): void | NotFoundException { if (!fileRecord.hasName(params.fileName)) { this.logger.debug(`could not find file with id: ${fileRecord.id} by filename`); throw new NotFoundException(ErrorType.FILE_NOT_FOUND); @@ -304,14 +304,10 @@ export class FilesStorageService { await this.deleteWithRollbackByError(fileRecords); } - public async deleteFilesOfParent(parentId: EntityId): Promise> { - const [fileRecords, count] = await this.getFileRecordsOfParent(parentId); - - if (count > 0) { + public async deleteFilesOfParent(fileRecords: FileRecord[]): Promise { + if (fileRecords.length > 0) { await this.delete(fileRecords); } - - return [fileRecords, count]; } // restore diff --git a/apps/server/src/modules/files-storage/service/preview.service.spec.ts b/apps/server/src/modules/files-storage/service/preview.service.spec.ts index ed4592b97da..f02f48aee21 100644 --- a/apps/server/src/modules/files-storage/service/preview.service.spec.ts +++ b/apps/server/src/modules/files-storage/service/preview.service.spec.ts @@ -2,33 +2,22 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; import { NotFoundException, UnprocessableEntityException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { Readable } from 'stream'; import { FileRecordParams } from '../controller/dto'; import { FileRecord, FileRecordParentType, ScanStatus } from '../entity'; import { ErrorType } from '../error'; import { FILES_STORAGE_S3_CONNECTION } from '../files-storage.config'; -import { createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; +import { createPath, createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; import { TestHelper } from '../helper/test-helper'; import { PreviewWidth } from '../interface'; import { PreviewOutputMimeTypes } from '../interface/preview-output-mime-types.enum'; -import { FileDtoBuilder, FileResponseBuilder } from '../mapper'; +import { FileResponseBuilder } from '../mapper'; import { FilesStorageService } from './files-storage.service'; import { PreviewService } from './preview.service'; -const streamMock = jest.fn(); -const resizeMock = jest.fn(); -const imageMagickMock = () => { - return { stream: streamMock, resize: resizeMock, data: Readable.from('text') }; -}; -jest.mock('gm', () => { - return { - subClass: () => imageMagickMock, - }; -}); - const buildFileRecordWithParams = (mimeType: string, scanStatus?: ScanStatus) => { const parentId = new ObjectId().toHexString(); const parentSchoolId = new ObjectId().toHexString(); @@ -62,8 +51,8 @@ const defaultPreviewParamsWithWidth = { describe('PreviewService', () => { let module: TestingModule; let previewService: PreviewService; - let fileStorageService: DeepMocked; let s3ClientAdapter: DeepMocked; + let previewProducer: DeepMocked; beforeAll(async () => { await setupEntities([FileRecord]); @@ -83,514 +72,369 @@ describe('PreviewService', () => { provide: LegacyLogger, useValue: createMock(), }, + { provide: PreviewProducer, useValue: createMock() }, ], }).compile(); previewService = module.get(PreviewService); - fileStorageService = module.get(FilesStorageService); s3ClientAdapter = module.get(FILES_STORAGE_S3_CONNECTION); - }); - - beforeEach(() => { - jest.resetAllMocks(); + previewProducer = module.get(PreviewProducer); }); afterAll(async () => { await module.close(); }); - describe('getPreview is called', () => { + afterEach(() => { + jest.resetAllMocks(); + }); + + describe('download is called', () => { describe('WHEN preview is possbile', () => { describe('WHEN forceUpdate is true', () => { - describe('WHEN width and outputformat are not set', () => { - describe('WHEN download of original and preview file is successfull', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const orignalMimeType = 'image/png'; - const format = orignalMimeType.split('/')[1]; - const { fileRecord } = buildFileRecordWithParams(orignalMimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, {}); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, orignalMimeType); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file is successfull', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; - - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); - }); - - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(response).toEqual(previewFileResponse); - }); - }); - - describe('WHEN download of original file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const error = new Error('testError'); - fileStorageService.download.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); - - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); - }); - - describe('WHEN create of preview file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const error = new Error('testError'); - s3ClientAdapter.create.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; - }; + it('calls previewProducer.generate with correct params', async () => { + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = setup(); - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + await previewService.download(fileRecord, previewParams, bytesRange); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, }); }); - describe('WHEN get of preview file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { forceUpdate: true }; + it('calls S3ClientAdapters get method', async () => { + const { fileRecord, previewParams, previewPath } = setup(); - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); + await previewService.download(fileRecord, previewParams); - const error = new Error('testError'); - s3ClientAdapter.get.mockRejectedValueOnce(error); + expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + }); - return { fileRecord, downloadParams, previewParams, error }; - }; + it('returns preview file response', async () => { + const { fileRecord, previewParams, previewFileResponse } = setup(); - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + const response = await previewService.download(fileRecord, previewParams); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); + expect(response).toEqual(previewFileResponse); }); }); - describe('WHEN width and outputFormat are set', () => { - describe('WHEN download of original and preview file is successfull', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - forceUpdate: true, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, previewParams.outputFormat); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - resizeMock.mockClear(); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file throws error and second is successfull', () => { + const setup = (error: Error | NotFoundException) => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks resize method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(resizeMock).toHaveBeenCalledWith(previewParams.width, undefined, '>'); - expect(resizeMock).toHaveBeenCalledTimes(1); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); + describe('WHEN error is a NotFoundException', () => { + it('calls previewProducer.generate with correct params', async () => { + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = + setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, + }); + expect(previewProducer.generate).toHaveBeenCalledTimes(2); }); it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, previewPath } = setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(2); }); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + describe('WHEN error is other error', () => { + it('should pass error', async () => { + const error = new Error('testError'); + const { fileRecord, previewParams } = setup(error); - expect(response).toEqual(previewFileResponse); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(error); }); }); + }); - describe('WHEN download of original file throws error', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - - const error = new Error('testError'); - fileStorageService.download.mockRejectedValueOnce(error); - - return { fileRecord, downloadParams, previewParams, error }; + describe('WHEN both gets of preview file throw error', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + const notFoundException = new NotFoundException(); + s3ClientAdapter.get.mockRejectedValueOnce(notFoundException).mockRejectedValueOnce(notFoundException); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + it('should pass error', async () => { + const { fileRecord, previewParams } = setup(); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrowError( - error - ); - }); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(); }); }); }); describe('WHEN forceUpdate is false', () => { - describe('WHEN width and outputFormat are set', () => { - describe('WHEN S3ClientAdapter get returns already stored preview file', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - resizeMock.mockClear(); - streamMock.mockClear(); - - return { - fileRecord, - downloadParams, - previewParams, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file is successfull', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: false, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + it('calls S3ClientAdapters get method', async () => { + const { fileRecord, previewParams, previewPath } = setup(); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); - expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); - expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); - }); + expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); + expect(s3ClientAdapter.get).toHaveBeenCalledTimes(1); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); + it('returns preview file response', async () => { + const { fileRecord, previewParams, previewFileResponse } = setup(); - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + const response = await previewService.download(fileRecord, previewParams); - expect(response).toEqual(previewFileResponse); - }); + expect(response).toEqual(previewFileResponse); + }); - it('does not call image magicks resize and stream method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); + it('does not call generate', async () => { + const { fileRecord, previewParams, bytesRange } = setup(); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(resizeMock).not.toHaveBeenCalled(); - expect(streamMock).not.toHaveBeenCalled(); - }); + expect(previewProducer.generate).toHaveBeenCalledTimes(0); }); + }); - describe('WHEN S3ClientAdapter get throws NotFoundException', () => { - const setup = () => { - const bytesRange = 'bytes=0-100'; - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const error = new NotFoundException(); - s3ClientAdapter.get.mockRejectedValueOnce(error); - - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - - const previewFile = TestHelper.createFile(); - s3ClientAdapter.get.mockResolvedValueOnce(previewFile); - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const name = `${fileNameWithoutExtension}.${format}`; - const previewFileResponse = FileResponseBuilder.build(previewFile, name); - - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const previewFileDto = FileDtoBuilder.build(hash, previewFile.data, previewParams.outputFormat); - const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); - - streamMock.mockClear(); - streamMock.mockReturnValueOnce(previewFileDto.data); - - resizeMock.mockClear(); - - return { - bytesRange, - fileRecord, - downloadParams, - previewParams, - format, - previewFileDto, - previewPath, - previewFileResponse, - }; + describe('WHEN first get of preview file throws error and second is successfull', () => { + const setup = (error: Error | NotFoundException) => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: false, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + s3ClientAdapter.get.mockRejectedValueOnce(error).mockResolvedValueOnce(previewFile); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange); - - expect(fileStorageService.download).toHaveBeenCalledWith(fileRecord, downloadParams, bytesRange); - }); - - it('calls image magicks resize method', async () => { - const { fileRecord, downloadParams, previewParams } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(resizeMock).toHaveBeenCalledWith(previewParams.width, undefined, '>'); - expect(resizeMock).toHaveBeenCalledTimes(1); - }); - - it('calls image magicks stream method', async () => { - const { fileRecord, downloadParams, previewParams, format } = setup(); - - await previewService.getPreview(fileRecord, downloadParams, previewParams); - - expect(streamMock).toHaveBeenCalledWith(format); - expect(streamMock).toHaveBeenCalledTimes(1); - }); - - it('calls S3ClientAdapters create method', async () => { - const { fileRecord, downloadParams, previewParams, previewFileDto, previewPath } = setup(); + describe('WHEN error is a NotFoundException', () => { + it('calls previewProducer.generate with correct params', async () => { + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, bytesRange, originPath, previewPath, format } = + setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams, bytesRange); - expect(s3ClientAdapter.create).toHaveBeenCalledWith(previewPath, previewFileDto); - expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + expect(previewProducer.generate).toHaveBeenCalledWith({ + originFilePath: originPath, + previewFilePath: previewPath, + previewOptions: { width: previewParams.width, format }, + }); + expect(previewProducer.generate).toHaveBeenCalledTimes(1); }); it('calls S3ClientAdapters get method', async () => { - const { fileRecord, downloadParams, previewParams, previewPath } = setup(); + const notFoundException = new NotFoundException(); + const { fileRecord, previewParams, previewPath } = setup(notFoundException); - await previewService.getPreview(fileRecord, downloadParams, previewParams); + await previewService.download(fileRecord, previewParams); expect(s3ClientAdapter.get).toHaveBeenCalledWith(previewPath, undefined); expect(s3ClientAdapter.get).toHaveBeenCalledTimes(2); }); + }); - it('returns preview file response', async () => { - const { fileRecord, downloadParams, previewParams, previewFileResponse } = setup(); - - const response = await previewService.getPreview(fileRecord, downloadParams, previewParams); + describe('WHEN error is other error', () => { + it('should pass error', async () => { + const error = new Error('testError'); + const { fileRecord, previewParams } = setup(error); - expect(response).toEqual(previewFileResponse); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(error); }); }); + }); - describe('WHEN S3ClientAdapter get throws other than NotFoundException', () => { - const setup = () => { - const mimeType = 'image/png'; - const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { - ...defaultPreviewParamsWithWidth, - }; - const format = previewParams.outputFormat.split('/')[1]; - - const error = new Error('testError'); - s3ClientAdapter.get.mockRejectedValueOnce(error); - - return { - fileRecord, - downloadParams, - previewParams, - format, - error, - }; + describe('WHEN both gets of preview file throw error', () => { + const setup = () => { + const bytesRange = 'bytes=0-100'; + const mimeType = 'image/png'; + const { fileRecord } = buildFileRecordWithParams(mimeType); + const previewParams = { + ...defaultPreviewParamsWithWidth, + forceUpdate: true, }; + const format = previewParams.outputFormat.split('/')[1]; + + const previewFile = TestHelper.createFile(); + const notFoundException = new NotFoundException(); + s3ClientAdapter.get.mockRejectedValueOnce(notFoundException).mockRejectedValueOnce(notFoundException); + + const fileNameWithoutExtension = fileRecord.name.split('.')[0]; + const name = `${fileNameWithoutExtension}.${format}`; + const previewFileResponse = FileResponseBuilder.build(previewFile, name); + + const hash = createPreviewNameHash(fileRecord.id, previewParams); + const previewPath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const originPath = createPath(fileRecord.getSchoolId(), fileRecord.id); + + return { + bytesRange, + fileRecord, + previewParams, + format, + previewPath, + originPath, + previewFileResponse, + }; + }; - it('passes error', async () => { - const { fileRecord, downloadParams, previewParams, error } = setup(); + it('should pass error', async () => { + const { fileRecord, previewParams } = setup(); - await expect(previewService.getPreview(fileRecord, downloadParams, previewParams)).rejects.toThrow(error); - }); + await expect(previewService.download(fileRecord, previewParams)).rejects.toThrow(); }); }); }); @@ -603,33 +447,23 @@ describe('PreviewService', () => { const mimeType = 'application/zip'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, }; }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + it('passes error', async () => { + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -639,33 +473,23 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.PENDING); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, }; }; - it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + it('passes error', async () => { + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -675,21 +499,14 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.ERROR); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; - const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); + const previewParams = { ...defaultPreviewParams, forceUpdate: true }; const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, @@ -697,11 +514,9 @@ describe('PreviewService', () => { }; it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); @@ -711,21 +526,13 @@ describe('PreviewService', () => { const mimeType = 'image/png'; const format = mimeType.split('/')[1]; const { fileRecord } = buildFileRecordWithParams(mimeType, ScanStatus.BLOCKED); - const downloadParams = { - fileRecordId: fileRecord.id, - fileName: fileRecord.name, - }; const previewParams = { ...defaultPreviewParams, forceUpdate: true }; - const originalFileResponse = TestHelper.createFileResponse(); - fileStorageService.download.mockResolvedValueOnce(originalFileResponse); - const error = new UnprocessableEntityException(ErrorType.PREVIEW_NOT_POSSIBLE); return { bytesRange, fileRecord, - downloadParams, previewParams, format, error, @@ -733,11 +540,9 @@ describe('PreviewService', () => { }; it('calls download with correct params', async () => { - const { fileRecord, downloadParams, previewParams, bytesRange, error } = setup(); + const { fileRecord, previewParams, bytesRange, error } = setup(); - await expect( - previewService.getPreview(fileRecord, downloadParams, previewParams, bytesRange) - ).rejects.toThrowError(error); + await expect(previewService.download(fileRecord, previewParams, bytesRange)).rejects.toThrowError(error); }); }); }); @@ -789,10 +594,10 @@ describe('PreviewService', () => { }; }; - it('does not pass error', async () => { - const { fileRecord } = setup(); + it('should throw error', async () => { + const { fileRecord, error } = setup(); - await previewService.deletePreviews([fileRecord]); + await expect(previewService.deletePreviews([fileRecord])).rejects.toThrowError(error); }); }); }); diff --git a/apps/server/src/modules/files-storage/service/preview.service.ts b/apps/server/src/modules/files-storage/service/preview.service.ts index 5ca2093351b..e27fbc0645a 100644 --- a/apps/server/src/modules/files-storage/service/preview.service.ts +++ b/apps/server/src/modules/files-storage/service/preview.service.ts @@ -1,64 +1,45 @@ import { Inject, Injectable, NotFoundException, UnprocessableEntityException } from '@nestjs/common'; +import { PreviewProducer } from '@shared/infra/preview-generator'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { LegacyLogger } from '@src/core/logger'; -import { subClass } from 'gm'; -import { PassThrough } from 'stream'; -import { DownloadFileParams, PreviewParams } from '../controller/dto'; +import { PreviewParams } from '../controller/dto'; import { FileRecord, PreviewStatus } from '../entity'; import { ErrorType } from '../error'; import { FILES_STORAGE_S3_CONNECTION } from '../files-storage.config'; -import { createPreviewDirectoryPath, createPreviewFilePath, createPreviewNameHash } from '../helper'; +import { createPreviewDirectoryPath, getPreviewName } from '../helper'; import { GetFileResponse, PreviewFileParams } from '../interface'; -import { PreviewOutputMimeTypes } from '../interface/preview-output-mime-types.enum'; -import { FileDtoBuilder, FileResponseBuilder } from '../mapper'; -import { FilesStorageService } from './files-storage.service'; +import { FileResponseBuilder, PreviewBuilder } from '../mapper'; @Injectable() export class PreviewService { constructor( @Inject(FILES_STORAGE_S3_CONNECTION) private readonly storageClient: S3ClientAdapter, - private readonly fileStorageService: FilesStorageService, - private logger: LegacyLogger + private logger: LegacyLogger, + private readonly previewProducer: PreviewProducer ) { this.logger.setContext(PreviewService.name); } - public async getPreview( + public async download( fileRecord: FileRecord, - downloadParams: DownloadFileParams, previewParams: PreviewParams, bytesRange?: string ): Promise { this.checkIfPreviewPossible(fileRecord); - const hash = createPreviewNameHash(fileRecord.id, previewParams); - const filePath = createPreviewFilePath(fileRecord.getSchoolId(), hash, fileRecord.id); + const previewFileParams = PreviewBuilder.buildParams(fileRecord, previewParams, bytesRange); - let response: GetFileResponse; - - const previewFileParams = { fileRecord, downloadParams, previewParams, hash, filePath, bytesRange }; - - if (previewParams.forceUpdate) { - response = await this.generatePreview(previewFileParams); - } else { - response = await this.tryGetPreviewOrGenerate(previewFileParams); - } + const response = await this.tryGetPreviewOrGenerate(previewFileParams); return response; } public async deletePreviews(fileRecords: FileRecord[]): Promise { - try { - const paths = fileRecords.map((fileRecord) => - createPreviewDirectoryPath(fileRecord.getSchoolId(), fileRecord.id) - ); + const paths = fileRecords.map((fileRecord) => createPreviewDirectoryPath(fileRecord.getSchoolId(), fileRecord.id)); - const promises = paths.map((path) => this.storageClient.deleteDirectory(path)); + const promises = paths.map((path) => this.storageClient.deleteDirectory(path)); - await Promise.all(promises); - } catch (error) { - this.logger.warn(error); - } + await Promise.all(promises); } private checkIfPreviewPossible(fileRecord: FileRecord): void | UnprocessableEntityException { @@ -72,79 +53,36 @@ export class PreviewService { let file: GetFileResponse; try { + if (params.previewParams.forceUpdate) { + await this.generatePreview(params); + } + file = await this.getPreviewFile(params); } catch (error) { if (!(error instanceof NotFoundException)) { throw error; } - file = await this.generatePreview(params); + await this.generatePreview(params); + file = await this.getPreviewFile(params); } return file; } private async getPreviewFile(params: PreviewFileParams): Promise { - const { fileRecord, filePath, bytesRange, previewParams } = params; - const name = this.getPreviewName(fileRecord, previewParams.outputFormat); - const file = await this.storageClient.get(filePath, bytesRange); + const { fileRecord, previewFilePath, bytesRange, previewParams } = params; + const name = getPreviewName(fileRecord, previewParams.outputFormat); + const file = await this.storageClient.get(previewFilePath, bytesRange); const response = FileResponseBuilder.build(file, name); return response; } - private async generatePreview(params: PreviewFileParams): Promise { - const { fileRecord, downloadParams, previewParams, hash, filePath, bytesRange } = params; - - const original = await this.fileStorageService.download(fileRecord, downloadParams, bytesRange); - const preview = this.resizeAndConvert(original, fileRecord, previewParams); - - const format = previewParams.outputFormat ?? fileRecord.mimeType; - const fileDto = FileDtoBuilder.build(hash, preview, format); - await this.storageClient.create(filePath, fileDto); - - const response = await this.getPreviewFile(params); - - return response; - } - - private resizeAndConvert( - original: GetFileResponse, - fileRecord: FileRecord, - previewParams: PreviewParams - ): PassThrough { - const mimeType = previewParams.outputFormat ?? fileRecord.mimeType; - const format = this.getFormat(mimeType); - const im = subClass({ imageMagick: '7+' }); - - const preview = im(original.data, fileRecord.name); - const { width } = previewParams; - - if (width) { - preview.resize(width, undefined, '>'); - } - - const result = preview.stream(format); - - return result; - } - - private getFormat(mimeType: string): string { - const format = mimeType.split('/')[1]; - - return format; - } - - private getPreviewName(fileRecord: FileRecord, outputFormat?: PreviewOutputMimeTypes): string { - if (!outputFormat) { - return fileRecord.name; - } - - const fileNameWithoutExtension = fileRecord.name.split('.')[0]; - const format = this.getFormat(outputFormat); - const name = `${fileNameWithoutExtension}.${format}`; + private async generatePreview(params: PreviewFileParams): Promise { + const payload = PreviewBuilder.buildPayload(params); - return name; + await this.previewProducer.generate(payload); } } diff --git a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts index a1aaf0342ee..b12006367aa 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-delete.uc.spec.ts @@ -1,5 +1,6 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ObjectId } from '@mikro-orm/mongodb'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { ForbiddenException } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; @@ -8,7 +9,6 @@ import { AntivirusService } from '@shared/infra/antivirus'; import { S3ClientAdapter } from '@shared/infra/s3-client'; import { fileRecordFactory, setupEntities } from '@shared/testing'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { FileRecordParams } from '../controller/dto'; import { FileRecord, FileRecordParentType } from '../entity'; import { FileStorageAuthorizationContext } from '../files-storage.const'; @@ -123,7 +123,7 @@ describe('FilesStorageUC delete methods', () => { const mockedResult = [[fileRecord], 0] as Counted; authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); - filesStorageService.deleteFilesOfParent.mockResolvedValueOnce(mockedResult); + filesStorageService.getFileRecordsOfParent.mockResolvedValueOnce(mockedResult); return { params, userId, mockedResult, requestParams, fileRecord }; }; @@ -143,11 +143,11 @@ describe('FilesStorageUC delete methods', () => { }); it('should call service with correct params', async () => { - const { requestParams, userId } = setup(); + const { requestParams, userId, fileRecord } = setup(); await filesStorageUC.deleteFilesOfParent(userId, requestParams); - expect(filesStorageService.deleteFilesOfParent).toHaveBeenCalledWith(requestParams.parentId); + expect(filesStorageService.deleteFilesOfParent).toHaveBeenCalledWith([fileRecord]); }); it('should call deletePreviews', async () => { @@ -189,10 +189,14 @@ describe('FilesStorageUC delete methods', () => { describe('WHEN service throws error', () => { const setup = () => { + const { fileRecords } = buildFileRecordsWithParams(); + + authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); const { requestParams, userId } = createParams(); const error = new Error('test'); authorizationReferenceService.checkPermissionByReferences.mockResolvedValueOnce(); + filesStorageService.getFileRecordsOfParent.mockResolvedValueOnce([fileRecords, fileRecords.length]); filesStorageService.deleteFilesOfParent.mockRejectedValueOnce(error); return { requestParams, userId, error }; diff --git a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts index 81b54553d1e..f0aa9dcc25a 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage-download-preview.uc.spec.ts @@ -111,7 +111,7 @@ describe('FilesStorageUC', () => { const previewFileResponse = TestHelper.createFileResponse(); filesStorageService.getFileRecord.mockResolvedValueOnce(fileRecord); - previewService.getPreview.mockResolvedValueOnce(previewFileResponse); + previewService.download.mockResolvedValueOnce(previewFileResponse); return { fileDownloadParams, previewParams, userId, fileRecord, singleFileParams, previewFileResponse }; }; @@ -129,12 +129,7 @@ describe('FilesStorageUC', () => { await filesStorageUC.downloadPreview(userId, fileDownloadParams, previewParams); - expect(previewService.getPreview).toHaveBeenCalledWith( - fileRecord, - fileDownloadParams, - previewParams, - undefined - ); + expect(previewService.download).toHaveBeenCalledWith(fileRecord, previewParams, undefined); }); it('should call checkPermission with correct params', async () => { @@ -211,7 +206,7 @@ describe('FilesStorageUC', () => { filesStorageService.getFileRecord.mockResolvedValueOnce(fileRecord); const error = new Error('test'); - previewService.getPreview.mockRejectedValueOnce(error); + previewService.download.mockRejectedValueOnce(error); return { fileDownloadParams, previewParams, userId, error }; }; diff --git a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts index 833d7575bdf..47eb7c79104 100644 --- a/apps/server/src/modules/files-storage/uc/files-storage.uc.ts +++ b/apps/server/src/modules/files-storage/uc/files-storage.uc.ts @@ -1,9 +1,9 @@ +import { AuthorizationContext } from '@modules/authorization'; +import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { HttpService } from '@nestjs/axios'; import { Injectable, NotFoundException } from '@nestjs/common'; import { Counted, EntityId } from '@shared/domain'; import { LegacyLogger } from '@src/core/logger'; -import { AuthorizationContext } from '@modules/authorization'; -import { AuthorizationReferenceService } from '@modules/authorization/domain'; import { AxiosRequestConfig, AxiosResponse } from 'axios'; import busboy from 'busboy'; import { Request } from 'express'; @@ -154,7 +154,9 @@ export class FilesStorageUC { await this.checkPermission(userId, parentType, parentId, FileStorageAuthorizationContext.read); - const result = this.previewService.getPreview(fileRecord, params, previewParams, bytesRange); + this.filesStorageService.checkFileName(fileRecord, params); + + const result = this.previewService.download(fileRecord, previewParams, bytesRange); return result; } @@ -162,8 +164,9 @@ export class FilesStorageUC { // delete public async deleteFilesOfParent(userId: EntityId, params: FileRecordParams): Promise> { await this.checkPermission(userId, params.parentType, params.parentId, FileStorageAuthorizationContext.delete); - const [fileRecords, count] = await this.filesStorageService.deleteFilesOfParent(params.parentId); + const [fileRecords, count] = await this.filesStorageService.getFileRecordsOfParent(params.parentId); await this.previewService.deletePreviews(fileRecords); + await this.filesStorageService.deleteFilesOfParent(fileRecords); return [fileRecords, count]; } @@ -173,8 +176,8 @@ export class FilesStorageUC { const { parentType, parentId } = fileRecord.getParentInfo(); await this.checkPermission(userId, parentType, parentId, FileStorageAuthorizationContext.delete); - await this.filesStorageService.delete([fileRecord]); await this.previewService.deletePreviews([fileRecord]); + await this.filesStorageService.delete([fileRecord]); return fileRecord; } diff --git a/apps/server/src/shared/infra/preview-generator/index.ts b/apps/server/src/shared/infra/preview-generator/index.ts new file mode 100644 index 00000000000..570b38cc9bd --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/index.ts @@ -0,0 +1,4 @@ +export * from './interface'; +export * from './preview-generator-consumer.module'; +export * from './preview-generator-producer.module'; +export * from './preview.producer'; diff --git a/apps/server/src/shared/infra/preview-generator/interface/index.ts b/apps/server/src/shared/infra/preview-generator/interface/index.ts new file mode 100644 index 00000000000..37aae418ee2 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/index.ts @@ -0,0 +1 @@ +export * from './preview'; diff --git a/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts b/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts new file mode 100644 index 00000000000..2924fc945bc --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/preview-consumer-config.ts @@ -0,0 +1,11 @@ +import { S3Config } from '@shared/infra/s3-client'; + +export interface PreviewModuleConfig { + NEST_LOG_LEVEL: string; + INCOMING_REQUEST_TIMEOUT: number; +} + +export interface PreviewConfig { + storageConfig: S3Config; + serverConfig: PreviewModuleConfig; +} diff --git a/apps/server/src/shared/infra/preview-generator/interface/preview.ts b/apps/server/src/shared/infra/preview-generator/interface/preview.ts new file mode 100644 index 00000000000..92ab2808151 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/interface/preview.ts @@ -0,0 +1,15 @@ +export interface PreviewOptions { + format: string; + width?: number; +} + +export interface PreviewFileOptions { + originFilePath: string; + previewFilePath: string; + previewOptions: PreviewOptions; +} + +export interface PreviewResponseMessage { + previewFilePath: string; + status: boolean; +} diff --git a/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts new file mode 100644 index 00000000000..04d56a991f1 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.spec.ts @@ -0,0 +1,37 @@ +import { PreviewActionsLoggable } from './preview-actions.loggable'; + +describe('PreviewActionsLoggable', () => { + describe('getLogMessage is called', () => { + const setup = () => { + const message = 'message'; + const payload = { + originFilePath: 'originFilePath', + previewFilePath: 'previewFilePath', + previewOptions: { + format: 'webp', + width: 100, + }, + }; + + const expectedResponse = { + message, + data: { + originFilePath: payload.originFilePath, + previewFilePath: payload.previewFilePath, + format: payload.previewOptions.format, + width: payload.previewOptions.width, + }, + }; + + return { message, payload, expectedResponse }; + }; + + it('should return log message', () => { + const { message, payload, expectedResponse } = setup(); + + const result = new PreviewActionsLoggable(message, payload).getLogMessage(); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts new file mode 100644 index 00000000000..e98f21d09be --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/loggable/preview-actions.loggable.ts @@ -0,0 +1,19 @@ +import { LogMessage, Loggable } from '@src/core/logger'; +import { PreviewFileOptions } from '../interface'; + +export class PreviewActionsLoggable implements Loggable { + constructor(private readonly message: string, private readonly payload: PreviewFileOptions) {} + + getLogMessage(): LogMessage { + const { originFilePath, previewFilePath, previewOptions } = this.payload; + return { + message: this.message, + data: { + originFilePath, + previewFilePath, + format: previewOptions.format, + width: previewOptions.width, + }, + }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts b/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts new file mode 100644 index 00000000000..9d352b81d9d --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator-consumer.module.ts @@ -0,0 +1,36 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { RabbitMQWrapperModule } from '@shared/infra/rabbitmq'; +import { S3ClientAdapter, S3ClientModule } from '@shared/infra/s3-client'; +import { createConfigModuleOptions } from '@src/config'; +import { Logger, LoggerModule } from '@src/core/logger'; +import { PreviewConfig } from './interface/preview-consumer-config'; +import { PreviewGeneratorConsumer } from './preview-generator.consumer'; +import { PreviewGeneratorService } from './preview-generator.service'; + +@Module({}) +export class PreviewGeneratorConsumerModule { + static register(config: PreviewConfig): DynamicModule { + const { storageConfig, serverConfig } = config; + const providers = [ + { + provide: PreviewGeneratorService, + useFactory: (logger: Logger, storageClient: S3ClientAdapter) => + new PreviewGeneratorService(storageClient, logger), + inject: [Logger, storageConfig.connectionName], + }, + PreviewGeneratorConsumer, + ]; + + return { + module: PreviewGeneratorConsumerModule, + imports: [ + LoggerModule, + S3ClientModule.register([storageConfig]), + RabbitMQWrapperModule, + ConfigModule.forRoot(createConfigModuleOptions(() => serverConfig)), + ], + providers, + }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts b/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts new file mode 100644 index 00000000000..d3f65299657 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator-producer.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { LoggerModule } from '@src/core/logger'; +import { RabbitMQWrapperModule } from '../rabbitmq'; +import { PreviewProducer } from './preview.producer'; + +@Module({ + imports: [LoggerModule, RabbitMQWrapperModule], + providers: [PreviewProducer], + exports: [PreviewProducer], +}) +export class PreviewGeneratorProducerModule {} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts new file mode 100644 index 00000000000..2caaad7abe8 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.spec.ts @@ -0,0 +1,28 @@ +import { PassThrough } from 'stream'; +import { PreviewGeneratorBuilder } from './preview-generator.builder'; + +describe('PreviewGeneratorBuilder', () => { + describe('buildFile is called', () => { + const setup = () => { + const preview = new PassThrough(); + const previewOptions = { + format: 'webp', + }; + + const expectedResponse = { + data: preview, + mimeType: previewOptions.format, + }; + + return { preview, previewOptions, expectedResponse }; + }; + + it('should return preview file', () => { + const { preview, previewOptions, expectedResponse } = setup(); + + const result = PreviewGeneratorBuilder.buildFile(preview, previewOptions); + + expect(result).toEqual(expectedResponse); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts new file mode 100644 index 00000000000..4c5561ed089 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.builder.ts @@ -0,0 +1,16 @@ +import { File } from '@shared/infra/s3-client'; +import { PassThrough } from 'stream'; +import { PreviewOptions } from './interface'; + +export class PreviewGeneratorBuilder { + public static buildFile(preview: PassThrough, previewOptions: PreviewOptions): File { + const { format } = previewOptions; + + const file = { + data: preview, + mimeType: format, + }; + + return file; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts new file mode 100644 index 00000000000..b1a24a30a57 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.spec.ts @@ -0,0 +1,80 @@ +import { DeepMocked, createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { Logger } from '@src/core/logger'; +import { PreviewFileOptions, PreviewResponseMessage } from './interface'; +import { PreviewGeneratorConsumer } from './preview-generator.consumer'; +import { PreviewGeneratorService } from './preview-generator.service'; + +describe('PreviewGeneratorConsumer', () => { + let module: TestingModule; + let previewGeneratorService: DeepMocked; + let service: PreviewGeneratorConsumer; + + beforeAll(async () => { + module = await Test.createTestingModule({ + providers: [ + PreviewGeneratorConsumer, + { + provide: PreviewGeneratorService, + useValue: createMock(), + }, + { + provide: Logger, + useValue: createMock(), + }, + ], + }).compile(); + + previewGeneratorService = module.get(PreviewGeneratorService); + service = module.get(PreviewGeneratorConsumer); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + afterAll(async () => { + await module.close(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generatePreview()', () => { + const setup = () => { + const payload: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const response: PreviewResponseMessage = { + previewFilePath: payload.previewFilePath, + status: true, + }; + previewGeneratorService.generatePreview.mockResolvedValueOnce(response); + + return { payload, response }; + }; + + it('should call previewGeneratorService.generatePreview with payload', async () => { + const { payload } = setup(); + + await service.generatePreview(payload); + + expect(previewGeneratorService.generatePreview).toBeCalledWith(payload); + }); + + it('should return expected value', async () => { + const { payload, response } = setup(); + + const result = await service.generatePreview(payload); + + expect(result).toEqual({ message: response }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts new file mode 100644 index 00000000000..8fc08d261f3 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.consumer.ts @@ -0,0 +1,27 @@ +import { RabbitPayload, RabbitRPC } from '@golevelup/nestjs-rabbitmq'; +import { Injectable } from '@nestjs/common'; +import { Logger } from '@src/core/logger'; +import { FilesPreviewEvents, FilesPreviewExchange } from '@src/shared/infra/rabbitmq'; +import { PreviewFileOptions } from './interface'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; +import { PreviewGeneratorService } from './preview-generator.service'; + +@Injectable() +export class PreviewGeneratorConsumer { + constructor(private readonly previewGeneratorService: PreviewGeneratorService, private logger: Logger) { + this.logger.setContext(PreviewGeneratorConsumer.name); + } + + @RabbitRPC({ + exchange: FilesPreviewExchange, + routingKey: FilesPreviewEvents.GENERATE_PREVIEW, + queue: FilesPreviewEvents.GENERATE_PREVIEW, + }) + public async generatePreview(@RabbitPayload() payload: PreviewFileOptions) { + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorConsumer.generatePreview', payload)); + + const response = await this.previewGeneratorService.generatePreview(payload); + + return { message: response }; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts new file mode 100644 index 00000000000..b8eeea612f5 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.service.spec.ts @@ -0,0 +1,151 @@ +import { DeepMocked, createMock } from '@golevelup/ts-jest'; +import { Test, TestingModule } from '@nestjs/testing'; +import { GetFile, S3ClientAdapter } from '@shared/infra/s3-client'; +import { Logger } from '@src/core/logger'; +import { Readable } from 'node:stream'; +import { PreviewGeneratorService } from './preview-generator.service'; + +const streamMock = jest.fn(); +const resizeMock = jest.fn(); +const imageMagickMock = () => { + return { stream: streamMock, resize: resizeMock, data: Readable.from('text') }; +}; +jest.mock('gm', () => { + return { + subClass: () => imageMagickMock, + }; +}); + +const createFile = (contentRange?: string): GetFile => { + const text = 'testText'; + const readable = Readable.from(text); + + const fileResponse = { + data: readable, + contentType: 'image/jpeg', + contentLength: text.length, + contentRange, + etag: 'testTag', + }; + + return fileResponse; +}; + +describe('PreviewGeneratorService', () => { + let module: TestingModule; + let service: PreviewGeneratorService; + let s3ClientAdapter: DeepMocked; + + beforeAll(async () => { + module = await Test.createTestingModule({ + providers: [ + PreviewGeneratorService, + { + provide: S3ClientAdapter, + useValue: createMock(), + }, + { + provide: Logger, + useValue: createMock(), + }, + ], + }).compile(); + + service = module.get(PreviewGeneratorService); + s3ClientAdapter = module.get(S3ClientAdapter); + }); + + afterAll(async () => { + await module.close(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generatePreview', () => { + const setup = (width = 500) => { + const params = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width, + }, + }; + const originFile = createFile(); + s3ClientAdapter.get.mockResolvedValueOnce(originFile); + + const data = Readable.from('text'); + streamMock.mockReturnValueOnce(data); + + const expectedFileData = { + data, + mimeType: params.previewOptions.format, + }; + + return { params, originFile, expectedFileData }; + }; + + describe('WHEN download of original and preview file is successful', () => { + it('should call storageClient get method with originFilePath', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(s3ClientAdapter.get).toBeCalledWith(params.originFilePath); + }); + + it('should call imagemagicks resize method', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(resizeMock).toHaveBeenCalledWith(params.previewOptions.width, undefined, '>'); + expect(resizeMock).toHaveBeenCalledTimes(1); + }); + + it('should call imagemagicks stream method', async () => { + const { params } = setup(); + + await service.generatePreview(params); + + expect(streamMock).toHaveBeenCalledWith(params.previewOptions.format); + expect(streamMock).toHaveBeenCalledTimes(1); + }); + + it('should call S3ClientAdapters create method', async () => { + const { params, expectedFileData } = setup(); + + await service.generatePreview(params); + + expect(s3ClientAdapter.create).toHaveBeenCalledWith(params.previewFilePath, expectedFileData); + expect(s3ClientAdapter.create).toHaveBeenCalledTimes(1); + }); + + it('should should return values', async () => { + const { params } = setup(); + const expectedValue = { previewFilePath: params.previewFilePath, status: true }; + + const result = await service.generatePreview(params); + + expect(result).toEqual(expectedValue); + }); + }); + + describe('WHEN previewParams.width not set', () => { + it('should not call imagemagicks resize method', async () => { + const { params } = setup(0); + + await service.generatePreview(params); + + expect(resizeMock).not.toHaveBeenCalledWith(params.previewOptions.width, undefined, '>'); + expect(resizeMock).not.toHaveBeenCalledTimes(1); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts b/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts new file mode 100644 index 00000000000..72dac25f076 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview-generator.service.ts @@ -0,0 +1,56 @@ +import { Injectable } from '@nestjs/common'; +import { GetFile, S3ClientAdapter } from '@shared/infra/s3-client'; +import { Logger } from '@src/core/logger'; +import { subClass } from 'gm'; +import { PassThrough } from 'stream'; +import { PreviewFileOptions, PreviewOptions, PreviewResponseMessage } from './interface'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; +import { PreviewGeneratorBuilder } from './preview-generator.builder'; + +@Injectable() +export class PreviewGeneratorService { + private imageMagick = subClass({ imageMagick: '7+' }); + + constructor(private readonly storageClient: S3ClientAdapter, private logger: Logger) { + this.logger.setContext(PreviewGeneratorService.name); + } + + public async generatePreview(params: PreviewFileOptions): Promise { + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorService.generatePreview:start', params)); + const { originFilePath, previewFilePath, previewOptions } = params; + + const original = await this.downloadOriginFile(originFilePath); + const preview = this.resizeAndConvert(original, previewOptions); + + const file = PreviewGeneratorBuilder.buildFile(preview, params.previewOptions); + + await this.storageClient.create(previewFilePath, file); + + this.logger.debug(new PreviewActionsLoggable('PreviewGeneratorService.generatePreview:end', params)); + + return { + previewFilePath, + status: true, + }; + } + + private async downloadOriginFile(pathToFile: string): Promise { + const file = await this.storageClient.get(pathToFile); + + return file; + } + + private resizeAndConvert(original: GetFile, previewParams: PreviewOptions): PassThrough { + const { format, width } = previewParams; + + const preview = this.imageMagick(original.data); + + if (width) { + preview.resize(width, undefined, '>'); + } + + const result = preview.stream(format); + + return result; + } +} diff --git a/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts b/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts new file mode 100644 index 00000000000..47adea158a6 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview.producer.spec.ts @@ -0,0 +1,128 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { InternalServerErrorException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; +import { setupEntities } from '@shared/testing'; +import { Logger } from '@src/core/logger'; +import { ErrorMapper, FilesPreviewEvents, FilesPreviewExchange } from '../rabbitmq'; +import { PreviewFileOptions } from './interface'; +import { PreviewProducer } from './preview.producer'; + +describe('PreviewProducer', () => { + let module: TestingModule; + let service: PreviewProducer; + let configService: DeepMocked; + let amqpConnection: DeepMocked; + + beforeAll(async () => { + await setupEntities(); + module = await Test.createTestingModule({ + providers: [ + PreviewProducer, + { + provide: Logger, + useValue: createMock(), + }, + { + provide: AmqpConnection, + useValue: createMock(), + }, + { + provide: ConfigService, + useValue: createMock(), + }, + ], + }).compile(); + + service = module.get(PreviewProducer); + amqpConnection = module.get(AmqpConnection); + configService = module.get(ConfigService); + }); + + afterAll(async () => { + await module.close(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generate', () => { + describe('when valid params are passed and amqp connection return with a message', () => { + const setup = () => { + const timeout = 10000; + + const params: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const message = []; + amqpConnection.request.mockResolvedValueOnce({ message }); + configService.get.mockReturnValue(timeout); + + const expectedParams = { + exchange: FilesPreviewExchange, + routingKey: FilesPreviewEvents.GENERATE_PREVIEW, + payload: params, + timeout, + }; + + return { params, expectedParams, message }; + }; + + it('should call the ampqConnection.', async () => { + const { params, expectedParams } = setup(); + + await service.generate(params); + + expect(amqpConnection.request).toHaveBeenCalledWith(expectedParams); + }); + + it('should return the response message.', async () => { + const { params, message } = setup(); + + const res = await service.generate(params); + + expect(res).toEqual(message); + }); + }); + + describe('when amqpConnection return with error in response', () => { + const setup = () => { + const params: PreviewFileOptions = { + originFilePath: 'file/test.jpeg', + previewFilePath: 'preview/text.webp', + previewOptions: { + format: 'webp', + width: 500, + }, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockResolvedValueOnce({ error }); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.generate(params)).rejects.toThrowError( + new InternalServerErrorException(null, { cause: error }) + ); + expect(spy).toBeCalled(); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/preview-generator/preview.producer.ts b/apps/server/src/shared/infra/preview-generator/preview.producer.ts new file mode 100644 index 00000000000..602e2503185 --- /dev/null +++ b/apps/server/src/shared/infra/preview-generator/preview.producer.ts @@ -0,0 +1,31 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { FilesPreviewEvents, FilesPreviewExchange, RpcMessageProducer } from '@shared/infra/rabbitmq'; +import { Logger } from '@src/core/logger'; +import { PreviewFileOptions, PreviewResponseMessage } from './interface'; +import { PreviewModuleConfig } from './interface/preview-consumer-config'; +import { PreviewActionsLoggable } from './loggable/preview-actions.loggable'; + +@Injectable() +export class PreviewProducer extends RpcMessageProducer { + constructor( + protected readonly amqpConnection: AmqpConnection, + private readonly logger: Logger, + protected readonly configService: ConfigService + ) { + const timeout = configService.get('INCOMING_REQUEST_TIMEOUT'); + + super(amqpConnection, FilesPreviewExchange, timeout); + this.logger.setContext(PreviewProducer.name); + } + + async generate(payload: PreviewFileOptions): Promise { + this.logger.debug(new PreviewActionsLoggable('PreviewProducer.generate:started', payload)); + const response = await this.request(FilesPreviewEvents.GENERATE_PREVIEW, payload); + + this.logger.debug(new PreviewActionsLoggable('PreviewProducer.generate:finished', payload)); + + return response; + } +} diff --git a/apps/server/src/modules/files-storage-client/mapper/error.mapper.spec.ts b/apps/server/src/shared/infra/rabbitmq/error.mapper.spec.ts similarity index 100% rename from apps/server/src/modules/files-storage-client/mapper/error.mapper.spec.ts rename to apps/server/src/shared/infra/rabbitmq/error.mapper.spec.ts diff --git a/apps/server/src/modules/files-storage-client/mapper/error.mapper.ts b/apps/server/src/shared/infra/rabbitmq/error.mapper.ts similarity index 100% rename from apps/server/src/modules/files-storage-client/mapper/error.mapper.ts rename to apps/server/src/shared/infra/rabbitmq/error.mapper.ts diff --git a/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts b/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts new file mode 100644 index 00000000000..0bab7491e07 --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/exchange/files-preview.ts @@ -0,0 +1,7 @@ +import { Configuration } from '@hpi-schul-cloud/commons/lib'; + +export const FilesPreviewExchange = Configuration.get('FILES_STORAGE__EXCHANGE') as string; + +export enum FilesPreviewEvents { + 'GENERATE_PREVIEW' = 'generate-preview', +} diff --git a/apps/server/src/shared/infra/rabbitmq/exchange/index.ts b/apps/server/src/shared/infra/rabbitmq/exchange/index.ts index f0d84c9e1a1..0cf6bd00d13 100644 --- a/apps/server/src/shared/infra/rabbitmq/exchange/index.ts +++ b/apps/server/src/shared/infra/rabbitmq/exchange/index.ts @@ -1 +1,2 @@ +export * from './files-preview'; export * from './files-storage'; diff --git a/apps/server/src/shared/infra/rabbitmq/index.ts b/apps/server/src/shared/infra/rabbitmq/index.ts index 0183c37b284..99f5887b9a8 100644 --- a/apps/server/src/shared/infra/rabbitmq/index.ts +++ b/apps/server/src/shared/infra/rabbitmq/index.ts @@ -1,3 +1,5 @@ +export * from './error.mapper'; export * from './exchange'; export * from './rabbitmq.module'; export * from './rpc-message'; +export * from './rpc-message-producer'; diff --git a/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts b/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts index 2d73162e6e3..946b642e779 100644 --- a/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts +++ b/apps/server/src/shared/infra/rabbitmq/rabbitmq.module.ts @@ -1,7 +1,7 @@ import { AmqpConnectionManager, RabbitMQModule } from '@golevelup/nestjs-rabbitmq'; import { Configuration } from '@hpi-schul-cloud/commons'; import { Global, Module, OnModuleDestroy } from '@nestjs/common'; -import { FilesStorageExchange } from './exchange'; +import { FilesPreviewExchange, FilesStorageExchange } from './exchange'; /** * https://www.npmjs.com/package/@golevelup/nestjs-rabbitmq#usage @@ -28,6 +28,10 @@ const imports = [ name: FilesStorageExchange, type: 'direct', }, + { + name: FilesPreviewExchange, + type: 'direct', + }, ], uri: Configuration.get('RABBITMQ_URI') as string, }), diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts new file mode 100644 index 00000000000..b2e94ea676b --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.spec.ts @@ -0,0 +1,130 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { ErrorMapper, RpcMessageProducer } from '.'; + +interface TestPayload { + value: boolean; +} + +interface TestResponse { + value: boolean; +} + +const TestEvent = 'test-event'; +const TestExchange = 'test-exchange'; +const timeout = 1000; + +class RpcMessageProducerImp extends RpcMessageProducer { + constructor(protected readonly amqpConnection: AmqpConnection) { + super(amqpConnection, TestExchange, timeout); + } + + async testRequest(payload: TestPayload): Promise { + const response = await this.request(TestEvent, payload); + + return response; + } +} + +describe('RpcMessageProducer', () => { + let service: RpcMessageProducerImp; + let amqpConnection: DeepMocked; + + beforeAll(() => { + amqpConnection = createMock(); + + service = new RpcMessageProducerImp(amqpConnection); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('generate', () => { + describe('when valid params are passed and amqp connection return with a message', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const message = []; + amqpConnection.request.mockResolvedValueOnce({ message }); + + const expectedParams = { + exchange: TestExchange, + routingKey: TestEvent, + payload: params, + timeout, + }; + + return { params, expectedParams, message }; + }; + + it('should call the ampqConnection.', async () => { + const { params, expectedParams } = setup(); + + await service.testRequest(params); + + expect(amqpConnection.request).toHaveBeenCalledWith(expectedParams); + }); + + it('should return the response message.', async () => { + const { params, message } = setup(); + + const res = await service.testRequest(params); + + expect(res).toEqual(message); + }); + }); + + describe('when amqpConnection return with error in response', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockResolvedValueOnce({ error }); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.testRequest(params)).rejects.toThrowError( + ErrorMapper.mapRpcErrorResponseToDomainError(error) + ); + expect(spy).toBeCalled(); + }); + }); + + describe('when amqpConnection throw an error', () => { + const setup = () => { + const params: TestPayload = { + value: true, + }; + + const error = new Error('An error from called service'); + + amqpConnection.request.mockRejectedValueOnce(error); + const spy = jest.spyOn(ErrorMapper, 'mapRpcErrorResponseToDomainError'); + + return { params, spy, error }; + }; + + it('should call error mapper and throw with error', async () => { + const { params, spy, error } = setup(); + + await expect(service.testRequest(params)).rejects.toThrowError(error); + expect(spy).not.toBeCalled(); + }); + }); + }); +}); diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts new file mode 100644 index 00000000000..8a239b2cbcd --- /dev/null +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message-producer.ts @@ -0,0 +1,37 @@ +import { AmqpConnection } from '@golevelup/nestjs-rabbitmq'; +import { ErrorMapper } from './error.mapper'; +import { RpcMessage } from './rpc-message'; + +export abstract class RpcMessageProducer { + constructor( + protected readonly amqpConnection: AmqpConnection, + protected readonly exchange: string, + protected readonly timeout: number + ) {} + + protected async request(event: string, payload: unknown) { + const response = await this.amqpConnection.request>(this.createRequest(event, payload)); + + this.checkError(response); + return response.message; + } + + // need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984 + // mapRpcErrorResponseToDomainError should also removed with this ticket + protected checkError(response: RpcMessage) { + const { error } = response; + if (error) { + const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error); + throw domainError; + } + } + + protected createRequest(event: string, payload: unknown) { + return { + exchange: this.exchange, + routingKey: event, + payload, + timeout: this.timeout, + }; + } +} diff --git a/apps/server/src/shared/infra/rabbitmq/rpc-message.ts b/apps/server/src/shared/infra/rabbitmq/rpc-message.ts index 0d512e73b4a..c7e0e7de41f 100644 --- a/apps/server/src/shared/infra/rabbitmq/rpc-message.ts +++ b/apps/server/src/shared/infra/rabbitmq/rpc-message.ts @@ -1,6 +1,6 @@ export interface IError extends Error { status?: number; - message: never; + message: string; } export interface RpcMessage { message: T; diff --git a/apps/server/src/shared/infra/s3-client/interface/index.ts b/apps/server/src/shared/infra/s3-client/interface/index.ts index dc4b76ad922..ad6ed9c81da 100644 --- a/apps/server/src/shared/infra/s3-client/interface/index.ts +++ b/apps/server/src/shared/infra/s3-client/interface/index.ts @@ -24,6 +24,5 @@ export interface CopyFiles { export interface File { data: Readable; - name: string; mimeType: string; } diff --git a/config/default.schema.json b/config/default.schema.json index ef92d3f1db5..a34d8e899ad 100644 --- a/config/default.schema.json +++ b/config/default.schema.json @@ -1344,14 +1344,7 @@ } } }, - "required": [ - "TEACHER_VISIBILITY_FOR_EXTERNAL_TEAM_INVITATION", - "STUDENT_TEAM_CREATION", - "BLOCK_DISPOSABLE_EMAIL_DOMAINS", - "HOST", - "ACTIVATION_LINK_PERIOD_OF_VALIDITY_SECONDS", - "AES_KEY" - ], + "required": [], "allOf": [ { "$ref": "#/definitions/FEATURE_ES_MERLIN_ENABLED", diff --git a/nest-cli.json b/nest-cli.json index c92fcacf1cf..73dea03c093 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -63,6 +63,15 @@ "tsConfigPath": "apps/server/tsconfig.app.json" } }, + "preview-generator-amqp": { + "type": "application", + "root": "apps/server", + "entryFile": "apps/preview-generator-consumer.app", + "sourceRoot": "apps/server/src", + "compilerOptions": { + "tsConfigPath": "apps/server/tsconfig.app.json" + } + }, "fwu-learning-contents": { "type": "application", "root": "apps/server", diff --git a/package.json b/package.json index 3c5a73df7d1..a82df72904f 100644 --- a/package.json +++ b/package.json @@ -61,11 +61,12 @@ "nest:start:management:dev": "nest start management --watch", "nest:start:management:debug": "nest start management --debug --watch", "nest:start:management:prod": "node dist/apps/server/apps/management.app", - "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp", - "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch", - "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch", + "nest:start:files-storage": "nest start files-storage & nest start files-storage-amqp & nest start preview-generator-amqp", + "nest:start:files-storage:dev": "nest start files-storage --watch & nest start files-storage-amqp --watch & nest start preview-generator-amqp --watch", + "nest:start:files-storage:debug": "nest start files-storage --debug --watch & nest start files-storage-amqp --debug --watch & nest start preview-generator-amqp --debug --watch", "nest:start:files-storage:prod": "node dist/apps/server/apps/files-storage.app", "nest:start:files-storage-amqp:prod": "node dist/apps/server/apps/files-storage-consumer.app", + "nest:start:preview-generator-amqp:prod": "node dist/apps/server/apps/preview-generator-consumer.app", "nest:start:fwu-learning-contents": "nest start fwu-learning-contents", "nest:start:fwu-learning-contents:debug": "nest start fwu-learning-contents --debug --watch", "nest:start:fwu-learning-contents:prod": "node dist/apps/server/apps/fwu-learning-contents.app",