Bc 5170 no preview for larger files (#4457)

hpi-schul-cloud · Oct 27, 2023 · 84c44c1 · 84c44c1
1 parent 3e112cc
commit 84c44c1
Show file tree

Hide file tree

Showing 69 changed files with 1,654 additions and 759 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -68,36 +68,36 @@ jobs:
           tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           labels: ${{ steps.docker_meta_img.outputs.labels }}
 
-      - name: Docker meta Service Name (file storage)
+      - name: Docker meta Service Name (file preview)
         id: docker_meta_img_file_storage
         uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
           tags: |
-            type=ref,event=branch,enable=false,priority=600,prefix=file-storage-
-            type=sha,enable=true,priority=600,prefix=file-storage-
+            type=ref,event=branch,enable=false,priority=600,prefix=file-preview-
+            type=sha,enable=true,priority=600,prefix=file-preview-
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
 
-      - name: test image exists (file storage)
+      - name: test image exists (file preview)
         run: |
-          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
+          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
 
-      - name: Set up Docker Buildx (file storage)
+      - name: Set up Docker Buildx (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/setup-buildx-action@v2
 
-      - name: Build and push ${{ github.repository }} (file storage)
+      - name: Build and push ${{ github.repository }} (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/build-push-action@v4
         with:
           build-args: |
             BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
-          tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }}
+          tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }}
           labels: |
             ${{ steps.docker_meta_img_file_storage.outputs.labels }}
 

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -48,14 +48,14 @@ jobs:
           tags: ${{ steps.docker_meta_img_hub.outputs.tags }}
           labels: ${{ steps.docker_meta_img_hub.outputs.labels }}
 
-      - name: Docker meta Service Name for docker hub (file storage)
+      - name: Docker meta Service Name for docker hub (file preview)
         id: docker_meta_img_hub_file_storage
         uses: docker/metadata-action@v4
         with:
           images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server
           tags: |
-            type=semver,pattern={{version}},prefix=file-storage-,onlatest=false
-            type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false
+            type=semver,pattern={{version}},prefix=file-preview-,onlatest=false
+            type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
       - name: Build and push ${{ github.repository }} (file-storage)
@@ -64,7 +64,7 @@ jobs:
           build-args: |
             BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
           tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }}

diff --git a/Dockerfile.filestorage → Dockerfile.filepreview b/Dockerfile.filestorage → Dockerfile.filepreview
diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml
@@ -83,3 +83,32 @@
       kubeconfig: ~/.kube/config
       namespace: "{{ NAMESPACE }}"
       template: amqp-files-deployment.yml.j2
+
+  - name: Preview Generator Deployment
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-deployment.yml.j2
+
+  - name: preview generator configmap
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-configmap.yml.j2
+      apply: yes
+
+  - name: preview generator Secret by 1Password
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-onepassword.yml.j2
+    when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool
+
+  - name: preview generator scaled object
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-scaled-object.yml.j2
+    when:
+     - KEDA_ENABLED is defined and KEDA_ENABLED|bool
+     - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool
diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2
@@ -29,7 +29,7 @@ spec:
         runAsNonRoot: true
       containers:
       - name: api-files
-        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 4444

diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: preview-generator-configmap
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+data:
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2
@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: preview-generator-deployment
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      #maxUnavailable: 1
+  revisionHistoryLimit: 4
+  paused: false
+  selector:
+    matchLabels:
+      app: preview-generator
+  template:
+    metadata:
+      labels:
+        app: preview-generator
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        runAsNonRoot: true
+      containers:
+      - name: preview-generator
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        imagePullPolicy: IfNotPresent
+        envFrom:
+        - configMapRef:
+            name: preview-generator-configmap
+        - secretRef:
+            name: preview-generator-secret
+        command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod']
+        resources:
+          limits:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }}
+          requests:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }}
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2
@@ -0,0 +1,9 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: preview-generator-secret
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator"
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2
@@ -0,0 +1,23 @@
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: preview-generator-rabbitmq-scaledobject
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  scaleTargetRef:
+    name: preview-generator-deployment
+  idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }}
+  minReplicaCount:  {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }}
+  maxReplicaCount:  {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }}
+  triggers:
+  - type: rabbitmq
+    metadata:
+      protocol: amqp
+      queueName: generate-preview
+      mode: QueueLength
+      value: "1"
+    authenticationRef:
+      name: rabbitmq-trigger-auth
diff --git a/apps/server/src/apps/files-storage-consumer.app.ts b/apps/server/src/apps/files-storage-consumer.app.ts
@@ -3,7 +3,7 @@
 import { NestFactory } from '@nestjs/core';
 
 // register source-map-support for debugging
-import { FilesStorageAMQPModule } from '@modules/files-storage';
+import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module';
 import { install as sourceMapInstall } from 'source-map-support';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/files-storage.app.ts b/apps/server/src/apps/files-storage.app.ts
@@ -8,9 +8,10 @@ import express from 'express';
 import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
+import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module';
+import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const';
 import { SwaggerDocumentOptions } from '@nestjs/swagger';
 import { LegacyLogger } from '@src/core/logger';
-import { API_VERSION_PATH, FilesStorageApiModule } from '@modules/files-storage';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/preview-generator-consumer.app.ts b/apps/server/src/apps/preview-generator-consumer.app.ts
@@ -0,0 +1,17 @@
+/* istanbul ignore file */
+/* eslint-disable no-console */
+import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module';
+import { NestFactory } from '@nestjs/core';
+import { install as sourceMapInstall } from 'source-map-support';
+
+async function bootstrap() {
+	sourceMapInstall();
+
+	const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule);
+	await nestApp.init();
+
+	console.log('#############################################');
+	console.log(`### Start Preview Generator AMQP Consumer ###`);
+	console.log('#############################################');
+}
+void bootstrap();
diff --git a/apps/server/src/modules/files-storage-client/mapper/index.ts b/apps/server/src/modules/files-storage-client/mapper/index.ts
@@ -1,4 +1,3 @@
-export * from './error.mapper';
+export * from './copy-files-of-parent-param.builder';
 export * from './files-storage-client.mapper';
 export * from './files-storage-param.builder';
-export * from './copy-files-of-parent-param.builder';
diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.spec.ts
@@ -3,10 +3,9 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
 import { ObjectId } from '@mikro-orm/mongodb';
 import { ConfigService } from '@nestjs/config';
 import { Test, TestingModule } from '@nestjs/testing';
-import { FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq';
+import { ErrorMapper, FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq';
 import { setupEntities } from '@shared/testing';
 import { LegacyLogger } from '@src/core/logger';
-import { ErrorMapper } from '../mapper';
 import { FilesStorageProducer } from './files-storage.producer';
 
 describe('FilesStorageProducer', () => {

diff --git a/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts b/apps/server/src/modules/files-storage-client/service/files-storage.producer.ts
@@ -2,7 +2,6 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq';
 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { EntityId } from '@shared/domain';
-import { RpcMessage } from '@shared/infra/rabbitmq/rpc-message';
 import { LegacyLogger } from '@src/core/logger';
 import {
 	FilesStorageEvents,
@@ -11,75 +10,45 @@ import {
 	ICopyFilesOfParentParams,
 	IFileDO,
 	IFileRecordParams,
+	RpcMessageProducer,
 } from '@src/shared/infra/rabbitmq';
 import { IFilesStorageClientConfig } from '../interfaces';
-import { ErrorMapper } from '../mapper/error.mapper';
 
 @Injectable()
-export class FilesStorageProducer {
-	private readonly timeout = 0;
-
+export class FilesStorageProducer extends RpcMessageProducer {
 	constructor(
+		protected readonly amqpConnection: AmqpConnection,
 		private readonly logger: LegacyLogger,
-		private readonly amqpConnection: AmqpConnection,
-		private readonly configService: ConfigService<IFilesStorageClientConfig, true>
+		protected readonly configService: ConfigService<IFilesStorageClientConfig, true>
 	) {
+		super(amqpConnection, FilesStorageExchange, configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API'));
 		this.logger.setContext(FilesStorageProducer.name);
-		this.timeout = this.configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API');
 	}
 
 	async copyFilesOfParent(payload: ICopyFilesOfParentParams): Promise<ICopyFileDO[]> {
 		this.logger.debug({ action: 'copyFilesOfParent:started', payload });
-		const response = await this.amqpConnection.request<RpcMessage<ICopyFileDO[]>>(
-			this.createRequest(FilesStorageEvents.COPY_FILES_OF_PARENT, payload)
-		);
+		const response = await this.request<ICopyFileDO[]>(FilesStorageEvents.COPY_FILES_OF_PARENT, payload);
 
 		this.logger.debug({ action: 'copyFilesOfParent:finished', payload });
 
-		this.checkError(response);
-		return response.message || [];
+		return response;
 	}
 
 	async listFilesOfParent(payload: IFileRecordParams): Promise<IFileDO[]> {
 		this.logger.debug({ action: 'listFilesOfParent:started', payload });
-		const response = await this.amqpConnection.request<RpcMessage<IFileDO[]>>(
-			this.createRequest(FilesStorageEvents.LIST_FILES_OF_PARENT, payload)
-		);
+		const response = await this.request<IFileDO[]>(FilesStorageEvents.LIST_FILES_OF_PARENT, payload);
 
 		this.logger.debug({ action: 'listFilesOfParent:finished', payload });
 
-		this.checkError(response);
-		return response.message || [];
+		return response;
 	}
 
 	async deleteFilesOfParent(payload: EntityId): Promise<IFileDO[]> {
 		this.logger.debug({ action: 'deleteFilesOfParent:started', payload });
-		const response = await this.amqpConnection.request<RpcMessage<IFileDO[]>>(
-			this.createRequest(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload)
-		);
+		const response = await this.request<IFileDO[]>(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload);
 
 		this.logger.debug({ action: 'deleteFilesOfParent:finished', payload });
 
-		this.checkError(response);
-		return response.message || [];
-	}
-
-	// need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984
-	// mapRpcErrorResponseToDomainError should also removed with this ticket
-	private checkError(response: RpcMessage<unknown>) {
-		const { error } = response;
-		if (error) {
-			const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error);
-			throw domainError;
-		}
-	}
-
-	private createRequest(event: FilesStorageEvents, payload: IFileRecordParams | ICopyFilesOfParentParams | EntityId) {
-		return {
-			exchange: FilesStorageExchange,
-			routingKey: event,
-			payload,
-			timeout: this.timeout,
-		};
+		return response;
 	}
 }