Merge branch 'main' of github.com:hpi-schul-cloud/schulcloud-server i…

…nto BC-5434-meta-data-endpoint
hpi-schul-cloud · Nov 1, 2023 · 389e127 · 389e127
2 parents 2590b94 + 0cb9d54
commit 389e127
Show file tree

Hide file tree

Showing 84 changed files with 1,855 additions and 819 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -68,36 +68,36 @@ jobs:
           tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           labels: ${{ steps.docker_meta_img.outputs.labels }}
 
-      - name: Docker meta Service Name (file storage)
+      - name: Docker meta Service Name (file preview)
         id: docker_meta_img_file_storage
         uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
           tags: |
-            type=ref,event=branch,enable=false,priority=600,prefix=file-storage-
-            type=sha,enable=true,priority=600,prefix=file-storage-
+            type=ref,event=branch,enable=false,priority=600,prefix=file-preview-
+            type=sha,enable=true,priority=600,prefix=file-preview-
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
 
-      - name: test image exists (file storage)
+      - name: test image exists (file preview)
         run: |
-          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
+          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
 
-      - name: Set up Docker Buildx (file storage)
+      - name: Set up Docker Buildx (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/setup-buildx-action@v2
 
-      - name: Build and push ${{ github.repository }} (file storage)
+      - name: Build and push ${{ github.repository }} (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/build-push-action@v4
         with:
           build-args: |
             BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
-          tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }}
+          tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }}
           labels: |
             ${{ steps.docker_meta_img_file_storage.outputs.labels }}
 

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -48,14 +48,14 @@ jobs:
           tags: ${{ steps.docker_meta_img_hub.outputs.tags }}
           labels: ${{ steps.docker_meta_img_hub.outputs.labels }}
 
-      - name: Docker meta Service Name for docker hub (file storage)
+      - name: Docker meta Service Name for docker hub (file preview)
         id: docker_meta_img_hub_file_storage
         uses: docker/metadata-action@v4
         with:
           images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server
           tags: |
-            type=semver,pattern={{version}},prefix=file-storage-,onlatest=false
-            type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false
+            type=semver,pattern={{version}},prefix=file-preview-,onlatest=false
+            type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
       - name: Build and push ${{ github.repository }} (file-storage)
@@ -64,7 +64,7 @@ jobs:
           build-args: |
             BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
           tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }}

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -94,7 +94,7 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
   nest_lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 6
     steps:
       - name: checkout
         uses: actions/checkout@v3

diff --git a/Dockerfile.filestorage → Dockerfile.filepreview b/Dockerfile.filestorage → Dockerfile.filepreview
diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml
@@ -83,3 +83,32 @@
       kubeconfig: ~/.kube/config
       namespace: "{{ NAMESPACE }}"
       template: amqp-files-deployment.yml.j2
+
+  - name: Preview Generator Deployment
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-deployment.yml.j2
+
+  - name: preview generator configmap
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-configmap.yml.j2
+      apply: yes
+
+  - name: preview generator Secret by 1Password
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-onepassword.yml.j2
+    when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool
+
+  - name: preview generator scaled object
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-scaled-object.yml.j2
+    when:
+     - KEDA_ENABLED is defined and KEDA_ENABLED|bool
+     - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool
diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2
@@ -29,7 +29,7 @@ spec:
         runAsNonRoot: true
       containers:
       - name: api-files
-        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 4444

diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: preview-generator-configmap
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+data:
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2
@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: preview-generator-deployment
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      #maxUnavailable: 1
+  revisionHistoryLimit: 4
+  paused: false
+  selector:
+    matchLabels:
+      app: preview-generator
+  template:
+    metadata:
+      labels:
+        app: preview-generator
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        runAsNonRoot: true
+      containers:
+      - name: preview-generator
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        imagePullPolicy: IfNotPresent
+        envFrom:
+        - configMapRef:
+            name: preview-generator-configmap
+        - secretRef:
+            name: preview-generator-secret
+        command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod']
+        resources:
+          limits:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }}
+          requests:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }}
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2
@@ -0,0 +1,9 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: preview-generator-secret
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator"
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2
@@ -0,0 +1,23 @@
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: preview-generator-rabbitmq-scaledobject
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  scaleTargetRef:
+    name: preview-generator-deployment
+  idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }}
+  minReplicaCount:  {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }}
+  maxReplicaCount:  {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }}
+  triggers:
+  - type: rabbitmq
+    metadata:
+      protocol: amqp
+      queueName: generate-preview
+      mode: QueueLength
+      value: "1"
+    authenticationRef:
+      name: rabbitmq-trigger-auth
diff --git a/apps/server/src/apps/files-storage-consumer.app.ts b/apps/server/src/apps/files-storage-consumer.app.ts
@@ -3,7 +3,7 @@
 import { NestFactory } from '@nestjs/core';
 
 // register source-map-support for debugging
-import { FilesStorageAMQPModule } from '@modules/files-storage';
+import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module';
 import { install as sourceMapInstall } from 'source-map-support';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/files-storage.app.ts b/apps/server/src/apps/files-storage.app.ts
@@ -8,9 +8,10 @@ import express from 'express';
 import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
+import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module';
+import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const';
 import { SwaggerDocumentOptions } from '@nestjs/swagger';
 import { LegacyLogger } from '@src/core/logger';
-import { API_VERSION_PATH, FilesStorageApiModule } from '@modules/files-storage';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/preview-generator-consumer.app.ts b/apps/server/src/apps/preview-generator-consumer.app.ts
@@ -0,0 +1,17 @@
+/* istanbul ignore file */
+/* eslint-disable no-console */
+import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module';
+import { NestFactory } from '@nestjs/core';
+import { install as sourceMapInstall } from 'source-map-support';
+
+async function bootstrap() {
+	sourceMapInstall();
+
+	const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule);
+	await nestApp.init();
+
+	console.log('#############################################');
+	console.log(`### Start Preview Generator AMQP Consumer ###`);
+	console.log('#############################################');
+}
+void bootstrap();
diff --git a/apps/server/src/core/error/filter/global-error.filter.spec.ts b/apps/server/src/core/error/filter/global-error.filter.spec.ts
@@ -1,14 +1,15 @@
 /* eslint-disable promise/valid-params */
 import { NotFound } from '@feathersjs/errors';
 import { createMock, DeepMocked } from '@golevelup/ts-jest';
-import { ArgumentsHost, BadRequestException, HttpStatus } from '@nestjs/common';
+import { ArgumentsHost, BadRequestException, HttpStatus, InternalServerErrorException } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { BusinessError } from '@shared/common';
 import { ErrorLogger, ErrorLogMessage, Loggable, LogMessage, ValidationErrorLogMessage } from '@src/core/logger';
 import { Response } from 'express';
 import util from 'util';
 import { ErrorResponse } from '../dto';
 import { ErrorLoggable } from '../loggable/error.loggable';
+import { ErrorUtils } from '../utils';
 import { GlobalErrorFilter } from './global-error.filter';
 
 class SampleBusinessError extends BusinessError {
@@ -42,6 +43,24 @@ class SampleLoggableException extends BadRequestException implements Loggable {
 	}
 }
 
+class SampleLoggableExceptionWithCause extends InternalServerErrorException implements Loggable {
+	constructor(private readonly testValue: string, error?: unknown) {
+		super(ErrorUtils.createHttpExceptionOptions(error));
+	}
+
+	getLogMessage(): ErrorLogMessage {
+		const message: ErrorLogMessage = {
+			type: 'WITH_CAUSE',
+			stack: this.stack,
+			data: {
+				testValue: this.testValue,
+			},
+		};
+
+		return message;
+	}
+}
+
 describe('GlobalErrorFilter', () => {
 	let module: TestingModule;
 	let service: GlobalErrorFilter<any>;
@@ -304,24 +323,101 @@ describe('GlobalErrorFilter', () => {
 					).toBeCalledWith(expectedResponse);
 				});
 			});
+
+			describe('when error has a cause error', () => {
+				const setup = () => {
+					const causeError = new Error('Cause error');
+					const error = new SampleLoggableExceptionWithCause('test', causeError);
+					const expectedResponse = new ErrorResponse(
+						'SAMPLE_WITH_CAUSE',
+						'Sample With Cause',
+						'Sample Loggable Exception With Cause',
+						HttpStatus.INTERNAL_SERVER_ERROR
+					);
+
+					const argumentsHost = setupHttpArgumentsHost();
+
+					return { error, argumentsHost, expectedResponse };
+				};
+
+				it('should set response status appropriately', () => {
+					const { error, argumentsHost } = setup();
+
+					service.catch(error, argumentsHost);
+
+					expect(argumentsHost.switchToHttp().getResponse<Response>().status).toBeCalledWith(
+						HttpStatus.INTERNAL_SERVER_ERROR
+					);
+				});
+
+				it('should send appropriate error response', () => {
+					const { error, argumentsHost, expectedResponse } = setup();
+
+					service.catch(error, argumentsHost);
+
+					expect(
+						argumentsHost.switchToHttp().getResponse<Response>().status(HttpStatus.INTERNAL_SERVER_ERROR).json
+					).toBeCalledWith(expectedResponse);
+				});
+			});
 		});
 
 		describe('when context is rmq', () => {
+			describe('when error is unknown error', () => {
+				const setup = () => {
+					const argumentsHost = createMock<ArgumentsHost>();
+					argumentsHost.getType.mockReturnValueOnce('rmq');
+
+					const error = new Error();
+
+					return { error, argumentsHost };
+				};
+
+				it('should return an RpcMessage with the error', () => {
+					const { error, argumentsHost } = setup();
+
+					const result = service.catch(error, argumentsHost);
+
+					expect(result).toEqual({ message: undefined, error });
+				});
+			});
+
+			describe('when error is a LoggableError', () => {
+				const setup = () => {
+					const causeError = new Error('Cause error');
+					const error = new SampleLoggableExceptionWithCause('test', causeError);
+					const argumentsHost = createMock<ArgumentsHost>();
+					argumentsHost.getType.mockReturnValueOnce('rmq');
+
+					return { error, argumentsHost };
+				};
+
+				it('should return appropriate error', () => {
+					const { error, argumentsHost } = setup();
+
+					const result = service.catch(error, argumentsHost);
+
+					expect(result).toEqual({ message: undefined, error });
+				});
+			});
+		});
+
+		describe('when context is other than rmq and http', () => {
 			const setup = () => {
 				const argumentsHost = createMock<ArgumentsHost>();
-				argumentsHost.getType.mockReturnValueOnce('rmq');
+				argumentsHost.getType.mockReturnValueOnce('other');
 
 				const error = new Error();
 
 				return { error, argumentsHost };
 			};
 
-			it('should return an RpcMessage with the error', () => {
+			it('should return undefined', () => {
 				const { error, argumentsHost } = setup();
 
 				const result = service.catch(error, argumentsHost);
 
-				expect(result).toEqual({ message: undefined, error });
+				expect(result).toBeUndefined();
 			});
 		});
 	});