Merge branch 'main' into BC-5521-implementation-KNL-Module

hpi-schul-cloud · Nov 1, 2023 · 72c84b4 · 72c84b4
2 parents a853ea7 + 0cb9d54
commit 72c84b4
Show file tree

Hide file tree

Showing 708 changed files with 10,730 additions and 10,363 deletions.
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -68,36 +68,36 @@ jobs:
           tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           labels: ${{ steps.docker_meta_img.outputs.labels }}
 
-      - name: Docker meta Service Name (file storage)
+      - name: Docker meta Service Name (file preview)
         id: docker_meta_img_file_storage
         uses: docker/metadata-action@v4
         with:
           images: ghcr.io/${{ github.repository }}
           tags: |
-            type=ref,event=branch,enable=false,priority=600,prefix=file-storage-
-            type=sha,enable=true,priority=600,prefix=file-storage-
+            type=ref,event=branch,enable=false,priority=600,prefix=file-preview-
+            type=sha,enable=true,priority=600,prefix=file-preview-
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
 
-      - name: test image exists (file storage)
+      - name: test image exists (file preview)
         run: |
-          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
+          echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
 
-      - name: Set up Docker Buildx (file storage)
+      - name: Set up Docker Buildx (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/setup-buildx-action@v2
 
-      - name: Build and push ${{ github.repository }} (file storage)
+      - name: Build and push ${{ github.repository }} (file preview)
         if: ${{ env.IMAGE_EXISTS == 0 }}
         uses: docker/build-push-action@v4
         with:
           build-args: |
             BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
-          tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }}
+          tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }}
           labels: |
             ${{ steps.docker_meta_img_file_storage.outputs.labels }}
 

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -48,14 +48,14 @@ jobs:
           tags: ${{ steps.docker_meta_img_hub.outputs.tags }}
           labels: ${{ steps.docker_meta_img_hub.outputs.labels }}
 
-      - name: Docker meta Service Name for docker hub (file storage)
+      - name: Docker meta Service Name for docker hub (file preview)
         id: docker_meta_img_hub_file_storage
         uses: docker/metadata-action@v4
         with:
           images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server
           tags: |
-            type=semver,pattern={{version}},prefix=file-storage-,onlatest=false
-            type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false
+            type=semver,pattern={{version}},prefix=file-preview-,onlatest=false
+            type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false
           labels: |
             org.opencontainers.image.title=schulcloud-file-storage
       - name: Build and push ${{ github.repository }} (file-storage)
@@ -64,7 +64,7 @@ jobs:
           build-args: |
             BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }}
           context: .
-          file: ./Dockerfile.filestorage
+          file: ./Dockerfile.filepreview
           platforms: linux/amd64
           push: true
           tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }}

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -83,14 +83,18 @@ jobs:
         run: |
           sudo apt-get install -y lcov
           find coverage -name *.info -exec echo -a {} \; | xargs lcov -o merged-lcov.info
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
       - name: SonarCloud upload coverage
-        uses: SonarSource/sonarcloud-github-action@v1.9
+        uses: SonarSource/sonarcloud-github-action@v2.0.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
   nest_lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    timeout-minutes: 6
     steps:
       - name: checkout
         uses: actions/checkout@v3

diff --git a/Dockerfile.filestorage → Dockerfile.filepreview b/Dockerfile.filestorage → Dockerfile.filepreview
diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml
@@ -83,3 +83,32 @@
       kubeconfig: ~/.kube/config
       namespace: "{{ NAMESPACE }}"
       template: amqp-files-deployment.yml.j2
+
+  - name: Preview Generator Deployment
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-deployment.yml.j2
+
+  - name: preview generator configmap
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-configmap.yml.j2
+      apply: yes
+
+  - name: preview generator Secret by 1Password
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-onepassword.yml.j2
+    when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool
+
+  - name: preview generator scaled object
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: preview-generator-scaled-object.yml.j2
+    when:
+     - KEDA_ENABLED is defined and KEDA_ENABLED|bool
+     - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool
diff --git a/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/api-files-deployment.yml.j2
@@ -29,7 +29,7 @@ spec:
         runAsNonRoot: true
       containers:
       - name: api-files
-        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 4444

diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: preview-generator-configmap
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+data:
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-deployment.yml.j2
@@ -0,0 +1,46 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: preview-generator-deployment
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      #maxUnavailable: 1
+  revisionHistoryLimit: 4
+  paused: false
+  selector:
+    matchLabels:
+      app: preview-generator
+  template:
+    metadata:
+      labels:
+        app: preview-generator
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        runAsNonRoot: true
+      containers:
+      - name: preview-generator
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        imagePullPolicy: IfNotPresent
+        envFrom:
+        - configMapRef:
+            name: preview-generator-configmap
+        - secretRef:
+            name: preview-generator-secret
+        command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod']
+        resources:
+          limits:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }}
+          requests:
+            cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }}
+            memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }}
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-onepassword.yml.j2
@@ -0,0 +1,9 @@
+apiVersion: onepassword.com/v1
+kind: OnePasswordItem
+metadata:
+  name: preview-generator-secret
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator"
diff --git a/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2 b/ansible/roles/schulcloud-server-core/templates/preview-generator-scaled-object.yml.j2
@@ -0,0 +1,23 @@
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: preview-generator-rabbitmq-scaledobject
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: preview-generator
+spec:
+  scaleTargetRef:
+    name: preview-generator-deployment
+  idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }}
+  minReplicaCount:  {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }}
+  maxReplicaCount:  {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }}
+  triggers:
+  - type: rabbitmq
+    metadata:
+      protocol: amqp
+      queueName: generate-preview
+      mode: QueueLength
+      value: "1"
+    authenticationRef:
+      name: rabbitmq-trigger-auth
diff --git a/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2 b/ansible/roles/schulcloud-server-init/templates/configmap_file_init.yml.j2
@@ -25,6 +25,9 @@ data:
     else
         echo "gg, hacky mongo replicaset"
     fi
+    {% if KEDA_NAMESPACE_ACTIVATOR_ENABLED is defined %}
+    curl -XPUT -H 'Content-Type: application/json'  -Lv 'https://activate.cd.dbildungscloud.dev/namespace' -d '{"name" : "{{ NAMESPACE }}"}'
+    {% endif %}
     curl --retry 360 --retry-connrefused --retry-delay 10 -X POST 'http://mgmt-svc:3333/api/management/database/seed?with-indexes=true'
 
     # Below is a series of a MongoDB-data initializations, meant for the development and testing

diff --git a/apps/server/src/apps/files-storage-consumer.app.ts b/apps/server/src/apps/files-storage-consumer.app.ts
@@ -3,7 +3,7 @@
 import { NestFactory } from '@nestjs/core';
 
 // register source-map-support for debugging
-import { FilesStorageAMQPModule } from '@src/modules/files-storage';
+import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module';
 import { install as sourceMapInstall } from 'source-map-support';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/files-storage.app.ts b/apps/server/src/apps/files-storage.app.ts
@@ -8,9 +8,10 @@ import express from 'express';
 import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
+import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module';
+import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const';
 import { SwaggerDocumentOptions } from '@nestjs/swagger';
 import { LegacyLogger } from '@src/core/logger';
-import { API_VERSION_PATH, FilesStorageApiModule } from '@src/modules/files-storage';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/fwu-learning-contents.app.ts b/apps/server/src/apps/fwu-learning-contents.app.ts
@@ -9,7 +9,7 @@ import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
 import { LegacyLogger } from '@src/core/logger';
-import { FwuLearningContentsModule } from '@src/modules/fwu-learning-contents';
+import { FwuLearningContentsModule } from '@modules/fwu-learning-contents';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/h5p-editor.app.ts b/apps/server/src/apps/h5p-editor.app.ts
@@ -9,7 +9,7 @@ import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
 import { LegacyLogger } from '@src/core/logger';
-import { H5PEditorModule } from '@src/modules/h5p-editor';
+import { H5PEditorModule } from '@modules/h5p-editor';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/management.app.ts b/apps/server/src/apps/management.app.ts
@@ -9,7 +9,7 @@ import { install as sourceMapInstall } from 'source-map-support';
 
 // application imports
 import { LegacyLogger } from '@src/core/logger';
-import { ManagementServerModule } from '@src/modules/management';
+import { ManagementServerModule } from '@modules/management';
 import { enableOpenApiDocs } from '@src/shared/controller/swagger';
 
 async function bootstrap() {

diff --git a/apps/server/src/apps/preview-generator-consumer.app.ts b/apps/server/src/apps/preview-generator-consumer.app.ts
@@ -0,0 +1,17 @@
+/* istanbul ignore file */
+/* eslint-disable no-console */
+import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module';
+import { NestFactory } from '@nestjs/core';
+import { install as sourceMapInstall } from 'source-map-support';
+
+async function bootstrap() {
+	sourceMapInstall();
+
+	const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule);
+	await nestApp.init();
+
+	console.log('#############################################');
+	console.log(`### Start Preview Generator AMQP Consumer ###`);
+	console.log('#############################################');
+}
+void bootstrap();
diff --git a/apps/server/src/apps/server.app.ts b/apps/server/src/apps/server.app.ts
@@ -7,19 +7,20 @@ import { ExpressAdapter } from '@nestjs/platform-express';
 import { enableOpenApiDocs } from '@shared/controller/swagger';
 import { Mail, MailService } from '@shared/infra/mail';
 import { LegacyLogger, Logger } from '@src/core/logger';
-import { AccountService } from '@src/modules/account/services/account.service';
-import { TeamService } from '@src/modules/teams/service/team.service';
-import { AccountValidationService } from '@src/modules/account/services/account.validation.service';
-import { AccountUc } from '@src/modules/account/uc/account.uc';
-import { CollaborativeStorageUc } from '@src/modules/collaborative-storage/uc/collaborative-storage.uc';
-import { RocketChatService } from '@src/modules/rocketchat';
-import { ServerModule } from '@src/modules/server';
+import { AccountService } from '@modules/account/services/account.service';
+import { TeamService } from '@modules/teams/service/team.service';
+import { AccountValidationService } from '@modules/account/services/account.validation.service';
+import { AccountUc } from '@modules/account/uc/account.uc';
+import { CollaborativeStorageUc } from '@modules/collaborative-storage/uc/collaborative-storage.uc';
+import { GroupService } from '@modules/group';
+import { RocketChatService } from '@modules/rocketchat';
+import { ServerModule } from '@modules/server';
 import express from 'express';
 import { join } from 'path';
 
 // register source-map-support for debugging
 import { install as sourceMapInstall } from 'source-map-support';
-import { FeathersRosterService } from '@src/modules/pseudonym';
+import { FeathersRosterService } from '@modules/pseudonym';
 import legacyAppPromise = require('../../../../src/app');
 
 import { AppStartLoggable } from './helpers/app-start-loggable';
@@ -82,6 +83,8 @@ async function bootstrap() {
 	feathersExpress.services['nest-team-service'] = nestApp.get(TeamService);
 	// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment
 	feathersExpress.services['nest-feathers-roster-service'] = nestApp.get(FeathersRosterService);
+	// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment,@typescript-eslint/no-unsafe-member-access
+	feathersExpress.services['nest-group-service'] = nestApp.get(GroupService);
 	// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access,@typescript-eslint/no-unsafe-assignment
 	feathersExpress.services['nest-orm'] = orm;
 

diff --git a/apps/server/src/console/api-test/test-bootstrap.console.ts b/apps/server/src/console/api-test/test-bootstrap.console.ts
@@ -1,7 +1,7 @@
 import { createMock } from '@golevelup/ts-jest';
 import { Test, TestingModule } from '@nestjs/testing';
 import { ConsoleWriterService } from '@shared/infra/console';
-import { DatabaseManagementUc } from '@src/modules/management/uc/database-management.uc';
+import { DatabaseManagementUc } from '@modules/management/uc/database-management.uc';
 import { AbstractBootstrapConsole, BootstrapConsole } from 'nestjs-console';
 
 export class TestBootstrapConsole extends AbstractBootstrapConsole<TestingModule> {

diff --git a/apps/server/src/console/console.module.ts b/apps/server/src/console/console.module.ts
@@ -7,11 +7,11 @@ import { ALL_ENTITIES } from '@shared/domain';
 import { ConsoleWriterModule } from '@shared/infra/console/console-writer/console-writer.module';
 import { KeycloakModule } from '@shared/infra/identity-management/keycloak/keycloak.module';
 import { DB_PASSWORD, DB_URL, DB_USERNAME, createConfigModuleOptions } from '@src/config';
-import { FilesModule } from '@src/modules/files';
-import { FileEntity } from '@src/modules/files/entity';
-import { FileRecord } from '@src/modules/files-storage/entity';
-import { ManagementModule } from '@src/modules/management/management.module';
-import { serverConfig } from '@src/modules/server';
+import { FilesModule } from '@modules/files';
+import { FileEntity } from '@modules/files/entity';
+import { FileRecord } from '@modules/files-storage/entity';
+import { ManagementModule } from '@modules/management/management.module';
+import { serverConfig } from '@modules/server';
 import { ConsoleModule } from 'nestjs-console';
 import { ServerConsole } from './server.console';