BC-7101 - collaboration service deployment (#4982)

Implement a deployment for the BoardCollaboration-WebSocket-Server. * initial commit * add configuration for board collaboration websocket uri --------- Co-authored-by: Thomas Feldtkeller <[email protected]>
hpi-schul-cloud · May 7, 2024 · 247ab0d · 247ab0d
1 parent ee13aaa
commit 247ab0d
Show file tree

Hide file tree

Showing 17 changed files with 266 additions and 42 deletions.
diff --git a/ansible/roles/schulcloud-server-core/tasks/main.yml b/ansible/roles/schulcloud-server-core/tasks/main.yml
@@ -204,7 +204,6 @@
      - KEDA_ENABLED is defined and KEDA_ENABLED|bool
      - SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool
 
-
   - name: admin api server deployment
     kubernetes.core.k8s:
       kubeconfig: ~/.kube/config
@@ -256,3 +255,32 @@
       namespace: "{{ NAMESPACE }}"
       template: tldraw-svc-monitor.yml.j2
     when: WITH_TLDRAW is defined and WITH_TLDRAW|bool
+
+  - name: BoardCollaboration configmap
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: board-collaboration-configmap.yml.j2
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
+
+  - name: BoardCollaboration deployment
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: board-collaboration-deployment.yml.j2
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
+
+  - name: BoardCollaboration service
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: board-collaboration-service.yml.j2
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
+
+  - name: BoardCollaboration ingress
+    kubernetes.core.k8s:
+      kubeconfig: ~/.kube/config
+      namespace: "{{ NAMESPACE }}"
+      template: board-collaboration-ingress.yml.j2
+      apply: yes
+      state: "{{ 'present' if WITH_BOARD_COLLABORATION else 'absent'}}"
diff --git a/ansible/roles/schulcloud-server-core/templates/board-collaboration-configmap.yml.j2 b/ansible/roles/schulcloud-server-core/templates/board-collaboration-configmap.yml.j2
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: board-collaboration-configmap
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: board-collaboration
+data:
+  NEST_LOG_LEVEL: "{{ NEST_LOG_LEVEL }}"
+  EXIT_ON_ERROR: "true"
diff --git a/ansible/roles/schulcloud-server-core/templates/board-collaboration-deployment.yml.j2 b/ansible/roles/schulcloud-server-core/templates/board-collaboration-deployment.yml.j2
@@ -0,0 +1,106 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: board-collaboration-deployment
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: board-collaboration
+    app.kubernetes.io/part-of: schulcloud-verbund
+    app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+    app.kubernetes.io/name: board-collaboration
+    app.kubernetes.io/component: server
+    app.kubernetes.io/managed-by: ansible
+    git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+    git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+spec:
+  replicas: {{ BOARD_COLLABORATION_SERVER_REPLICAS|default("1", true) }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      #maxUnavailable: 1
+  revisionHistoryLimit: 4
+  paused: false
+  selector:
+    matchLabels:
+      app: board-collaboration
+  template:
+    metadata:
+      labels:
+        app: board-collaboration
+        app.kubernetes.io/part-of: schulcloud-verbund
+        app.kubernetes.io/version: {{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        app.kubernetes.io/name: board-collaboration
+        app.kubernetes.io/component: board-collaboration
+        app.kubernetes.io/managed-by: ansible
+        git.branch: {{ SCHULCLOUD_SERVER_BRANCH_NAME }}
+        git.repo: {{ SCHULCLOUD_SERVER_REPO_NAME }}
+    spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        runAsNonRoot: true
+      containers:
+      - name: board-collaboration
+        image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 4450
+          name: websocket
+          protocol: TCP
+        # - containerPort: 9090
+        #  name: api-metrics
+        #  protocol: TCP
+        envFrom:
+        - configMapRef:
+            name: api-configmap
+        - configMapRef:
+            name: board-collaboration-configmap
+        - secretRef:
+            name: api-secret
+        command: ['npm', 'run', 'nest:start:board-collaboration:prod']
+        resources:
+          limits:
+            cpu: {{ BOARD_COLLABORATION_CPU_LIMITS|default("2000m", true) }}
+            memory: {{ BOARD_COLLABORATION_MEMORY_LIMITS|default("2Gi", true) }}
+          requests:
+            cpu: {{ BOARD_COLLABORATION_CPU_REQUESTS|default("100m", true) }}
+            memory: {{ BOARD_COLLABORATION_MEMORY_REQUESTS|default("150Mi", true) }}
+{% if AFFINITY_ENABLE is defined and AFFINITY_ENABLE|bool %}
+      affinity:
+        podAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 9
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/part-of
+                  operator: In
+                  values:
+                  - schulcloud-verbund
+              topologyKey: "kubernetes.io/hostname"
+              namespaceSelector: {}
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+{% if ANIT_AFFINITY_NODEPOOL_ENABLE is defined and ANIT_AFFINITY_NODEPOOL_ENABLE|bool %}
+          - weight: 10
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - board-collaboration
+              topologyKey: {{ ANIT_AFFINITY_NODEPOOL_TOPOLOGY_KEY }}
+{% endif %}
+          - weight: 20
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - board-collaboration
+              topologyKey: "topology.kubernetes.io/zone"
+{% endif %}
diff --git a/ansible/roles/schulcloud-server-core/templates/board-collaboration-ingress.yml.j2 b/ansible/roles/schulcloud-server-core/templates/board-collaboration-ingress.yml.j2
@@ -0,0 +1,43 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: board-collaboration-ingress
+  namespace: {{ NAMESPACE }}
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m"
+    nginx.org/client-max-body-size: "{{ INGRESS_MAX_BODY_SIZE|default("2560") }}m"
+    # The following properties added with BC-3606.
+    # The header size of the request is too big. For e.g. state and the permanent growing jwt.
+    # Nginx throws away the Location header, resulting in the 502 Bad Gateway.
+    nginx.ingress.kubernetes.io/client-header-buffer-size: 100k
+    nginx.ingress.kubernetes.io/http2-max-header-size: 96k
+    nginx.ingress.kubernetes.io/large-client-header-buffers: 4 100k
+    nginx.ingress.kubernetes.io/proxy-buffer-size: 96k
+    nginx.org/websocket-services: "board-collaboration-svc"
+{% if CLUSTER_ISSUER is defined %}
+    cert-manager.io/cluster-issuer: {{ CLUSTER_ISSUER }}
+{% endif %}
+
+spec:
+  ingressClassName: {{ INGRESS_CLASS }}
+{% if CLUSTER_ISSUER is defined or (TLS_ENABLED is defined and TLS_ENABLED|bool) %}
+  tls:
+  - hosts:
+      - {{ DOMAIN }}
+{% if CLUSTER_ISSUER is defined %}
+    secretName: {{ DOMAIN }}-tls
+{% endif %}
+{% endif %}
+  rules:
+  - host: {{ DOMAIN }}
+    http:
+      paths:
+      - path: /board-collaboration
+        backend:
+          service:
+            name: board-collaboration-svc
+            port:
+              number: 4450
+        pathType: Prefix
diff --git a/ansible/roles/schulcloud-server-core/templates/board-collaboration-service.yml.j2 b/ansible/roles/schulcloud-server-core/templates/board-collaboration-service.yml.j2
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: board-collaboration-svc
+  namespace: {{ NAMESPACE }}
+  labels:
+    app: board-collaboration
+spec:
+  type: ClusterIP
+  ports:
+    # port for WebSocket connection
+    - port: 4450
+      targetPort: 4450
+      protocol: TCP
+      name: websocket
+    #- port: {{ PORT_METRICS_SERVER }}
+    #  targetPort: 9090 # TODO
+    #  protocol: TCP
+    #  name: api-metrics
+  selector:
+    app: board-collaboration
diff --git a/apps/server/src/apps/board-collaboration.app.ts b/apps/server/src/apps/board-collaboration.app.ts
@@ -30,7 +30,7 @@ async function bootstrap() {
 	await nestApp.init();
 
 	const port = 4450;
-	const basePath = '/api/v3';
+	const basePath = '/board-collaboration';
 
 	nestApp.setGlobalPrefix(basePath);
 	await nestApp.listen(port);

diff --git a/apps/server/src/modules/board/board-ws-api.module.ts b/apps/server/src/modules/board/board-ws-api.module.ts
@@ -3,12 +3,12 @@ import { CourseRepo } from '@shared/repo';
 import { LoggerModule } from '@src/core/logger';
 import { AuthorizationModule } from '../authorization';
 import { BoardModule } from './board.module';
-import { SocketGateway } from './gateway/socket.gateway';
+import { BoardCollaborationGateway } from './gateway/board-collaboration.gateway';
 import { BoardUc, ColumnUc } from './uc';
 
 @Module({
 	imports: [BoardModule, forwardRef(() => AuthorizationModule), LoggerModule],
-	providers: [SocketGateway, ColumnUc, BoardUc, CourseRepo],
+	providers: [BoardCollaborationGateway, ColumnUc, BoardUc, CourseRepo],
 	exports: [],
 })
 export class BoardWsApiModule {}
diff --git a/...d/gateway/api-test/socket.gateway.spec.ts → ...-test/board-collaboration.gateway.spec.ts b/...d/gateway/api-test/socket.gateway.spec.ts → ...-test/board-collaboration.gateway.spec.ts
@@ -14,10 +14,10 @@ import {
 import { getSocketApiClient, waitForEvent } from '@shared/testing/test-socket-api-client';
 import { Socket } from 'socket.io-client';
 import { BoardCollaborationTestingModule } from '../../board-collaboration.testing.module';
-import { SocketGateway } from '../socket.gateway';
+import { BoardCollaborationGateway } from '../board-collaboration.gateway';
 
-describe('SocketGateway', () => {
-	let gateway: SocketGateway;
+describe(BoardCollaborationGateway.name, () => {
+	let ws: BoardCollaborationGateway;
 	let app: INestApplication;
 	let ioClient: Socket;
 	let em: EntityManager;
@@ -28,8 +28,8 @@ describe('SocketGateway', () => {
 		}).compile();
 		app = testingModule.createNestApplication();
 
-		em = app.get<EntityManager>(EntityManager);
-		gateway = app.get<SocketGateway>(SocketGateway);
+		em = app.get(EntityManager);
+		ws = app.get(BoardCollaborationGateway);
 
 		await app.listen(0);
 	});
@@ -65,7 +65,7 @@ describe('SocketGateway', () => {
 	};
 
 	it('should be defined', () => {
-		expect(gateway).toBeDefined();
+		expect(ws).toBeDefined();
 	});
 
 	describe('create card', () => {