Skip to content

Commit

Permalink
Merge branch 'main' into BC-4453-submission-item-file
Browse files Browse the repository at this point in the history
  • Loading branch information
virgilchiriac authored Oct 27, 2023
2 parents 34592eb + 84c44c1 commit df70336
Show file tree
Hide file tree
Showing 69 changed files with 1,654 additions and 759 deletions.
18 changes: 9 additions & 9 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,36 +68,36 @@ jobs:
tags: ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
labels: ${{ steps.docker_meta_img.outputs.labels }}

- name: Docker meta Service Name (file storage)
- name: Docker meta Service Name (file preview)
id: docker_meta_img_file_storage
uses: docker/metadata-action@v4
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=ref,event=branch,enable=false,priority=600,prefix=file-storage-
type=sha,enable=true,priority=600,prefix=file-storage-
type=ref,event=branch,enable=false,priority=600,prefix=file-preview-
type=sha,enable=true,priority=600,prefix=file-preview-
labels: |
org.opencontainers.image.title=schulcloud-file-storage
- name: test image exists (file storage)
- name: test image exists (file preview)
run: |
echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
echo "IMAGE_EXISTS=$(docker manifest inspect ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }} > /dev/null && echo 1 || echo 0)" >> $GITHUB_ENV
- name: Set up Docker Buildx (file storage)
- name: Set up Docker Buildx (file preview)
if: ${{ env.IMAGE_EXISTS == 0 }}
uses: docker/setup-buildx-action@v2

- name: Build and push ${{ github.repository }} (file storage)
- name: Build and push ${{ github.repository }} (file preview)
if: ${{ env.IMAGE_EXISTS == 0 }}
uses: docker/build-push-action@v4
with:
build-args: |
BASE_IMAGE=ghcr.io/${{ github.repository }}:${{ needs.branch_meta.outputs.sha }}
context: .
file: ./Dockerfile.filestorage
file: ./Dockerfile.filepreview
platforms: linux/amd64
push: true
tags: ghcr.io/${{ github.repository }}:file-storage-${{ needs.branch_meta.outputs.sha }}
tags: ghcr.io/${{ github.repository }}:file-preview-${{ needs.branch_meta.outputs.sha }}
labels: |
${{ steps.docker_meta_img_file_storage.outputs.labels }}
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,14 @@ jobs:
tags: ${{ steps.docker_meta_img_hub.outputs.tags }}
labels: ${{ steps.docker_meta_img_hub.outputs.labels }}

- name: Docker meta Service Name for docker hub (file storage)
- name: Docker meta Service Name for docker hub (file preview)
id: docker_meta_img_hub_file_storage
uses: docker/metadata-action@v4
with:
images: docker.io/schulcloud/schulcloud-server, quay.io/schulcloudverbund/schulcloud-server
tags: |
type=semver,pattern={{version}},prefix=file-storage-,onlatest=false
type=semver,pattern={{major}}.{{minor}},prefix=file-storage-,onlatest=false
type=semver,pattern={{version}},prefix=file-preview-,onlatest=false
type=semver,pattern={{major}}.{{minor}},prefix=file-preview-,onlatest=false
labels: |
org.opencontainers.image.title=schulcloud-file-storage
- name: Build and push ${{ github.repository }} (file-storage)
Expand All @@ -64,7 +64,7 @@ jobs:
build-args: |
BASE_IMAGE=quay.io/schulcloudverbund/schulcloud-server:${{ github.ref_name }}
context: .
file: ./Dockerfile.filestorage
file: ./Dockerfile.filepreview
platforms: linux/amd64
push: true
tags: ${{ steps.docker_meta_img_hub_file_storage.outputs.tags }}
Expand Down
File renamed without changes.
29 changes: 29 additions & 0 deletions ansible/roles/schulcloud-server-core/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,32 @@
kubeconfig: ~/.kube/config
namespace: "{{ NAMESPACE }}"
template: amqp-files-deployment.yml.j2

- name: Preview Generator Deployment
kubernetes.core.k8s:
kubeconfig: ~/.kube/config
namespace: "{{ NAMESPACE }}"
template: preview-generator-deployment.yml.j2

- name: preview generator configmap
kubernetes.core.k8s:
kubeconfig: ~/.kube/config
namespace: "{{ NAMESPACE }}"
template: preview-generator-configmap.yml.j2
apply: yes

- name: preview generator Secret by 1Password
kubernetes.core.k8s:
kubeconfig: ~/.kube/config
namespace: "{{ NAMESPACE }}"
template: preview-generator-onepassword.yml.j2
when: ONEPASSWORD_OPERATOR is defined and ONEPASSWORD_OPERATOR|bool

- name: preview generator scaled object
kubernetes.core.k8s:
kubeconfig: ~/.kube/config
namespace: "{{ NAMESPACE }}"
template: preview-generator-scaled-object.yml.j2
when:
- KEDA_ENABLED is defined and KEDA_ENABLED|bool
- SCALED_PREVIEW_GENERATOR_ENABLED is defined and SCALED_PREVIEW_GENERATOR_ENABLED|bool
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ spec:
runAsNonRoot: true
containers:
- name: api-files
image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-storage-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
image: {{ SCHULCLOUD_SERVER_IMAGE }}:{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
imagePullPolicy: IfNotPresent
ports:
- containerPort: 4444
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: preview-generator-configmap
namespace: {{ NAMESPACE }}
labels:
app: preview-generator
data:
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: preview-generator-deployment
namespace: {{ NAMESPACE }}
labels:
app: preview-generator
spec:
replicas: {{ AMQP_FILE_PREVIEW_REPLICAS|default("1", true) }}
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
#maxUnavailable: 1
revisionHistoryLimit: 4
paused: false
selector:
matchLabels:
app: preview-generator
template:
metadata:
labels:
app: preview-generator
spec:
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
runAsNonRoot: true
containers:
- name: preview-generator
image: {{ SCHULCLOUD_SERVER_IMAGE }}:file-preview-{{ SCHULCLOUD_SERVER_IMAGE_TAG }}
imagePullPolicy: IfNotPresent
envFrom:
- configMapRef:
name: preview-generator-configmap
- secretRef:
name: preview-generator-secret
command: ['npm', 'run', 'nest:start:preview-generator-amqp:prod']
resources:
limits:
cpu: {{ AMQP_FILE_PREVIEW_CPU_LIMITS|default("4000m", true) }}
memory: {{ AMQP_FILE_PREVIEW_MEMORY_LIMITS|default("4000Mi", true) }}
requests:
cpu: {{ AMQP_FILE_PREVIEW_CPU_REQUESTS|default("100m", true) }}
memory: {{ AMQP_FILE_PREVIEW_MEMORY_REQUESTS|default("250Mi", true) }}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
apiVersion: onepassword.com/v1
kind: OnePasswordItem
metadata:
name: preview-generator-secret
namespace: {{ NAMESPACE }}
labels:
app: preview-generator
spec:
itemPath: "vaults/{{ ONEPASSWORD_OPERATOR_VAULT }}/items/preview-generator"
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
name: preview-generator-rabbitmq-scaledobject
namespace: {{ NAMESPACE }}
labels:
app: preview-generator
spec:
scaleTargetRef:
name: preview-generator-deployment
idleReplicaCount: {{ AMQP_FILE_PREVIEW_IDLE_REPLICA_COUNT|default("1", true) }}
minReplicaCount: {{ AMQP_FILE_PREVIEW_MIN_REPLICA_COUNT|default("1", true) }}
maxReplicaCount: {{ AMQP_FILE_PREVIEW_MAX_REPLICA_COUNT|default("5", true) }}
triggers:
- type: rabbitmq
metadata:
protocol: amqp
queueName: generate-preview
mode: QueueLength
value: "1"
authenticationRef:
name: rabbitmq-trigger-auth
2 changes: 1 addition & 1 deletion apps/server/src/apps/files-storage-consumer.app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import { NestFactory } from '@nestjs/core';

// register source-map-support for debugging
import { FilesStorageAMQPModule } from '@modules/files-storage';
import { FilesStorageAMQPModule } from '@modules/files-storage/files-storage-amqp.module';
import { install as sourceMapInstall } from 'source-map-support';

async function bootstrap() {
Expand Down
3 changes: 2 additions & 1 deletion apps/server/src/apps/files-storage.app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ import express from 'express';
import { install as sourceMapInstall } from 'source-map-support';

// application imports
import { FilesStorageApiModule } from '@modules/files-storage/files-storage-api.module';
import { API_VERSION_PATH } from '@modules/files-storage/files-storage.const';
import { SwaggerDocumentOptions } from '@nestjs/swagger';
import { LegacyLogger } from '@src/core/logger';
import { API_VERSION_PATH, FilesStorageApiModule } from '@modules/files-storage';
import { enableOpenApiDocs } from '@src/shared/controller/swagger';

async function bootstrap() {
Expand Down
17 changes: 17 additions & 0 deletions apps/server/src/apps/preview-generator-consumer.app.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/* istanbul ignore file */
/* eslint-disable no-console */
import { PreviewGeneratorAMQPModule } from '@modules/files-storage/files-preview-amqp.module';
import { NestFactory } from '@nestjs/core';
import { install as sourceMapInstall } from 'source-map-support';

async function bootstrap() {
sourceMapInstall();

const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule);
await nestApp.init();

console.log('#############################################');
console.log(`### Start Preview Generator AMQP Consumer ###`);
console.log('#############################################');
}
void bootstrap();
3 changes: 1 addition & 2 deletions apps/server/src/modules/files-storage-client/mapper/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
export * from './error.mapper';
export * from './copy-files-of-parent-param.builder';
export * from './files-storage-client.mapper';
export * from './files-storage-param.builder';
export * from './copy-files-of-parent-param.builder';
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest';
import { ObjectId } from '@mikro-orm/mongodb';
import { ConfigService } from '@nestjs/config';
import { Test, TestingModule } from '@nestjs/testing';
import { FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq';
import { ErrorMapper, FileRecordParentType, FilesStorageEvents, FilesStorageExchange } from '@shared/infra/rabbitmq';
import { setupEntities } from '@shared/testing';
import { LegacyLogger } from '@src/core/logger';
import { ErrorMapper } from '../mapper';
import { FilesStorageProducer } from './files-storage.producer';

describe('FilesStorageProducer', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { AmqpConnection } from '@golevelup/nestjs-rabbitmq';
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { EntityId } from '@shared/domain';
import { RpcMessage } from '@shared/infra/rabbitmq/rpc-message';
import { LegacyLogger } from '@src/core/logger';
import {
FilesStorageEvents,
Expand All @@ -11,75 +10,45 @@ import {
ICopyFilesOfParentParams,
IFileDO,
IFileRecordParams,
RpcMessageProducer,
} from '@src/shared/infra/rabbitmq';
import { IFilesStorageClientConfig } from '../interfaces';
import { ErrorMapper } from '../mapper/error.mapper';

@Injectable()
export class FilesStorageProducer {
private readonly timeout = 0;

export class FilesStorageProducer extends RpcMessageProducer {
constructor(
protected readonly amqpConnection: AmqpConnection,
private readonly logger: LegacyLogger,
private readonly amqpConnection: AmqpConnection,
private readonly configService: ConfigService<IFilesStorageClientConfig, true>
protected readonly configService: ConfigService<IFilesStorageClientConfig, true>
) {
super(amqpConnection, FilesStorageExchange, configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API'));
this.logger.setContext(FilesStorageProducer.name);
this.timeout = this.configService.get('INCOMING_REQUEST_TIMEOUT_COPY_API');
}

async copyFilesOfParent(payload: ICopyFilesOfParentParams): Promise<ICopyFileDO[]> {
this.logger.debug({ action: 'copyFilesOfParent:started', payload });
const response = await this.amqpConnection.request<RpcMessage<ICopyFileDO[]>>(
this.createRequest(FilesStorageEvents.COPY_FILES_OF_PARENT, payload)
);
const response = await this.request<ICopyFileDO[]>(FilesStorageEvents.COPY_FILES_OF_PARENT, payload);

this.logger.debug({ action: 'copyFilesOfParent:finished', payload });

this.checkError(response);
return response.message || [];
return response;
}

async listFilesOfParent(payload: IFileRecordParams): Promise<IFileDO[]> {
this.logger.debug({ action: 'listFilesOfParent:started', payload });
const response = await this.amqpConnection.request<RpcMessage<IFileDO[]>>(
this.createRequest(FilesStorageEvents.LIST_FILES_OF_PARENT, payload)
);
const response = await this.request<IFileDO[]>(FilesStorageEvents.LIST_FILES_OF_PARENT, payload);

this.logger.debug({ action: 'listFilesOfParent:finished', payload });

this.checkError(response);
return response.message || [];
return response;
}

async deleteFilesOfParent(payload: EntityId): Promise<IFileDO[]> {
this.logger.debug({ action: 'deleteFilesOfParent:started', payload });
const response = await this.amqpConnection.request<RpcMessage<IFileDO[]>>(
this.createRequest(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload)
);
const response = await this.request<IFileDO[]>(FilesStorageEvents.DELETE_FILES_OF_PARENT, payload);

this.logger.debug({ action: 'deleteFilesOfParent:finished', payload });

this.checkError(response);
return response.message || [];
}

// need to be fixed with https://ticketsystem.dbildungscloud.de/browse/BC-2984
// mapRpcErrorResponseToDomainError should also removed with this ticket
private checkError(response: RpcMessage<unknown>) {
const { error } = response;
if (error) {
const domainError = ErrorMapper.mapRpcErrorResponseToDomainError(error);
throw domainError;
}
}

private createRequest(event: FilesStorageEvents, payload: IFileRecordParams | ICopyFilesOfParentParams | EntityId) {
return {
exchange: FilesStorageExchange,
routingKey: event,
payload,
timeout: this.timeout,
};
return response;
}
}
Loading

0 comments on commit df70336

Please sign in to comment.