BC-6338 - When rabbitmq restarts the preview service gets stuck

ansible/roles/schulcloud-server-core/templates/preview-generator-configmap.yml.j2

@@ -7,3 +7,4 @@ metadata:
     app: preview-generator
 data:
   NEST_LOG_LEVEL: "info"
+  EXIT_ON_ERROR: "true"

apps/server/src/apps/files-storage-consumer.app.ts

@@ -9,7 +9,7 @@ import { install as sourceMapInstall } from 'source-map-support';
 async function bootstrap() {
 	sourceMapInstall();
 
-	const nestApp = await NestFactory.createMicroservice(FilesStorageAMQPModule);
+	const nestApp = await NestFactory.create(FilesStorageAMQPModule);
 	await nestApp.init();
 
 	console.log('#########################################');

apps/server/src/apps/preview-generator-consumer.app.ts

@@ -7,7 +7,7 @@ import { install as sourceMapInstall } from 'source-map-support';
 async function bootstrap() {
 	sourceMapInstall();
 
-	const nestApp = await NestFactory.createMicroservice(PreviewGeneratorAMQPModule);
+	const nestApp = await NestFactory.create(PreviewGeneratorAMQPModule);
 	await nestApp.init();
 
 	console.log('#############################################');

apps/server/src/core/logger/interfaces/logger-config.ts

@@ -1,3 +1,4 @@
 export interface LoggerConfig {
 	NEST_LOG_LEVEL: string;
+	EXIT_ON_ERROR?: boolean;
 }

apps/server/src/core/logger/logger.module.ts

@@ -14,7 +14,7 @@ import { Logger } from './logger';
 				return {
 					levels: winston.config.syslog.levels,
 					level: configService.get<string>('NEST_LOG_LEVEL'),
-					exitOnError: false,
+					exitOnError: configService.get<boolean>('EXIT_ON_ERROR') ?? false,
 					transports: [
 						new winston.transports.Console({
 							handleExceptions: true,

apps/server/src/infra/healthcheck/healthcheck.module.ts

@@ -0,0 +1,16 @@
+import { DynamicModule, Module, Type } from '@nestjs/common';
+import { TerminusModule } from '@nestjs/terminus';
+import { AmqpConsumerHealthIndicator } from './indicators/amqp-consumer.indictor';
+
+@Module({})
+export class HealthCheckModule {
+	static register(controllers?: Type<any>[]): DynamicModule {
+		return {
+			module: HealthCheckModule,
+			imports: [TerminusModule],
+			providers: [AmqpConsumerHealthIndicator],
+			exports: [AmqpConsumerHealthIndicator],
+			controllers,
+		};
+	}
+}

apps/server/src/infra/healthcheck/indicators/amqp-consumer.indictor.ts

@@ -0,0 +1,26 @@
+import { AmqpConnection } from '@golevelup/nestjs-rabbitmq';
+import { HealthCheckError, HealthIndicator } from '@nestjs/terminus';
+
+export class AmqpConsumerHealthIndicator extends HealthIndicator {
+	private amqpConnection!: AmqpConnection;
+
+	setAmqpConnection(connection: AmqpConnection) {
+		this.amqpConnection = connection;
+	}
+
+	public isHealthy(key: string) {
+		const { consumerTags, channel } = this.amqpConnection;
+
+		console.log('CHANNEL', channel);
+		console.log('CONSUMER_TAGS', consumerTags);
+
+		const isHealthy = consumerTags.length > 0;
+
+		const result = this.getStatus(key, isHealthy);
+
+		if (isHealthy) {
+			return result;
+		}
+		throw new HealthCheckError('Consumer check failed', result);
+	}
+}

apps/server/src/infra/preview-generator/preview-generator-consumer.module.ts

@@ -1,7 +1,9 @@
-import { DynamicModule, Module } from '@nestjs/common';
-import { ConfigModule } from '@nestjs/config';
+import { HealthCheckModule } from '@infra/healthcheck/healthcheck.module';
 import { RabbitMQWrapperModule } from '@infra/rabbitmq';
 import { S3ClientAdapter, S3ClientModule } from '@infra/s3-client';
+import { DynamicModule, Module } from '@nestjs/common';
+import { ConfigModule } from '@nestjs/config';
+import { ScheduleModule } from '@nestjs/schedule';
 import { createConfigModuleOptions } from '@src/config';
 import { Logger, LoggerModule } from '@src/core/logger';
 import { PreviewConfig } from './interface/preview-consumer-config';
@@ -25,6 +27,8 @@ export class PreviewGeneratorConsumerModule {
 		return {
 			module: PreviewGeneratorConsumerModule,
 			imports: [
+				HealthCheckModule.register(),
+				ScheduleModule.forRoot(),
 				LoggerModule,
 				S3ClientModule.register([storageConfig]),
 				RabbitMQWrapperModule,

apps/server/src/infra/preview-generator/preview-generator.consumer.ts

@@ -1,15 +1,28 @@
-import { RabbitPayload, RabbitRPC } from '@golevelup/nestjs-rabbitmq';
+import { AmqpConnection, RabbitPayload, RabbitRPC } from '@golevelup/nestjs-rabbitmq';
+import { AmqpConsumerHealthIndicator } from '@infra/healthcheck/indicators/amqp-consumer.indictor';
 import { FilesPreviewEvents, FilesPreviewExchange } from '@infra/rabbitmq';
 import { Injectable } from '@nestjs/common';
+import { Cron } from '@nestjs/schedule';
 import { Logger } from '@src/core/logger';
 import { PreviewFileOptions } from './interface';
 import { PreviewActionsLoggable } from './loggable/preview-actions.loggable';
 import { PreviewGeneratorService } from './preview-generator.service';
 
 @Injectable()
 export class PreviewGeneratorConsumer {
-	constructor(private readonly previewGeneratorService: PreviewGeneratorService, private logger: Logger) {
+	constructor(
+		private readonly previewGeneratorService: PreviewGeneratorService,
+		private logger: Logger,
+		private amqpConnection: AmqpConnection,
+		private health: AmqpConsumerHealthIndicator
+	) {
 		this.logger.setContext(PreviewGeneratorConsumer.name);
+		health.setAmqpConnection(amqpConnection);
+	}
+
+	@Cron('5 * * * * *')
+	healty() {
+		console.log(this.health.isHealthy('preview-generator-consumer'));
 	}
 
 	@RabbitRPC({

apps/server/src/infra/preview-generator/preview-generator.service.ts

@@ -1,5 +1,5 @@
 import { GetFile, S3ClientAdapter } from '@infra/s3-client';
-import { Injectable, UnprocessableEntityException } from '@nestjs/common';
+import { Injectable, OnModuleInit, UnprocessableEntityException } from '@nestjs/common';
 import { Logger } from '@src/core/logger';
 import m, { subClass } from 'gm';
 import { PassThrough } from 'stream';
@@ -9,7 +9,11 @@ import { PreviewNotPossibleException } from './loggable/preview-exception';
 import { PreviewGeneratorBuilder } from './preview-generator.builder';
 
 @Injectable()
-export class PreviewGeneratorService {
+export class PreviewGeneratorService implements OnModuleInit {
+	onModuleInit() {
+		console.log('onModuleInit', PreviewGeneratorService.name);
+	}
+
 	private imageMagick = subClass({ imageMagick: '7+' });
 
 	constructor(private readonly storageClient: S3ClientAdapter, private logger: Logger) {

config/default.schema.json

@@ -180,7 +180,7 @@
 			"default": "",
 			"description": "Add custom domain to the list of blocked domains (comma separated list)."
 		},
-		"BLOCKLIST_OF_EMAIL_DOMAINS":{
+		"BLOCKLIST_OF_EMAIL_DOMAINS": {
 			"type": "string",
 			"default": "",
 			"description": "Add custom domain to the list of blocked domains (comma separated list)."
@@ -1034,6 +1034,11 @@
 			"description": "Nest Log level for api. The http flag is for request logging. The http flag do only work by api methods with added 'request logging interceptor'.",
 			"enum": ["emerg", "alert", "crit", "error", "warning", "notice", "info", "debug"]
 		},
+		"EXIT_ON_ERROR": {
+			"type": "boolean",
+			"default": false,
+			"description": "By default, the application is not terminated in our project after an uncaughtException has been logged. If this is not the desired behavior, set exitOnError to true."
+		},
 		"SYSTEM_LOG_LEVEL": {
 			"type": "string",
 			"default": "requestError",
@@ -1452,7 +1457,16 @@
 		"TLDRAW": {
 			"type": "object",
 			"description": "Configuration of tldraw related settings",
-			"required": ["PING_TIMEOUT", "SOCKET_PORT","GC_ENABLED", "DB_FLUSH_SIZE", "MAX_DOCUMENT_SIZE", "ASSETS_ENABLED", "ASSETS_MAX_SIZE", "ASSETS_ALLOWED_EXTENSIONS_LIST"],
+			"required": [
+				"PING_TIMEOUT",
+				"SOCKET_PORT",
+				"GC_ENABLED",
+				"DB_FLUSH_SIZE",
+				"MAX_DOCUMENT_SIZE",
+				"ASSETS_ENABLED",
+				"ASSETS_MAX_SIZE",
+				"ASSETS_ALLOWED_EXTENSIONS_LIST"
+			],
 			"properties": {
 				"SOCKET_PORT": {
 					"type": "number",