From d78be85fce79cceb502de2bf2bab2b1586192523 Mon Sep 17 00:00:00 2001
From: ThanKarab <tkarabatsis@hotmail.com>
Date: Wed, 30 Oct 2024 21:44:50 +0200
Subject: [PATCH] Exareme2 integration with ELK stack

New exareme2 version with logs ready for the elk integration.
Added the exareme2 log patterns in logstash.
Added support for multiline logs, changed ports to tcp instead of udp.
---
 .versions_env                             |  2 +-
 dev/docker-compose.yml                    | 35 +++++++++++++++++------
 elk_stack/docker-compose.yml              |  2 +-
 elk_stack/logstash/pipeline/logstash.conf | 14 +++++++--
 4 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/.versions_env b/.versions_env
index 5850499a..982cbf34 100644
--- a/.versions_env
+++ b/.versions_env
@@ -1,4 +1,4 @@
-EXAREME2=0.22.0
+EXAREME2=0.23.0
 PORTALBACKEND=8.1.0
 GATEWAY=1.7.0
 FRONTEND=9.4.0
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index c12ed4b3..d48a42fb 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -58,7 +58,6 @@ services:
     volumes:
       - ../data:/opt/data
 
-
   exareme2_global_mipdb:
     image: madgik/exareme2_mipdb:${EXAREME2}
     environment:
@@ -74,11 +73,12 @@ services:
     volumes:
       - ../data:/opt/data
     environment:
+      - FEDERATION=dementia
       - WORKER_IDENTIFIER=local
       - WORKER_ROLE=LOCALWORKER
       - DATA_PATH=/opt/data
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - CONTROLLER_IP=172.17.0.1
       - CONTROLLER_PORT=5000
       - CELERY_TASKS_TIMEOUT=20
@@ -94,17 +94,22 @@ services:
       - exareme2_local_monetdb
       - exareme2_local_rabbitmq
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   exareme2_global:
     image: madgik/exareme2_worker:${EXAREME2}
     volumes:
       - ../data:/opt/data
     environment:
+      - FEDERATION=dementia
       - WORKER_IDENTIFIER=global
       - WORKER_ROLE=GLOBALWORKER
       - DATA_PATH=/opt/data
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - CONTROLLER_IP=172.17.0.1
       - CONTROLLER_PORT=5000
       - CELERY_TASKS_TIMEOUT=20
@@ -120,14 +125,20 @@ services:
       - exareme2_global_monetdb
       - exareme2_global_rabbitmq
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   exareme2_controller:
     image: madgik/exareme2_controller:${EXAREME2}
     ports:
       - '5000:5000'
     environment:
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - FEDERATION=dementia
+      - NODE_IDENTIFIER=controller
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - DEPLOYMENT_TYPE=LOCAL
       - FLOWER_EXECUTION_TIMEOUT=30
       - WORKER_LANDSCAPE_AGGREGATOR_UPDATE_INTERVAL=500
@@ -142,6 +153,10 @@ services:
     volumes:
       - ./config/:/opt/config/
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   portalbackend_db:
     image: postgres:11.20-alpine
@@ -175,7 +190,7 @@ services:
       - '8080:8080'
       - '8089:8089'
     environment:
-      ### API ###
+      ### LOGGER ###
       LOG_LEVEL: INFO
       LOG_LEVEL_FRAMEWORK: INFO
       ALGORITHM_UPDATE_INTERVAL: 30     # seconds
@@ -199,6 +214,10 @@ services:
     volumes:
       - ./config:/opt/portal/api
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   gateway-db:
     image: postgres
diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml
index 6a6186c2..ccbe7928 100644
--- a/elk_stack/docker-compose.yml
+++ b/elk_stack/docker-compose.yml
@@ -29,7 +29,7 @@ services:
     ports:
       - "5044:5044"
       - "9600:9600"
-      - "5010:5010/udp"
+      - "5010:5010/tcp"
     networks:
       - elk
     depends_on:
diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf
index fe55c11a..3e751944 100644
--- a/elk_stack/logstash/pipeline/logstash.conf
+++ b/elk_stack/logstash/pipeline/logstash.conf
@@ -1,8 +1,13 @@
 input
 {
-  udp {
+  tcp {
     port => 5010
     type => syslog
+    codec => multiline {
+      pattern => "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}"
+      negate => true
+      what => "previous"
+    }
   }
 }
 
@@ -13,11 +18,14 @@ filter {
   
   grok {
     match => [      
+    
+      # ----- Match EXAREME2 logs -----
+      "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
       
-      # ----- Match user generated logs -----
+      # ----- Match PORTAL-BACKEND user generated logs -----
       "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
       
-      # ----- Match system generated logs -----
+      # ----- Match PORTAL-BACKEND system generated logs -----
       "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
     ]