Exareme2 integration with ELK stack

New exareme2 version with logs ready for the elk integration. Added the exareme2 log patterns in logstash. Added support for multiline logs, changed ports to tcp instead of udp.
HBPMedical · Nov 1, 2024 · d78be85 · d78be85
1 parent c96d3f4
commit d78be85
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 13 deletions.
diff --git a/.versions_env b/.versions_env
@@ -1,4 +1,4 @@
-EXAREME2=0.22.0
+EXAREME2=0.23.0
 PORTALBACKEND=8.1.0
 GATEWAY=1.7.0
 FRONTEND=9.4.0

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
@@ -58,7 +58,6 @@ services:
     volumes:
       - ../data:/opt/data
 
-
   exareme2_global_mipdb:
     image: madgik/exareme2_mipdb:${EXAREME2}
     environment:
@@ -74,11 +73,12 @@ services:
     volumes:
       - ../data:/opt/data
     environment:
+      - FEDERATION=dementia
       - WORKER_IDENTIFIER=local
       - WORKER_ROLE=LOCALWORKER
       - DATA_PATH=/opt/data
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - CONTROLLER_IP=172.17.0.1
       - CONTROLLER_PORT=5000
       - CELERY_TASKS_TIMEOUT=20
@@ -94,17 +94,22 @@ services:
       - exareme2_local_monetdb
       - exareme2_local_rabbitmq
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   exareme2_global:
     image: madgik/exareme2_worker:${EXAREME2}
     volumes:
       - ../data:/opt/data
     environment:
+      - FEDERATION=dementia
       - WORKER_IDENTIFIER=global
       - WORKER_ROLE=GLOBALWORKER
       - DATA_PATH=/opt/data
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - CONTROLLER_IP=172.17.0.1
       - CONTROLLER_PORT=5000
       - CELERY_TASKS_TIMEOUT=20
@@ -120,14 +125,20 @@ services:
       - exareme2_global_monetdb
       - exareme2_global_rabbitmq
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   exareme2_controller:
     image: madgik/exareme2_controller:${EXAREME2}
     ports:
       - '5000:5000'
     environment:
-      - LOG_LEVEL=DEBUG
-      - FRAMEWORK_LOG_LEVEL=INFO
+      - FEDERATION=dementia
+      - NODE_IDENTIFIER=controller
+      - LOG_LEVEL=INFO
+      - FRAMEWORK_LOG_LEVEL=WARNING
       - DEPLOYMENT_TYPE=LOCAL
       - FLOWER_EXECUTION_TIMEOUT=30
       - WORKER_LANDSCAPE_AGGREGATOR_UPDATE_INTERVAL=500
@@ -142,6 +153,10 @@ services:
     volumes:
       - ./config/:/opt/config/
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   portalbackend_db:
     image: postgres:11.20-alpine
@@ -175,7 +190,7 @@ services:
       - '8080:8080'
       - '8089:8089'
     environment:
-      ### API ###
+      ### LOGGER ###
       LOG_LEVEL: INFO
       LOG_LEVEL_FRAMEWORK: INFO
       ALGORITHM_UPDATE_INTERVAL: 30     # seconds
@@ -199,6 +214,10 @@ services:
     volumes:
       - ./config:/opt/portal/api
     restart: unless-stopped
+    logging:
+      driver: "syslog"
+      options:
+        syslog-address: "tcp://localhost:5010"
 
   gateway-db:
     image: postgres

diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml
@@ -29,7 +29,7 @@ services:
     ports:
       - "5044:5044"
       - "9600:9600"
-      - "5010:5010/udp"
+      - "5010:5010/tcp"
     networks:
       - elk
     depends_on:

diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf
@@ -1,8 +1,13 @@
 input
 {
-  udp {
+  tcp {
     port => 5010
     type => syslog
+    codec => multiline {
+      pattern => "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}"
+      negate => true
+      what => "previous"
+    }
   }
 }
 
@@ -13,11 +18,14 @@ filter {
 
   grok {
     match => [      
+
+      # ----- Match EXAREME2 logs -----
+      "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
 
-      # ----- Match user generated logs -----
+      # ----- Match PORTAL-BACKEND user generated logs -----
       "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
 
-      # ----- Match system generated logs -----
+      # ----- Match PORTAL-BACKEND system generated logs -----
       "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
     ]