Add logstash support for docker and kubernetes

Tag messages coming from docker or k8s. Docker logs should have their prefix removed and then parsed the same way as k8s logs.
HBPMedical · Dec 2, 2024 · c232718 · c232718
1 parent e0d4da5
commit c232718
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 14 deletions.
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
@@ -98,7 +98,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   exareme2_global:
     image: madgik/exareme2_worker:${EXAREME2}
@@ -129,7 +129,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   exareme2_controller:
     image: madgik/exareme2_controller:${EXAREME2}
@@ -157,7 +157,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   portalbackend_db:
     image: postgres:11.20-alpine
@@ -219,7 +219,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   gateway-db:
     image: postgres

diff --git a/dev/stop.sh b/dev/stop.sh
@@ -1,5 +1,5 @@
 #!/bin/env bash
 
-docker-compose --env-file ../.versions_env down
+docker compose --env-file ../.versions_env down
 rm ../data/local.db
 rm ../data/global.db
diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml
@@ -30,6 +30,7 @@ services:
       - "5044:5044"
       - "9600:9600"
       - "5010:5010/tcp"
+      - "5020:5020/tcp"
     networks:
       - elk
     depends_on:

diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf
@@ -1,8 +1,14 @@
 input
 {
-  tcp {
+  beats {
     port => 5010
-    type => syslog
+    add_field => { "source" => "kubernetes" }
+  }
+
+  tcp {
+    port => 5020
+    type => "syslog"
+    add_field => { "source" => "docker" }
     codec => multiline {
       pattern => "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}"
       negate => true
@@ -12,21 +18,22 @@ input
 }
 
 filter {
-  mutate{
-    gsub => [ "message", "\\n", "" ]
+  if [source] == "docker" {
+    mutate {        #### Remove docker syslog driver prefix (only regex for multiline removal)
+      gsub => ["message", "<\d+>\w+\s+\d+\s+\d{2}:\d{2}:\d{2}\s+[^:]+:\s+", ""]
+    }
   }
 
   grok {
-    match => [      
-
+    match => [
       # ----- Match EXAREME2 logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
 
       # ----- Match PORTAL-BACKEND user generated logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
 
       # ----- Match PORTAL-BACKEND system generated logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
     ]
 
     add_tag => [ "parsed" ]