From 150f02383bec4c727f9dd27ac23f1a0c4be43015 Mon Sep 17 00:00:00 2001
From: ThanKarab <tkarabatsis@hotmail.com>
Date: Fri, 29 Nov 2024 01:44:08 +0200
Subject: [PATCH] Add logstash support for docker and kubernetes

Tag messages coming from docker or k8s. Docker logs should have their prefix removed and then parsed the same way as k8s logs.
---
 dev/docker-compose.yml                    |  8 ++++----
 dev/stop.sh                               |  2 +-
 elk_stack/docker-compose.yml              |  1 +
 elk_stack/logstash/pipeline/logstash.conf | 25 +++++++++++++++--------
 4 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index 5c00882a..90b43343 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -98,7 +98,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   exareme2_global:
     image: madgik/exareme2_worker:${EXAREME2}
@@ -129,7 +129,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   exareme2_controller:
     image: madgik/exareme2_controller:${EXAREME2}
@@ -157,7 +157,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   portalbackend_db:
     image: postgres:11.20-alpine
@@ -219,7 +219,7 @@ services:
 #    logging:   # Used for sending logs to ELK
 #      driver: "syslog"
 #      options:
-#        syslog-address: "tcp://localhost:5010"
+#        syslog-address: "tcp://localhost:5020"
 
   gateway-db:
     image: postgres
diff --git a/dev/stop.sh b/dev/stop.sh
index af014369..a6f17a8c 100755
--- a/dev/stop.sh
+++ b/dev/stop.sh
@@ -1,5 +1,5 @@
 #!/bin/env bash
 
-docker-compose --env-file ../.versions_env down
+docker compose --env-file ../.versions_env down
 rm ../data/local.db
 rm ../data/global.db
diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml
index ccbe7928..2d483cef 100644
--- a/elk_stack/docker-compose.yml
+++ b/elk_stack/docker-compose.yml
@@ -30,6 +30,7 @@ services:
       - "5044:5044"
       - "9600:9600"
       - "5010:5010/tcp"
+      - "5020:5020/tcp"
     networks:
       - elk
     depends_on:
diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf
index c61c1c79..f154504f 100644
--- a/elk_stack/logstash/pipeline/logstash.conf
+++ b/elk_stack/logstash/pipeline/logstash.conf
@@ -1,8 +1,14 @@
 input
 {
-  tcp {
+  beats {
     port => 5010
-    type => syslog
+    add_field => { "source" => "kubernetes" }
+  }
+
+  tcp {
+    port => 5020
+    type => "syslog"
+    add_field => { "source" => "docker" }
     codec => multiline {
       pattern => "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}"
       negate => true
@@ -12,21 +18,22 @@ input
 }
 
 filter {
-  mutate{
-    gsub => [ "message", "\\n", "" ]
+  if [source] == "docker" {
+    mutate {        #### Remove docker syslog driver prefix (only regex for multiline removal)
+      gsub => ["message", "<\d+>\w+\s+\d+\s+\d{2}:\d{2}:\d{2}\s+[^:]+:\s+", ""]
+    }
   }
   
   grok {
-    match => [      
-    
+    match => [
       # ----- Match EXAREME2 logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}",
       
       # ----- Match PORTAL-BACKEND user generated logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}",
       
       # ----- Match PORTAL-BACKEND system generated logs -----
-      "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
+      "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}"
     ]
     
     add_tag => [ "parsed" ]