diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 5c00882..90b4334 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -98,7 +98,7 @@ services: # logging: # Used for sending logs to ELK # driver: "syslog" # options: -# syslog-address: "tcp://localhost:5010" +# syslog-address: "tcp://localhost:5020" exareme2_global: image: madgik/exareme2_worker:${EXAREME2} @@ -129,7 +129,7 @@ services: # logging: # Used for sending logs to ELK # driver: "syslog" # options: -# syslog-address: "tcp://localhost:5010" +# syslog-address: "tcp://localhost:5020" exareme2_controller: image: madgik/exareme2_controller:${EXAREME2} @@ -157,7 +157,7 @@ services: # logging: # Used for sending logs to ELK # driver: "syslog" # options: -# syslog-address: "tcp://localhost:5010" +# syslog-address: "tcp://localhost:5020" portalbackend_db: image: postgres:11.20-alpine @@ -219,7 +219,7 @@ services: # logging: # Used for sending logs to ELK # driver: "syslog" # options: -# syslog-address: "tcp://localhost:5010" +# syslog-address: "tcp://localhost:5020" gateway-db: image: postgres diff --git a/dev/stop.sh b/dev/stop.sh index af01436..a6f17a8 100755 --- a/dev/stop.sh +++ b/dev/stop.sh @@ -1,5 +1,5 @@ #!/bin/env bash -docker-compose --env-file ../.versions_env down +docker compose --env-file ../.versions_env down rm ../data/local.db rm ../data/global.db diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml index ccbe792..2d483ce 100644 --- a/elk_stack/docker-compose.yml +++ b/elk_stack/docker-compose.yml @@ -30,6 +30,7 @@ services: - "5044:5044" - "9600:9600" - "5010:5010/tcp" + - "5020:5020/tcp" networks: - elk depends_on: diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf index c61c1c7..f154504 100644 --- a/elk_stack/logstash/pipeline/logstash.conf +++ b/elk_stack/logstash/pipeline/logstash.conf @@ -1,8 +1,14 @@ input { - tcp { + beats { port => 5010 - type => syslog + add_field => { "source" => "kubernetes" } + } + + tcp { + port => 5020 + type => "syslog" + add_field => { "source" => "docker" } codec => multiline { pattern => "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}" negate => true @@ -12,21 +18,22 @@ input } filter { - mutate{ - gsub => [ "message", "\\n", "" ] + if [source] == "docker" { + mutate { #### Remove docker syslog driver prefix (only regex for multiline removal) + gsub => ["message", "<\d+>\w+\s+\d+\s+\d{2}:\d{2}:\d{2}\s+[^:]+:\s+", ""] + } } grok { - match => [ - + match => [ # ----- Match EXAREME2 logs ----- - "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}", + "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}", # ----- Match PORTAL-BACKEND user generated logs ----- - "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}", + "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}", # ----- Match PORTAL-BACKEND system generated logs ----- - "message", "<%{NUMBER}>%{MONTH}%{SPACE}%{MONTHDAY}%{SPACE}%{TIME}%{SPACE}%{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}" + "message", "%{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}" ] add_tag => [ "parsed" ]