diff --git a/.versions_env b/.versions_env index 5850499a..982cbf34 100644 --- a/.versions_env +++ b/.versions_env @@ -1,4 +1,4 @@ -EXAREME2=0.22.0 +EXAREME2=0.23.0 PORTALBACKEND=8.1.0 GATEWAY=1.7.0 FRONTEND=9.4.0 diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index c12ed4b3..d48a42fb 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -58,7 +58,6 @@ services: volumes: - ../data:/opt/data - exareme2_global_mipdb: image: madgik/exareme2_mipdb:${EXAREME2} environment: @@ -74,11 +73,12 @@ services: volumes: - ../data:/opt/data environment: + - FEDERATION=dementia - WORKER_IDENTIFIER=local - WORKER_ROLE=LOCALWORKER - DATA_PATH=/opt/data - - LOG_LEVEL=DEBUG - - FRAMEWORK_LOG_LEVEL=INFO + - LOG_LEVEL=INFO + - FRAMEWORK_LOG_LEVEL=WARNING - CONTROLLER_IP=172.17.0.1 - CONTROLLER_PORT=5000 - CELERY_TASKS_TIMEOUT=20 @@ -94,17 +94,22 @@ services: - exareme2_local_monetdb - exareme2_local_rabbitmq restart: unless-stopped + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:5010" exareme2_global: image: madgik/exareme2_worker:${EXAREME2} volumes: - ../data:/opt/data environment: + - FEDERATION=dementia - WORKER_IDENTIFIER=global - WORKER_ROLE=GLOBALWORKER - DATA_PATH=/opt/data - - LOG_LEVEL=DEBUG - - FRAMEWORK_LOG_LEVEL=INFO + - LOG_LEVEL=INFO + - FRAMEWORK_LOG_LEVEL=WARNING - CONTROLLER_IP=172.17.0.1 - CONTROLLER_PORT=5000 - CELERY_TASKS_TIMEOUT=20 @@ -120,14 +125,20 @@ services: - exareme2_global_monetdb - exareme2_global_rabbitmq restart: unless-stopped + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:5010" exareme2_controller: image: madgik/exareme2_controller:${EXAREME2} ports: - '5000:5000' environment: - - LOG_LEVEL=DEBUG - - FRAMEWORK_LOG_LEVEL=INFO + - FEDERATION=dementia + - NODE_IDENTIFIER=controller + - LOG_LEVEL=INFO + - FRAMEWORK_LOG_LEVEL=WARNING - DEPLOYMENT_TYPE=LOCAL - FLOWER_EXECUTION_TIMEOUT=30 - WORKER_LANDSCAPE_AGGREGATOR_UPDATE_INTERVAL=500 @@ -142,6 +153,10 @@ services: volumes: - ./config/:/opt/config/ restart: unless-stopped + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:5010" portalbackend_db: image: postgres:11.20-alpine @@ -175,7 +190,7 @@ services: - '8080:8080' - '8089:8089' environment: - ### API ### + ### LOGGER ### LOG_LEVEL: INFO LOG_LEVEL_FRAMEWORK: INFO ALGORITHM_UPDATE_INTERVAL: 30 # seconds @@ -199,6 +214,10 @@ services: volumes: - ./config:/opt/portal/api restart: unless-stopped + logging: + driver: "syslog" + options: + syslog-address: "tcp://localhost:5010" gateway-db: image: postgres diff --git a/elk_stack/docker-compose.yml b/elk_stack/docker-compose.yml index 6a6186c2..ccbe7928 100644 --- a/elk_stack/docker-compose.yml +++ b/elk_stack/docker-compose.yml @@ -29,7 +29,7 @@ services: ports: - "5044:5044" - "9600:9600" - - "5010:5010/udp" + - "5010:5010/tcp" networks: - elk depends_on: diff --git a/elk_stack/logstash/pipeline/logstash.conf b/elk_stack/logstash/pipeline/logstash.conf index fe55c11a..3e751944 100644 --- a/elk_stack/logstash/pipeline/logstash.conf +++ b/elk_stack/logstash/pipeline/logstash.conf @@ -1,8 +1,13 @@ input { - udp { + tcp { port => 5010 type => syslog + codec => multiline { + pattern => "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601} %{GREEDYDATA}" + negate => true + what => "previous" + } } } @@ -13,11 +18,14 @@ filter { grok { match => [ + + # ----- Match EXAREME2 logs ----- + "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel} - %{DATA:method} - \[%{DATA:federation}\] - \[%{DATA:service}\] - \[%{DATA:node_id}\] - \[%{DATA:request_id}\] - %{GREEDYDATA:log_message}", - # ----- Match user generated logs ----- + # ----- Match PORTAL-BACKEND user generated logs ----- "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - User -> %{DATA:user} , Endpoint -> \(%{WORD:http_method}\) %{URIPATH:http_path} , Info -> %{GREEDYDATA:log_message}", - # ----- Match system generated logs ----- + # ----- Match PORTAL-BACKEND system generated logs ----- "message", "<%{NUMBER}>%{MONTH} %{MONTHDAY} %{TIME} %{DATA}: %{TIMESTAMP_ISO8601:log_timestamp} - %{LOGLEVEL:loglevel}%{SPACE}- %{DATA:logger} - \[%{DATA:federation}\] - \[%{DATA:service}\] - %{GREEDYDATA:log_message}" ]