Merge pull request #14

Integrate ELK stack

.env-sample

@@ -0,0 +1,7 @@
+DB_USER=
+DB_PASS=
+RABBIT_USER=
+RABBIT_PASS=
+TELEGRAM_BOT_TOKEN=
+SPRING_DATASOURCE_URL=
+SPRING_JPA_HIBERNATE_DDL_AUTO=

.gitignore

@@ -34,3 +34,46 @@ build/
 /data/
 /postgres-data/
 /redis-data/
+/rabbit-data/
+
+# Ignore the logs directory
+logs/community.log
+*.log
+
+# Ignore the Docker volumes
+volumes/
+redis-data/
+
+# Ignore environment variable files
+.env
+*.env
+
+# Ignore IntelliJ IDEA project files
+.idea/
+
+# Ignore Node.js modules
+node_modules/
+# Ignore temporary files
+*.swp
+*.temp
+
+# Ignore Spring Boot specific files
+*.pid
+
+# Ignore Prometheus data
+prometheus/
+
+# Ignore local configuration files
+local_config.yml
+
+# Ignore compiled Python files
+__pycache__/
+*.py[cod]
+
+# Ignore system files
+.DS_Store
+Thumbs.db
+
+# Ignore docker-compose override files
+docker-compose.override.yml
+

docker-compose.yml

@@ -22,8 +22,9 @@ services:
     image: 'postgres:13.1-alpine'
     container_name: db
     environment:
-      - POSTGRES_USER=compose-postgres
-      - POSTGRES_PASSWORD=compose-postgres
+      - POSTGRES_DB=${DB_NAME}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_PASSWORD=${DB_PASS}
     ports:
       - "5432:5432"
 
@@ -39,11 +40,11 @@ services:
     image: rabbitmq:3-management
     container_name: rabbitmq
     ports:
-      - 5672:5672
-      - 15672:15672
+      - "5672:5672"
+      - "15672:15672"
     environment:
-      - RABBITMQ_DEFAULT_USER=alex
-      - RABBITMQ_DEFAULT_PASS=alex
+      - RABBITMQ_DEFAULT_USER=${RABBIT_USER}
+      - RABBITMQ_DEFAULT_PASS=${RABBIT_PASS}
 
   app:
     image: overpathz/communitybot
@@ -56,12 +57,14 @@ services:
       - rabbitmq
       - db
     environment:
-      - SPRING_DATASOURCE_URL=jdbc:postgresql://db:5432/compose-postgres
-      - SPRING_DATASOURCE_USERNAME=compose-postgres
-      - SPRING_DATASOURCE_PASSWORD=compose-postgres
-      - SPRING_JPA_HIBERNATE_DDL_AUTO=update
-      - SPRING_RABBITMQ_HOST=rabbitmq
-      - SPRING_RABBITMQ_PORT=5672
-      - SPRING_RABBITMQ_USERNAME=alex
-      - SPRING_RABBITMQ_PASSWORD=alex
-      - SPRING_PROFILES_ACTIVE=prometheus
+      SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${DB_NAME}
+      SPRING_DATASOURCE_USERNAME: ${DB_USER}
+      SPRING_DATASOURCE_PASSWORD: ${DB_PASS}
+      SPRING_JPA_HIBERNATE_DDL_AUTO: update
+      SPRING_RABBITMQ_USERNAME: ${RABBIT_USER}
+      SPRING_RABBITMQ_PASSWORD: ${RABBIT_PASS}
+      SPRING_RABBITMQ_HOST: ${RABBIT_HOST}
+      TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN}
+      SPRING_PROFILES_ACTIVE: prometheus
+    volumes:
+      - ./logs:/app/logs

elk/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.8'
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.13
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    networks:
+      - elk-network
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:7.17.13
+    ports:
+      - "50000:50000"
+    volumes:
+      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+      - ./logstash/pipeline:/usr/share/logstash/pipeline
+      - ../logs:/usr/share/logstash/logs
+    command: [ "-f", "/usr/share/logstash/pipeline/logstash.conf"]
+    depends_on:
+      - elasticsearch
+    networks:
+      - elk-network
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:7.17.13
+    ports:
+      - "5601:5601"
+    depends_on:
+      - logstash
+    networks:
+      - elk-network
+
+volumes:
+  esdata:
+    driver: local
+
+networks:
+  elk-network:
+    driver: bridge

elk/logstash/config/logstash.yml

@@ -0,0 +1,5 @@
+http.host: "0.0.0.0"
+log.level: debug
+node.name: logstash
+path.config: /usr/share/logstash/pipeline/logstash.conf
+path.logs: /usr/share/logstash/logs

elk/logstash/pipeline/logstash.conf

@@ -0,0 +1,17 @@
+input {
+  file {
+    path => "/usr/share/logstash/logs/*.log"
+    start_position => "beginning"
+    sincedb_path => "/dev/null"
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+  elasticsearch {
+    hosts => ["elasticsearch:9200"]
+    index => "service-%{+YYYY.MM.dd}"
+  }
+}

pom.xml

@@ -130,6 +130,11 @@
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-actuator</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>net.logstash.logback</groupId>
+			<artifactId>logstash-logback-encoder</artifactId>
+			<version>5.3</version>
+		</dependency>
 	</dependencies>
 
 	<build>

src/main/resources/application.yml

@@ -39,12 +39,12 @@ spring:
     redis:
       url: redis://redis:6379
   datasource:
-    url: jdbc:postgresql://localhost:5432/community
-    username: postgres
-    password: postgres
+    url: jdbc:postgresql://db:5432/${DB_NAME}
+    username: ${DB_USER}
+    password: ${DB_PASS}
   rabbitmq:
-    username: alex
-    password: alex
+    username: ${RABBIT_USER}
+    password: ${RABBIT_USER}
 
 amazon:
   s3:
@@ -55,6 +55,8 @@ amazon:
 logging:
   level:
     root: info
+  file:
+    name: logs/community.log
 server:
   port: 8080