From b4866aeb76c4b7bbffe84ce1fa461e6e4d69a3fe Mon Sep 17 00:00:00 2001 From: overpathz Date: Mon, 8 Jul 2024 12:57:01 +0300 Subject: [PATCH 1/8] update gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index f0e0037..bb031e5 100644 --- a/.gitignore +++ b/.gitignore @@ -60,9 +60,6 @@ node_modules/ # Ignore Spring Boot specific files *.pid -# Ignore Prometheus data -prometheus/ - # Ignore local configuration files local_config.yml From 111659e508f371150658a311d44673ad0f323229 Mon Sep 17 00:00:00 2001 From: overpathz Date: Mon, 8 Jul 2024 12:57:48 +0300 Subject: [PATCH 2/8] grafana/prometheus --- docker-compose.yml | 57 +++++++----------- {elk => elk-compose}/docker-compose.yml | 0 .../logstash/config/logstash.yml | 0 .../logstash/pipeline/logstash.conf | 0 monitoring-compose/docker-compose.yml | 23 +++++++ .../prometheus}/prometheus.yml | 2 +- run-project.bat | 60 +++++++++++++++++++ run-project.sh | 55 +++++++++++++++++ 8 files changed, 159 insertions(+), 38 deletions(-) rename {elk => elk-compose}/docker-compose.yml (100%) rename {elk => elk-compose}/logstash/config/logstash.yml (100%) rename {elk => elk-compose}/logstash/pipeline/logstash.conf (100%) create mode 100644 monitoring-compose/docker-compose.yml rename {prometheus => monitoring-compose/prometheus}/prometheus.yml (76%) create mode 100644 run-project.bat create mode 100755 run-project.sh diff --git a/docker-compose.yml b/docker-compose.yml index c60277b..7ad7708 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,28 @@ version: '1' services: - prometheus: - image: prom/prometheus - ports: - - "9090:9090" - volumes: - - ./prometheus:/etc/prometheus - command: - - --config.file=/etc/prometheus/prometheus.yml - depends_on: - - app - grafana: - image: grafana/grafana + app: + image: overpathz/communitybot + build: . + container_name: community-app ports: - - "3000:3000" + - "8080:8080" depends_on: - - prometheus + - redis + - rabbitmq + - db + environment: + SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${DB_NAME} + SPRING_DATASOURCE_USERNAME: ${DB_USER} + SPRING_DATASOURCE_PASSWORD: ${DB_PASS} + SPRING_JPA_HIBERNATE_DDL_AUTO: update + SPRING_RABBITMQ_USERNAME: ${RABBIT_USER} + SPRING_RABBITMQ_PASSWORD: ${RABBIT_PASS} + SPRING_RABBITMQ_HOST: ${RABBIT_HOST} + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} + SPRING_PROFILES_ACTIVE: prometheus + volumes: + - ./logs:/app/logs db: image: 'postgres:13.1-alpine' @@ -45,26 +51,3 @@ services: environment: - RABBITMQ_DEFAULT_USER=${RABBIT_USER} - RABBITMQ_DEFAULT_PASS=${RABBIT_PASS} - - app: - image: overpathz/communitybot - build: . - container_name: community-app - ports: - - "8080:8080" - depends_on: - - redis - - rabbitmq - - db - environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://db:5432/${DB_NAME} - SPRING_DATASOURCE_USERNAME: ${DB_USER} - SPRING_DATASOURCE_PASSWORD: ${DB_PASS} - SPRING_JPA_HIBERNATE_DDL_AUTO: update - SPRING_RABBITMQ_USERNAME: ${RABBIT_USER} - SPRING_RABBITMQ_PASSWORD: ${RABBIT_PASS} - SPRING_RABBITMQ_HOST: ${RABBIT_HOST} - TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} - SPRING_PROFILES_ACTIVE: prometheus - volumes: - - ./logs:/app/logs diff --git a/elk/docker-compose.yml b/elk-compose/docker-compose.yml similarity index 100% rename from elk/docker-compose.yml rename to elk-compose/docker-compose.yml diff --git a/elk/logstash/config/logstash.yml b/elk-compose/logstash/config/logstash.yml similarity index 100% rename from elk/logstash/config/logstash.yml rename to elk-compose/logstash/config/logstash.yml diff --git a/elk/logstash/pipeline/logstash.conf b/elk-compose/logstash/pipeline/logstash.conf similarity index 100% rename from elk/logstash/pipeline/logstash.conf rename to elk-compose/logstash/pipeline/logstash.conf diff --git a/monitoring-compose/docker-compose.yml b/monitoring-compose/docker-compose.yml new file mode 100644 index 0000000..fdb1fcd --- /dev/null +++ b/monitoring-compose/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.8' + +services: + prometheus: + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus:/etc/prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + healthcheck: + test: [ "CMD-SHELL", "curl -f $${SPRING_APP_HEALTH_URL} || exit 1" ] + interval: 30s + timeout: 10s + retries: 3 + + grafana: + image: grafana/grafana + ports: + - "3000:3000" + depends_on: + - prometheus \ No newline at end of file diff --git a/prometheus/prometheus.yml b/monitoring-compose/prometheus/prometheus.yml similarity index 76% rename from prometheus/prometheus.yml rename to monitoring-compose/prometheus/prometheus.yml index c2ba9f5..cd2d572 100644 --- a/prometheus/prometheus.yml +++ b/monitoring-compose/prometheus/prometheus.yml @@ -5,4 +5,4 @@ scrape_configs: - job_name: 'app' metrics_path: '/actuator/prometheus' static_configs: - - targets: ['app:8080'] \ No newline at end of file + - targets: ['192.168.10.19:8080'] \ No newline at end of file diff --git a/run-project.bat b/run-project.bat new file mode 100644 index 0000000..82252be --- /dev/null +++ b/run-project.bat @@ -0,0 +1,60 @@ +@echo off +setlocal enabledelayedexpansion + +set BASE_DIR=%cd% +set REBUILD_CONTAINERS=false + +rem Check for rebuild flag +if "%1"=="--rebuild" ( + set REBUILD_CONTAINERS=true +) + +rem Function to wait for service +:wait_for_service +set service_name=%1 +set url=%2 +set retry_count=20 +set retry_interval=10 + +echo Waiting for %service_name% to be ready at %url%... + +for /L %%i in (1,1,%retry_count%) do ( + powershell -Command "try { $response = Invoke-WebRequest -Uri %url% -UseBasicParsing; if ($response.StatusCode -eq 200) { exit 0 } } catch { exit 1 }" + if !errorlevel! == 0 ( + echo %service_name% is up! + exit /b 0 + ) + echo Waiting for %service_name%... (Attempt %%i/%retry_count%) + timeout /t %retry_interval% >nul +) + +echo Error: %service_name% is not responding at %url% +exit /b 1 + +rem Start ELK stack +echo Starting ELK stack... +docker-compose -f "%BASE_DIR%\elk-compose\docker-compose.yml" up -d + +rem Wait for ELK stack to be up +call :wait_for_service "Elasticsearch" "http://localhost:9200" +if %errorlevel% neq 0 exit /b 1 +call :wait_for_service "Kibana" "http://localhost:5601" +if %errorlevel% neq 0 exit /b 1 + +rem Start app stack +echo Starting app stack... +if "%REBUILD_CONTAINERS%"=="true" ( + docker-compose -f "%BASE_DIR%\docker-compose.yml" up -d --build +) else ( + docker-compose -f "%BASE_DIR%\docker-compose.yml" up -d +) + +rem Wait for the app to be up (modify according to your app's health check) +call :wait_for_service "App" "http://localhost:8080/health" +if %errorlevel% neq 0 exit /b 1 + +rem Start Grafana stack +echo Starting Grafana stack... +docker-compose -f "%BASE_DIR%\monitoring-compose\docker-compose.yml" up -d + +echo All services are up and running! diff --git a/run-project.sh b/run-project.sh new file mode 100755 index 0000000..23e947f --- /dev/null +++ b/run-project.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +function wait_for_service() { + local service_name=$1 + local url=$2 + local retry_count=20 + local retry_interval=10 + + echo "Waiting for $service_name to be ready at $url..." + + for i in $(seq 1 $retry_count); do + if curl -s $url > /dev/null; then + echo "$service_name is up!" + return 0 + fi + echo "Waiting for $service_name... (Attempt $i/$retry_count)" + sleep $retry_interval + done + + echo "Error: $service_name is not responding at $url" + return 1 +} + +BASE_DIR=$(pwd) +REBUILD_CONTAINERS=false + +# Check for rebuild flag +if [[ "$1" == "--rebuild" ]]; then + REBUILD_CONTAINERS=true +fi + +# Start ELK stack +echo "Starting ELK stack..." +docker-compose -f "$BASE_DIR/elk-compose/docker-compose.yml" up -d + +# Wait for ELK stack to be up +wait_for_service "Elasticsearch" "http://localhost:9200" || exit 1 +wait_for_service "Kibana" "http://localhost:5601" || exit 1 + +# Start app stack +echo "Starting app stack..." +if [ "$REBUILD_CONTAINERS" = true ]; then + docker-compose -f "$BASE_DIR/docker-compose.yml" up -d --build +else + docker-compose -f "$BASE_DIR/docker-compose.yml" up -d +fi + +# Wait for the app to be up (modify according to your app's health check) +wait_for_service "App" "http://localhost:8080/health" || exit 1 + +# Start Grafana stack +echo "Starting Grafana stack..." +docker-compose -f "$BASE_DIR/monitoring-compose/docker-compose.yml" up -d + +echo "All services are up and running!" From b262e22b93278088ab8872d3275c2785331d0ef8 Mon Sep 17 00:00:00 2001 From: overpathz Date: Mon, 8 Jul 2024 13:01:50 +0300 Subject: [PATCH 3/8] Update env sample --- .env-sample | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.env-sample b/.env-sample index 5816273..b847a98 100644 --- a/.env-sample +++ b/.env-sample @@ -1,7 +1,11 @@ -DB_USER= -DB_PASS= -RABBIT_USER= -RABBIT_PASS= -TELEGRAM_BOT_TOKEN= -SPRING_DATASOURCE_URL= -SPRING_JPA_HIBERNATE_DDL_AUTO= \ No newline at end of file +DB_NAME=community +DB_USER=backend +DB_PASS=superstrongpassword +RABBIT_HOST=rabbitmq +RABBIT_USER=community-rabbit +RABBIT_PASS=superstrongpassword +TELEGRAM_BOT_TOKEN= +SPRING_DATASOURCE_URL=jdbc:postgresql://db:5432/community +SPRING_JPA_HIBERNATE_DDL_AUTO=update +SERVER_URL=http://localhost:8080 +SPRING_APP_HEALTH_URL=http://localhost:8080/actuator/health From a077dd5fdaf204b2bb1548c97e31c26fb1ca0a8e Mon Sep 17 00:00:00 2001 From: Oleksandr K Date: Mon, 8 Jul 2024 13:16:36 +0300 Subject: [PATCH 4/8] Update README.md --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fac3d65..8547c84 100644 --- a/README.md +++ b/README.md @@ -40,19 +40,19 @@ This bot is designed to connect people based on their interests and preferences. 4. **Set Environment Variables** - Create a `.env` file in the root directory and add the following: + Create a `.env` file in the root directory by the .env-sample one - ```env - TELEGRAM_BOT_USERNAME=your_bot_username - TELEGRAM_BOT_TOKEN=your_bot_token - TELEGRAM_BOT_WEBHOOK_PATH=webhook_path - ``` +5. **Get your IPv4 address for prometheus.yml** + java -jar ip-resolver.jar -5. **Run the Application** +6. **Go to project root directory and run command** + ./run-project.sh + if no acccess -> chmod +x run-project.sh + try again -> ./run-project.sh + +7. **It should run all containers including monitoring and logging** +image - ```bash - mvn spring-boot:run - ``` ## Usage From 01a2d500bfe38002257266af059d649dfe265c4f Mon Sep 17 00:00:00 2001 From: Oleksandr K Date: Mon, 8 Jul 2024 13:18:01 +0300 Subject: [PATCH 5/8] Update README.md --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8547c84..6f32946 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,15 @@ This bot is designed to connect people based on their interests and preferences. Create a `.env` file in the root directory by the .env-sample one -5. **Get your IPv4 address for prometheus.yml** - java -jar ip-resolver.jar +5. **Get your IPv4 address for prometheus.yml**
+ ```java -jar ip-resolver.jar``` -6. **Go to project root directory and run command** - ./run-project.sh +6. **Go to project root directory and run command**
+ ``` + ./run-project.sh if no acccess -> chmod +x run-project.sh try again -> ./run-project.sh + ``` 7. **It should run all containers including monitoring and logging** image From f14b9922381e4aa393f96402597203781ac6ebbf Mon Sep 17 00:00:00 2001 From: overpathz Date: Mon, 8 Jul 2024 13:18:48 +0300 Subject: [PATCH 6/8] create util app to get ipv4 for prometheus --- ip-resolver.jar | Bin 0 -> 2107 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ip-resolver.jar diff --git a/ip-resolver.jar b/ip-resolver.jar new file mode 100644 index 0000000000000000000000000000000000000000..81d02e1650dd69f5074be09a6bc79d6c330f8c69 GIT binary patch literal 2107 zcmaKtdpy(oAIHC8&P;Au<`<6Zj!QP>Qc`5mm>F})$SpDV>#*85l^%tYj1?l69|}W7 zxkkc9(e@jeNy#lK)QBX0C;H9#{T^M;IsM+B$LI0*d>)_I>-~OzK7V|kc&w-x6aa(4 zKt;%mJMc|F0Wkn~*cFAqIG_!1C-m|B_6HtEHxvL#_hbMh4Ou2sLP^ zdnn#B;-(czOS9L$Ag^|KbpH2<4>Hl!ZN)=r$fk9Pb!~x5*tn6zS=n$3vtXeh>ht}N z`t6@%7lHuoci!M5j3L?n+oC2SQiODnW|7g2yQ_n5kL0b0h0^kKW(%PV+5;{~4%#2y z)MvP4cuqFdWX6nKGA}nec;`*;Jxf8sa=A*Yno5+%M6~f;R9CEh7*cMe1$szyhsn8k zVqu!tAeZym&7X zAB1Rf*BO?6qUlm8E1uaO?WLVL%%Ig5W{gM<=qmcGP!kiWyDrCEe?uFf$K|SYX4do2 zk%`ojc8;X{d4su=n8^y0C=T0i#7=39!ed_|nX{Ij%O>x-F_}e^y9lXYgV$b%{WEK< zRKTe=F5HPeib_K6skzf05eC+!1|ltvZ+}0q`v9+TTZ1C2IhhbwfW8E}rPCc6Aa#OC zerfCQ3U0Numjx<2Q0Wihox2Xda?dcqNDI4Pw`9rmnr7Lh;A6#8oDwhb*-%q#LrP`D zM2Km-!tjoun^bkqM7#>jbtb($zh1z+K`$Eb_|@uRe%0Ri<-LKeZ}yC91pNHW{^=}b zHeJ7rh?-Vn!WGQ|`AK+R9TYt(Iu%MTJlxB zT2NIuVXda)G5;Tadd)DEY-sy2C&k9yr}hbzq%50g|Lxjb?_8`8>22h=v2 zI&~6SpBk5e1Pi#xC(Lt@WxBHgZ0M0n&D+-}r@I^mTtA-<4aYcEo_8ic#;~1SC=p|W zT#GVxmDJP5*HoHQZl=!=_uYXO)C5CQ!#PJaHTB;+5|x?StFEk|x%ySF;@CENdBd?j zHt8Vtryd4_oGq<=gzTSUalXJG$43+E6+e-n`hGP!csjVXfe(6L}GaLZ+@Cr zuySD#@>EXue*K`VmCpnb@6S73rN*NrEqt<`kTJO*`9RId4+N-qKv7goTNGMQ~wa}gI zdej#x5>ynze^Gv50<}F7BPQTKxAC$PJzY}%0H4_>;&XbMX}wQeC%YxBlQ(*Bl9-fl ziaCC0g0K~y_o-UGZTf?iaOU-1YoC0M8?4i+89kKA>K@rCv*Av8$A+A(XeGq#E4VLc ziphlW;XYda8;Zl3t0=DYB7CTKxurJ4APuSPx8#%zyM~sj>GPTEuB1RQ%YHi^e(L)D z>6EFer!>|XC$PvV>#<&OirSL4tkj{aPZ#+h;vN`srfF^bkPl|AFvWR+en{Xj-1Djp} z0YH_IDtPP`5h(EWr~ei_o2dIfzQE>B;M>ds0e%druURe%mtV5{Kil`E>x8!dBJ9V7 vHX-&k_`>C;<6ESCf7NfFVH0X!W6PF5d;b=3cq~YK(*zQZ147H}AG`kmTvJlD literal 0 HcmV?d00001 From 2f2cec8bdeb6d806ba2f87e666ddd5bd738dbe8a Mon Sep 17 00:00:00 2001 From: overpathz Date: Tue, 9 Jul 2024 11:48:38 +0300 Subject: [PATCH 7/8] Add flyway to set up migration schemes --- pom.xml | 5 ++ src/main/resources/application-dev.yml | 69 +++++++++++++++++++ src/main/resources/application.yml | 13 +++- .../db/migration/V1__Initial_schema.sql | 53 ++++++++++++++ 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 src/main/resources/application-dev.yml create mode 100644 src/main/resources/db/migration/V1__Initial_schema.sql diff --git a/pom.xml b/pom.xml index 6a2b73a..1387b7b 100644 --- a/pom.xml +++ b/pom.xml @@ -135,6 +135,11 @@ logstash-logback-encoder 5.3 + + org.flywaydb + flyway-core + + diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml new file mode 100644 index 0000000..c15f659 --- /dev/null +++ b/src/main/resources/application-dev.yml @@ -0,0 +1,69 @@ +telegram: + bot: + username: communitystagebot + token: 7483072284:AAEKReYVUKfbCu8i1kQHZW7IuV_W-ejMkCY + +community: + likes-queue: like-events + payment-queue: payment-events + +management: + endpoints: + web: + exposure: + include: info,health,metrics,prometheus + endpoint: + health: + show-details: always + metrics: + enabled: true + prometheus: + enabled: true + +spring: + app: + rabbit: + like-event-queue-name: like-events + payment-event-queue-name: payment-events + jpa: + hibernate: + ddl-auto: create-drop + show-sql: true + flyway: + enabled: false + data: + mongodb: + host: localhost + port: 27017 + database: mydatabase + username: admin + password: admin123 + redis: + url: redis://redis:6379 + datasource: + url: jdbc:postgresql://db:5432/${DB_NAME} + username: ${DB_USER} + password: ${DB_PASS} + rabbitmq: + username: ${RABBIT_USER} + password: ${RABBIT_USER} + +amazon: + s3: + access-key: . + secret-key: . + bucket-name: . + +logging: + level: + root: info + file: + name: logs/community.log +server: + port: 8080 + +paypal: + client: + id: . + secret: . + mode: sandbox \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b335ae0..33ddfdb 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -27,8 +27,17 @@ spring: payment-event-queue-name: payment-events jpa: hibernate: - ddl-auto: create-drop - show-sql: true + ddl-auto: none + properties: + hibernate: + show_sql: true + format_sql: true + flyway: + locations: classpath:db/migration + enabled: true + baseline-version: 1 + baseline-description: Initial version + clean-disabled: true data: mongodb: host: localhost diff --git a/src/main/resources/db/migration/V1__Initial_schema.sql b/src/main/resources/db/migration/V1__Initial_schema.sql new file mode 100644 index 0000000..edd0537 --- /dev/null +++ b/src/main/resources/db/migration/V1__Initial_schema.sql @@ -0,0 +1,53 @@ +-- V1__Initial_schema.sql + +-- Create the table for Gender enum +CREATE TYPE gender_enum AS ENUM ('MAN', 'WOMAN', 'OTHER', 'ANYONE'); + +-- Create the table for UserFlowState enum +CREATE TYPE userflowstate_enum AS ENUM ( + 'START', 'NAME', 'CITY', 'AGE', 'GENDER', 'LOOKING', + 'TOPIC', 'DESCRIPTION', 'APPROVE', 'MATCH', 'PHOTO', 'SETTINGS' +); + +-- Create the table for Subscription enum +CREATE TYPE subscription_enum AS ENUM ('VIP', 'PREMIUM', 'NONE'); + +-- Create the users table +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + telegram_user_id BIGINT UNIQUE NOT NULL, + name VARCHAR(255), + username VARCHAR(255), + city VARCHAR(255), + age BIGINT, + gender gender_enum, + looking_for gender_enum DEFAULT 'ANYONE', + description TEXT, + photo_id VARCHAR(255), + user_flow_state userflowstate_enum DEFAULT 'START', + subscription subscription_enum DEFAULT 'NONE' +); + +-- Create the topics table +CREATE TABLE topics ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL +); + +-- Create the user_topic join table for the many-to-many relationship between users and topics +CREATE TABLE user_topic ( + user_id BIGINT NOT NULL, + topic_id BIGINT NOT NULL, + PRIMARY KEY (user_id, topic_id), + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (topic_id) REFERENCES topics(id) ON DELETE CASCADE +); + +-- Create the user_likes table +CREATE TABLE user_likes ( + id SERIAL PRIMARY KEY, + user_id BIGINT NOT NULL, + liked_user_id BIGINT NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE, + FOREIGN KEY (liked_user_id) REFERENCES users(id) ON DELETE CASCADE +); From f59497f6e00cfb7f08f6e0e68739026487a0f6dd Mon Sep 17 00:00:00 2001 From: overpathz Date: Thu, 11 Jul 2024 11:28:17 +0300 Subject: [PATCH 8/8] Use .env for S3 properties --- src/main/resources/application.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 33ddfdb..deaf308 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -57,9 +57,9 @@ spring: amazon: s3: - access-key: . - secret-key: . - bucket-name: . + access-key: ${S3_ACCESS_KEY} + secret-key: ${S3_SECRET_KEY} + bucket-name: ${S3_BUCKET_NAME} logging: level: