From 5f1bb75264cbad7c86326a69d95862b04a910932 Mon Sep 17 00:00:00 2001
From: VladYoSlav <vaifaer@gmail.com>
Date: Wed, 23 Oct 2024 17:59:02 +0300
Subject: [PATCH] feat(docker compose): celery & flower

---
 .env.example                  | 12 +++++++--
 .gitignore                    |  4 +++
 Dockerfile                    | 15 ++++++++++-
 Makefile                      | 13 +++++++---
 dev-docker-compose.yaml       | 47 ++++++++++++++++++++++++++++++++++
 poetry.lock                   | 48 ++++++++++++++++++++++++++++++++++-
 pyproject.toml                |  1 +
 scripts/celery.sh             |  6 +++++
 scripts/flower.sh             | 20 +++++++++++++++
 scripts/start.sh              |  8 ++++++
 {uploads => volumes}/.gitkeep |  0
 11 files changed, 167 insertions(+), 7 deletions(-)
 create mode 100644 scripts/celery.sh
 create mode 100644 scripts/flower.sh
 create mode 100644 scripts/start.sh
 rename {uploads => volumes}/.gitkeep (100%)

diff --git a/.env.example b/.env.example
index e6197fae..372fc79d 100644
--- a/.env.example
+++ b/.env.example
@@ -1,11 +1,19 @@
 POSTGRES_DIALECT_DRIVER=postgresql+psycopg
 POSTGRES_USER=admin
 POSTGRES_PASSWORD=admin
-POSTGRES_HOST=localhost
+POSTGRES_HOST=postgres
 POSTGRES_DB=desbordante
 POSTGRES_PORT=5432
 
 RABBITMQ_DEFAULT_USER=guest
 RABBITMQ_DEFAULT_PASSWORD=guest
-RABBITMQ_HOST=localhost
+RABBITMQ_HOST=rabbitmq
 RABBITMQ_PORT=5672
+RABBITMQ_HTTP_PORT=15672
+
+BACKEND_PORT=8000
+UPLOADED_FILES_DIR_PATH=/volumes/uploads
+
+FLOWER_USER=admin
+FLOWER_PASSWORD=admin
+FLOWER_PORT=5555
diff --git a/.gitignore b/.gitignore
index 62a78fb3..ac5d05c1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,3 +113,7 @@ coverage-report-html/
 
 .mypy_cache/
 .ruff_cache/
+
+# Volumes
+volumes/*
+!volumes/.gitkeep
diff --git a/Dockerfile b/Dockerfile
index 97ef2f30..1d7ec574 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -59,6 +59,18 @@ ENV PATH="$VIRTUAL_ENVIRONMENT_PATH/bin:$PATH"
 RUN groupadd -g 1001 python_application && \
     useradd -r -u 1001 -g python_application python_application
 
+COPY ./scripts/start.sh /start
+RUN sed -i 's/\r$//g' /start
+RUN chmod +x /start
+
+COPY ./scripts/celery.sh /celery
+RUN sed -i 's/\r$//g' /celery
+RUN chmod +x /celery
+
+COPY ./scripts/flower.sh /flower
+RUN sed -i 's/\r$//g' /flower
+RUN chmod +x /flower
+
 # Set the WORKDIR to the application root.
 # https://www.uvicorn.org/settings/#development
 # https://docs.docker.com/engine/reference/builder/#workdir
@@ -78,7 +90,8 @@ EXPOSE ${APPLICATION_SERVER_PORT}
 USER 1001
 
 # Run the uvicorn application server.
-CMD exec uvicorn --workers 1 --host 0.0.0.0 --port $APPLICATION_SERVER_PORT internal:app
+CMD /start
+
 
 FROM server-setup-build-stage as install-dependencies-build-stage
 # install [tool.poetry.dependencies]
diff --git a/Makefile b/Makefile
index 83d7a6b8..b5e02b2b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: env install-deps up open-db pg-revision pg-migrate pg-downgrade celery-worker app init lint test check-types
+.PHONY: env volumes install-deps up open-db pg-revision pg-migrate pg-downgrade celery-worker app init lint test check-types
 
 ifeq ($(shell test -e '.env' && echo -n yes), yes)
 	include .env
@@ -12,6 +12,13 @@ env:
 	@echo >> .env
 	@echo "SECRET_KEY=$$(openssl rand -hex 32)" >> .env
 
+## Create folders for volumes
+volumes:
+	@for volume in postgres rabbitmq uploads; do \
+    	mkdir -p ./volumes/$$volume; \
+    	chmod 777 ./volumes/$$volume; \
+    done
+
 ## Install dependencies
 install-deps:
 	poetry install
@@ -40,7 +47,7 @@ pg-downgrade:
 
 ## Run celery worker in watch mode
 celery-worker:
-	watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A internal.infrastructure.background_task.celery worker --loglevel=info --concurrency=1
+	poetry run watchmedo auto-restart --directory=./ --pattern='*.py' --recursive -- celery -A internal.infrastructure.background_task.celery worker --loglevel=info --concurrency=1
 
 ## Run application server in watch mode
 app:
@@ -48,7 +55,7 @@ app:
 
 ## Initiate repository
 init:
-	make env install-deps
+	make env volumes install-deps
 
 ## Run all formatters and linters in project
 lint:
diff --git a/dev-docker-compose.yaml b/dev-docker-compose.yaml
index f68cbe63..817bd572 100644
--- a/dev-docker-compose.yaml
+++ b/dev-docker-compose.yaml
@@ -9,6 +9,8 @@ services:
       - .env
     ports:
       - '${POSTGRES_PORT}:5432'
+    volumes:
+      - ./volumes/postgres:/var/lib/postgresql/data
 
   rabbitmq:
     container_name: desbordante-rabbitmq
@@ -17,3 +19,48 @@ services:
       - .env
     ports:
       - "${RABBITMQ_PORT}:5672"
+      - "${RABBITMQ_HTTP_PORT}:15672"
+    volumes:
+      - ./volumes/rabbitmq:/var/lib/rabbitmq
+
+#  backend:
+#    build:
+#      context: .
+#    volumes:
+#      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+#    env_file:
+#      - .env
+#    depends_on:
+#      - postgres
+#      - rabbitmq
+#    restart: always
+#    ports:
+#      - "${BACKEND_PORT}:8000"
+
+  celery:
+    build:
+      context: .
+    command: /celery
+    volumes:
+      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+    env_file:
+      - .env
+    depends_on:
+      - rabbitmq
+    restart: always
+
+
+  flower:
+    build:
+      context: .
+    command: /flower
+    volumes:
+      - ./volumes/uploads:${UPLOADED_FILES_DIR_PATH}
+    env_file:
+      - .env
+    depends_on:
+      - rabbitmq
+      - celery
+    restart: always
+    ports:
+      - "${FLOWER_PORT}:5555"
diff --git a/poetry.lock b/poetry.lock
index 082e6e96..331091f0 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -826,6 +826,24 @@ docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.
 testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"]
 typing = ["typing-extensions (>=4.12.2)"]
 
+[[package]]
+name = "flower"
+version = "2.0.1"
+description = "Celery Flower"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"},
+    {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"},
+]
+
+[package.dependencies]
+celery = ">=5.0.5"
+humanize = "*"
+prometheus-client = ">=0.8.0"
+pytz = "*"
+tornado = ">=5.0.0,<7.0.0"
+
 [[package]]
 name = "greenlet"
 version = "3.1.1"
@@ -1024,6 +1042,20 @@ http2 = ["h2 (>=3,<5)"]
 socks = ["socksio (==1.*)"]
 zstd = ["zstandard (>=0.18.0)"]
 
+[[package]]
+name = "humanize"
+version = "4.11.0"
+description = "Python humanize utilities"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"},
+    {file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"},
+]
+
+[package.extras]
+tests = ["freezegun", "pytest", "pytest-cov"]
+
 [[package]]
 name = "identify"
 version = "2.6.1"
@@ -1729,6 +1761,20 @@ nodeenv = ">=0.11.1"
 pyyaml = ">=5.1"
 virtualenv = ">=20.10.0"
 
+[[package]]
+name = "prometheus-client"
+version = "0.21.0"
+description = "Python client for the Prometheus monitoring system."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"},
+    {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"},
+]
+
+[package.extras]
+twisted = ["twisted"]
+
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.48"
@@ -3397,4 +3443,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "571f60a81d586f2c69d1b1080d22ca5493d9740c3e8215d45d105b70e831af1b"
+content-hash = "d7fe0a2c084441e383b2e7c948fba8188a316f94fefa1376fcda82e575c81ccb"
diff --git a/pyproject.toml b/pyproject.toml
index b2101c9c..7a4e4b92 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ pandas-stubs = "^2.2.0.240218"
 python-multipart = "^0.0.9"
 aiofiles = "^23.2.1"
 cfgv = "^3.4.0"
+flower = "^2.0.1"
 
 
 [tool.poetry.group.dev.dependencies]
diff --git a/scripts/celery.sh b/scripts/celery.sh
new file mode 100644
index 00000000..61327900
--- /dev/null
+++ b/scripts/celery.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+celery -A internal.infrastructure.background_task.celery worker --loglevel=info
\ No newline at end of file
diff --git a/scripts/flower.sh b/scripts/flower.sh
new file mode 100644
index 00000000..81c97ccd
--- /dev/null
+++ b/scripts/flower.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+worker_ready() {
+    celery -A internal.infrastructure.background_task.celery inspect ping
+}
+
+until worker_ready; do
+  >&2 echo 'Celery workers not available'
+  sleep 1
+done
+>&2 echo 'Celery workers is available'
+
+celery \
+    --app=internal.infrastructure.background_task.celery \
+    flower \
+    --port=5555
+    --basic-auth=${FLOWER_USER}:${FLOWER_PASSWORD}
\ No newline at end of file
diff --git a/scripts/start.sh b/scripts/start.sh
new file mode 100644
index 00000000..b497ed44
--- /dev/null
+++ b/scripts/start.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+alembic -c internal/infrastructure/data_storage/relational/postgres/migrations/alembic.ini upgrade head
+uvicorn --workers 1 --host 0.0.0.0 --port $APPLICATION_SERVER_PORT internal:app
\ No newline at end of file
diff --git a/uploads/.gitkeep b/volumes/.gitkeep
similarity index 100%
rename from uploads/.gitkeep
rename to volumes/.gitkeep