From eb0022837eb995171a0bf000e0e00f020226ea00 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 00:13:13 +0100 Subject: [PATCH] chore: Use celery & Redis --- .env | 4 ++++ .gitignore | 6 ++++-- backend/Dockerfile | 4 ---- backend/notifications/logic.py | 19 ++++++++++++++++++- backend/notifications/signals.py | 4 ++-- backend/requirements.txt | 3 ++- backend/ypovoli/__init__.py | 5 +++++ backend/ypovoli/celery.py | 22 ++++++++++++++++++++++ backend/ypovoli/settings.py | 21 +++++++++++++++++++++ data/nginx/nginx.conf | 8 ++++---- development.sh | 19 +++++++++++++++---- development.yml | 32 ++++++++++++++++++++++++++++++-- frontend/Dockerfile | 2 +- 13 files changed, 128 insertions(+), 21 deletions(-) create mode 100644 backend/ypovoli/celery.py diff --git a/.env b/.env index 118bb629..0d3f97be 100644 --- a/.env +++ b/.env @@ -7,3 +7,7 @@ DATADIR="./data" BACKEND_DIR="./backend" FRONTEND_DIR="./frontend" + +REDIS_IP="192.168.90.10" +REDIS_PORT=6379 +REDIS_PASSWORD="oqOsNX1PXGOX5soJtKkw" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3984195e..5dafea5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .tool-versions -data/postgres/* -data/nginx/ssl/* \ No newline at end of file +data/* + +!data/nginx/nginx.conf +!data/nginx/ssl/.gitkeep diff --git a/backend/Dockerfile b/backend/Dockerfile index bce03b5a..0365d5f9 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -11,7 +11,3 @@ COPY requirements.txt /code/ RUN pip install -r requirements.txt COPY . /code/ - -RUN ./setup.sh - -CMD ["python", "manage.py", "runsslserver", "192.168.90.2:8080"] \ No newline at end of file diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py index 99104eb9..f7140134 100644 --- a/backend/notifications/logic.py +++ b/backend/notifications/logic.py @@ -2,7 +2,9 @@ from smtplib import SMTPException from typing import Dict, List +from celery import shared_task from django.core import mail +from django.core.cache import cache from django.utils.translation import gettext as _ from notifications.models import Notification from ypovoli.settings import EMAIL_CUSTOM @@ -17,6 +19,17 @@ def get_message_dict(notification: Notification) -> Dict[str, str]: } +# Call the function after 60 seconds and no more than once in that period +def schedule_send_mails(): + print("Hiii") + if not cache.get("notifications_send_mails"): + print("Not in cache yet") + cache.set("notifications_send_mails", True) + _send_mails.apply_async(countdown=60) + else: + print("Already in cache") + + # Try to send one email and set the result def _send_mail(mail: mail.EmailMessage, result: List[bool]): try: @@ -27,7 +40,11 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]): # Send all unsent emails -def send_mails(): +@shared_task +def _send_mails(): + print("Sending") + cache.set("notifications_send_mails", False) + notifications = Notification.objects.filter(is_sent=False) # No notifications to send diff --git a/backend/notifications/signals.py b/backend/notifications/signals.py index 6d7e993f..7a817e5a 100644 --- a/backend/notifications/signals.py +++ b/backend/notifications/signals.py @@ -6,7 +6,7 @@ from authentication.models import User from django.dispatch import Signal, receiver from django.urls import reverse -from notifications.logic import send_mails +from notifications.logic import schedule_send_mails from notifications.serializers import NotificationSerializer notification_create = Signal() @@ -33,7 +33,7 @@ def notification_creation( serializer.save() - send_mails() + schedule_send_mails() return True diff --git a/backend/requirements.txt b/backend/requirements.txt index 610d422a..d1dab654 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -7,4 +7,5 @@ requests==2.31.0 cas-client==1.0.0 psycopg2-binary==2.9.9 djangorestframework-simplejwt==5.3.1 -celery[redis]==5.3.6 \ No newline at end of file +celery[redis]==5.3.6 +django-redis==5.4.0 \ No newline at end of file diff --git a/backend/ypovoli/__init__.py b/backend/ypovoli/__init__.py index e69de29b..5568b6d7 100644 --- a/backend/ypovoli/__init__.py +++ b/backend/ypovoli/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/backend/ypovoli/celery.py b/backend/ypovoli/celery.py new file mode 100644 index 00000000..4195e638 --- /dev/null +++ b/backend/ypovoli/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ypovoli.settings") + +app = Celery("ypovoli") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f"Request: {self.request!r}") diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 1e496534..22b29a82 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os from datetime import timedelta from pathlib import Path @@ -137,3 +138,23 @@ "subject": "[Ypovoli] New Notification", "timeout": 2, } + +REDIS_CUSTOM = { + "host": os.environ.get("REDIS_IP", "localhost"), + "port": os.environ.get("REDIS_PORT", 6379), + "password": os.environ.get("REDIS_PASSWORD", ""), + "db_django": 0, + "db_celery": 1, +} + +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_django']}", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + } +} + +CELERY_BROKER_URL = f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}" diff --git a/data/nginx/nginx.conf b/data/nginx/nginx.conf index a3a6b6d0..c46ebf37 100644 --- a/data/nginx/nginx.conf +++ b/data/nginx/nginx.conf @@ -24,8 +24,8 @@ http { listen 443 ssl; listen [::]:443 ssl; - ssl_certificate certificate.crt; - ssl_certificate_key private.key; + ssl_certificate ssl/certificate.crt; + ssl_certificate_key ssl/private.key; location / { proxy_pass https://frontend; @@ -39,8 +39,8 @@ http { listen 8080 ssl; listen [::]:8080 ssl; - ssl_certificate certificate.crt; - ssl_certificate_key private.key; + ssl_certificate ssl/certificate.crt; + ssl_certificate_key ssl/private.key; location / { proxy_pass https://backend; diff --git a/development.sh b/development.sh index ed637eb4..24697323 100755 --- a/development.sh +++ b/development.sh @@ -1,11 +1,12 @@ echo "Checking for existing SSL certificates..." if [ ! -f "data/nginx/ssl/private.key" ] || [ ! -f "data/nginx/ssl/certificate.crt" ]; then -echo "Generating SSL certificates..." + echo "Generating SSL certificates..." sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout data/nginx/ssl/private.key \ -out data/nginx/ssl/certificate.crt \ - -subj "/C=BE/ST=/L=/O=/OU=/CN=" + -subj "/C=BE/ST=/L=/O=/OU=/CN=" > /dev/null + echo "SSL certificates generated." else echo "SSL certificates already exist, skipping generation." fi @@ -13,5 +14,15 @@ fi echo "Starting services..." docker-compose -f development.yml up -d -echo "Following logs..." -docker-compose -f development.yml logs --follow --tail 50 backend \ No newline at end of file +echo "-------------------------------------" +echo "Following backend logs..." +echo "Press CTRL + C to stop all containers" +echo "-------------------------------------" + +docker-compose -f development.yml logs --follow --tail 50 backend + +echo "Cleaning up..." + +docker-compose -f development.yml down + +echo "Done." diff --git a/development.yml b/development.yml index d72ee963..58a532e3 100644 --- a/development.yml +++ b/development.yml @@ -39,7 +39,7 @@ services: - 8080:8080 volumes: - $DATADIR/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - $DATADIR/nginx/ssl:/etc/nginx/ + - $DATADIR/nginx/ssl:/etc/nginx/ssl:ro depends_on: - backend - frontend @@ -50,11 +50,25 @@ services: build: context: $BACKEND_DIR dockerfile: Dockerfile + command: bash -c "./setup.sh && python manage.py runsslserver 192.168.90.2:8080" expose: - 8000 volumes: - $BACKEND_DIR:/code + celery: + <<: *common-keys-selab + container_name: celery + build: + context: $BACKEND_DIR + dockerfile: Dockerfile + command: celery -A ypovoli worker -l INFO + volumes: + - $BACKEND_DIR:/code + depends_on: + - backend + - redis + frontend: <<: *common-keys-selab container_name: frontend @@ -64,4 +78,18 @@ services: expose: - 3000 depends_on: - - backend \ No newline at end of file + - backend + + redis: + <<: *common-keys-selab + container_name: redis + image: redis:latest + networks: + selab_network: + ipv4_address: $REDIS_IP + expose: + - $REDIS_PORT + entrypoint: redis-server --appendonly yes --requirepass $REDIS_PASSWORD --maxmemory 512mb --maxmemory-policy allkeys-lru + volumes: + - $DATADIR/redis:/data + \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5254aa37..d3a568de 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,7 +5,7 @@ RUN npm install COPY ./ . RUN npm run build -FROM nginx as production-stage +FROM nginx as development-stage EXPOSE 3000 RUN mkdir /app COPY nginx.conf /etc/nginx/conf.d/default.conf