chore: Use celery & Redis

.env

@@ -7,3 +7,7 @@ DATADIR="./data"
 BACKEND_DIR="./backend"
 
 FRONTEND_DIR="./frontend"
+
+REDIS_IP="192.168.90.10"
+REDIS_PORT=6379
+REDIS_PASSWORD="oqOsNX1PXGOX5soJtKkw"

.gitignore

@@ -1,3 +1,5 @@
 .tool-versions
-data/postgres/*
-data/nginx/ssl/*
+data/*
+
+!data/nginx/nginx.conf
+!data/nginx/ssl/.gitkeep

backend/Dockerfile

@@ -11,7 +11,3 @@ COPY requirements.txt /code/
 RUN pip install -r requirements.txt
 
 COPY . /code/
-
-RUN ./setup.sh
-
-CMD ["python", "manage.py", "runsslserver", "192.168.90.2:8080"]

backend/notifications/logic.py

@@ -2,7 +2,9 @@
 from smtplib import SMTPException
 from typing import Dict, List
 
+from celery import shared_task
 from django.core import mail
+from django.core.cache import cache
 from django.utils.translation import gettext as _
 from notifications.models import Notification
 from ypovoli.settings import EMAIL_CUSTOM
@@ -17,6 +19,17 @@ def get_message_dict(notification: Notification) -> Dict[str, str]:
     }
 
 
+# Call the function after 60 seconds and no more than once in that period
+def schedule_send_mails():
+    print("Hiii")
+    if not cache.get("notifications_send_mails"):
+        print("Not in cache yet")
+        cache.set("notifications_send_mails", True)
+        _send_mails.apply_async(countdown=60)
+    else:
+        print("Already in cache")
+
+
 # Try to send one email and set the result
 def _send_mail(mail: mail.EmailMessage, result: List[bool]):
     try:
@@ -27,7 +40,11 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]):
 
 
 # Send all unsent emails
-def send_mails():
+@shared_task
+def _send_mails():
+    print("Sending")
+    cache.set("notifications_send_mails", False)
+
     notifications = Notification.objects.filter(is_sent=False)
 
     # No notifications to send

backend/notifications/signals.py

@@ -6,7 +6,7 @@
 from authentication.models import User
 from django.dispatch import Signal, receiver
 from django.urls import reverse
-from notifications.logic import send_mails
+from notifications.logic import schedule_send_mails
 from notifications.serializers import NotificationSerializer
 
 notification_create = Signal()
@@ -33,7 +33,7 @@ def notification_creation(
 
     serializer.save()
 
-    send_mails()
+    schedule_send_mails()
 
     return True
 

backend/requirements.txt

@@ -7,4 +7,5 @@ requests==2.31.0
 cas-client==1.0.0
 psycopg2-binary==2.9.9
 djangorestframework-simplejwt==5.3.1
-celery[redis]==5.3.6
+celery[redis]==5.3.6
+django-redis==5.4.0

backend/ypovoli/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

backend/ypovoli/celery.py

@@ -0,0 +1,22 @@
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ypovoli.settings")
+
+app = Celery("ypovoli")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+    print(f"Request: {self.request!r}")

backend/ypovoli/settings.py

@@ -10,6 +10,7 @@
 https://docs.djangoproject.com/en/5.0/ref/settings/
 """
 
+import os
 from datetime import timedelta
 from pathlib import Path
 
@@ -137,3 +138,23 @@
     "subject": "[Ypovoli] New Notification",
     "timeout": 2,
 }
+
+REDIS_CUSTOM = {
+    "host": os.environ.get("REDIS_IP", "localhost"),
+    "port": os.environ.get("REDIS_PORT", 6379),
+    "password": os.environ.get("REDIS_PASSWORD", ""),
+    "db_django": 0,
+    "db_celery": 1,
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_django']}",
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+        },
+    }
+}
+
+CELERY_BROKER_URL = f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}"

data/nginx/nginx.conf

@@ -24,8 +24,8 @@ http {
         listen 443 ssl;
         listen [::]:443 ssl;
 
-        ssl_certificate certificate.crt;
-        ssl_certificate_key private.key;
+        ssl_certificate ssl/certificate.crt;
+        ssl_certificate_key ssl/private.key;
 
         location / {
             proxy_pass https://frontend;
@@ -39,8 +39,8 @@ http {
         listen 8080 ssl;
         listen [::]:8080 ssl;
 
-        ssl_certificate certificate.crt;
-        ssl_certificate_key private.key;
+        ssl_certificate ssl/certificate.crt;
+        ssl_certificate_key ssl/private.key;
 
         location / {
             proxy_pass https://backend;

development.sh

@@ -1,17 +1,28 @@
 echo "Checking for existing SSL certificates..."
 
 if [ ! -f "data/nginx/ssl/private.key" ] || [ ! -f "data/nginx/ssl/certificate.crt" ]; then
-echo "Generating SSL certificates..."
+    echo "Generating SSL certificates..."
     sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
     -keyout data/nginx/ssl/private.key \
     -out data/nginx/ssl/certificate.crt \
-    -subj "/C=BE/ST=/L=/O=/OU=/CN="
+    -subj "/C=BE/ST=/L=/O=/OU=/CN=" > /dev/null
+    echo "SSL certificates generated."
 else
     echo "SSL certificates already exist, skipping generation."
 fi
 
 echo "Starting services..."
 docker-compose -f development.yml up -d
 
-echo "Following logs..."
-docker-compose -f development.yml logs --follow --tail 50 backend
+echo "-------------------------------------"
+echo "Following backend logs..."
+echo "Press CTRL + C to stop all containers"
+echo "-------------------------------------"
+
+docker-compose -f development.yml logs --follow --tail 50 backend
+
+echo "Cleaning up..."
+
+docker-compose -f development.yml down
+
+echo "Done."

development.yml

@@ -39,7 +39,7 @@ services:
       - 8080:8080
     volumes:
       - $DATADIR/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
-      - $DATADIR/nginx/ssl:/etc/nginx/
+      - $DATADIR/nginx/ssl:/etc/nginx/ssl:ro
     depends_on:
       - backend
       - frontend
@@ -50,11 +50,25 @@ services:
     build:
       context: $BACKEND_DIR
       dockerfile: Dockerfile
+    command: bash -c "./setup.sh && python manage.py runsslserver 192.168.90.2:8080"
     expose:
       - 8000
     volumes:
       - $BACKEND_DIR:/code
 
+  celery:
+    <<: *common-keys-selab
+    container_name: celery
+    build:
+      context: $BACKEND_DIR
+      dockerfile: Dockerfile
+    command: celery -A ypovoli worker -l INFO
+    volumes:
+      - $BACKEND_DIR:/code
+    depends_on:
+      - backend
+      - redis
+
   frontend:
     <<: *common-keys-selab
     container_name: frontend
@@ -64,4 +78,18 @@ services:
     expose:
       - 3000
     depends_on:
-      - backend
+      - backend
+
+  redis:
+    <<: *common-keys-selab
+    container_name: redis
+    image: redis:latest
+    networks:
+      selab_network:
+        ipv4_address: $REDIS_IP
+    expose:
+      - $REDIS_PORT
+    entrypoint: redis-server --appendonly yes --requirepass $REDIS_PASSWORD --maxmemory 512mb --maxmemory-policy allkeys-lru
+    volumes:
+      - $DATADIR/redis:/data
+

frontend/Dockerfile

@@ -5,7 +5,7 @@ RUN npm install
 COPY ./ .
 RUN npm run build
 
-FROM nginx as production-stage
+FROM nginx as development-stage
 EXPOSE 3000
 RUN mkdir /app
 COPY nginx.conf /etc/nginx/conf.d/default.conf