From ac5e55a185bb0a36966a270001e275b1462bf03a Mon Sep 17 00:00:00 2001 From: Topvennie Date: Sun, 10 Mar 2024 17:23:23 +0100 Subject: [PATCH 01/13] feat(notifications): send mail function #7 --- .../locale/en/LC_MESSAGES/django.po | 3 + .../locale/nl/LC_MESSAGES/django.po | 3 + backend/notifications/logic.py | 68 +++++++++++++++++++ backend/notifications/models.py | 5 ++ backend/notifications/serializers.py | 7 +- backend/notifications/signals.py | 9 ++- backend/notifications/urls.py | 6 +- backend/notifications/views.py | 26 +++++++ backend/requirements.txt | 3 +- backend/ypovoli/settings.py | 15 ++-- 10 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 backend/notifications/logic.py diff --git a/backend/notifications/locale/en/LC_MESSAGES/django.po b/backend/notifications/locale/en/LC_MESSAGES/django.po index 5db896e0..465520e9 100644 --- a/backend/notifications/locale/en/LC_MESSAGES/django.po +++ b/backend/notifications/locale/en/LC_MESSAGES/django.po @@ -17,6 +17,9 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +# Email Template +msgid "Email %(name)s %(title)s %(description)s" +msgstr "Dear %(name)s\nYou have a new notification.\n%(title)s\n%(description)s\n\n- Ypovoli" # Score Added msgid "Title: Score added" msgstr "Score Added" diff --git a/backend/notifications/locale/nl/LC_MESSAGES/django.po b/backend/notifications/locale/nl/LC_MESSAGES/django.po index 4d9cb70f..5a854108 100644 --- a/backend/notifications/locale/nl/LC_MESSAGES/django.po +++ b/backend/notifications/locale/nl/LC_MESSAGES/django.po @@ -17,6 +17,9 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" +# Email Template +msgid "Email %(name)s %(title)s %(description)s" +msgstr "Beste %(name)s\nU heeft een nieuwe notificatie.\n%(title)s\n%(description)s\n\n- Ypovoli" # Score Added msgid "Title: Score added" msgstr "Score toegevoegd" diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py new file mode 100644 index 00000000..99104eb9 --- /dev/null +++ b/backend/notifications/logic.py @@ -0,0 +1,68 @@ +import threading +from smtplib import SMTPException +from typing import Dict, List + +from django.core import mail +from django.utils.translation import gettext as _ +from notifications.models import Notification +from ypovoli.settings import EMAIL_CUSTOM + + +# Returns a dictionary with the title and description of the notification +def get_message_dict(notification: Notification) -> Dict[str, str]: + return { + "title": _(notification.template_id.title_key), + "description": _(notification.template_id.description_key) + % notification.arguments, + } + + +# Try to send one email and set the result +def _send_mail(mail: mail.EmailMessage, result: List[bool]): + try: + mail.send(fail_silently=False) + result[0] = True + except SMTPException: + result[0] = False + + +# Send all unsent emails +def send_mails(): + notifications = Notification.objects.filter(is_sent=False) + + # No notifications to send + if notifications.count() == 0: + return + + # Connection with the mail server + connection = mail.get_connection() + + for notification in notifications: + message = get_message_dict(notification) + content = _("Email %(name)s %(title)s %(description)s") % { + "name": notification.user.username, + "title": message["title"], + "description": message["description"], + } + + # Construct the email + email = mail.EmailMessage( + subject=EMAIL_CUSTOM["subject"], + body=content, + from_email=EMAIL_CUSTOM["from"], + to=[notification.user.email], + connection=connection, + ) + + # Send the email with a timeout + result: List[bool] = [False] + thread = threading.Thread(target=_send_mail, args=(email, result)) + thread.start() + thread.join(timeout=EMAIL_CUSTOM["timeout"]) + + # If the email was not sent, continue + if thread.is_alive() or not result[0]: + continue + + # Mark the notification as sent + notification.sent() diff --git a/backend/notifications/models.py b/backend/notifications/models.py index d2827892..c9c2fbde 100644 --- a/backend/notifications/models.py +++ b/backend/notifications/models.py @@ -22,3 +22,8 @@ class Notification(models.Model): is_sent = models.BooleanField( default=False ) # Whether the notification has been sent (email) + + # Mark the notification as read + def sent(self): + self.is_sent = True + self.save() diff --git a/backend/notifications/serializers.py b/backend/notifications/serializers.py index 4a24cd59..d4c488ba 100644 --- a/backend/notifications/serializers.py +++ b/backend/notifications/serializers.py @@ -2,7 +2,7 @@ from typing import Dict, List from authentication.models import User -from django.utils.translation import gettext as _ +from notifications.logic import get_message_dict from notifications.models import Notification, NotificationTemplate from rest_framework import serializers @@ -57,10 +57,7 @@ def validate(self, data: Dict[str, str]) -> Dict[str, str]: # Get the message from the template and arguments def get_message(self, obj: Notification) -> Dict[str, str]: - return { - "title": _(obj.template_id.title_key), - "description": _(obj.template_id.description_key) % obj.arguments, - } + return get_message_dict(obj) class Meta: model = Notification diff --git a/backend/notifications/signals.py b/backend/notifications/signals.py index 2ded382f..6d7e993f 100644 --- a/backend/notifications/signals.py +++ b/backend/notifications/signals.py @@ -6,14 +6,19 @@ from authentication.models import User from django.dispatch import Signal, receiver from django.urls import reverse +from notifications.logic import send_mails from notifications.serializers import NotificationSerializer notification_create = Signal() +# TODO: Remove send_mails call @receiver(notification_create) def notification_creation( - type: NotificationType, user: User, arguments: Dict[str, str], **kwargs + type: NotificationType, + user: User, + arguments: Dict[str, str], + **kwargs, # Required by django ) -> bool: serializer = NotificationSerializer( data={ @@ -28,6 +33,8 @@ def notification_creation( serializer.save() + send_mails() + return True diff --git a/backend/notifications/urls.py b/backend/notifications/urls.py index e80acd66..a9dacf25 100644 --- a/backend/notifications/urls.py +++ b/backend/notifications/urls.py @@ -1,6 +1,8 @@ from django.urls import path -from notifications.views import NotificationView +from notifications.views import NotificationView, TestingView +# TODO: Remove test urlpatterns = [ - path("/", NotificationView.as_view(), name="notification-detail"), + path("tmp//", NotificationView.as_view(), name="notification-detail"), + path("test/", TestingView.as_view(), name="notification-test"), ] diff --git a/backend/notifications/views.py b/backend/notifications/views.py index 5f4dd772..c83d2da8 100644 --- a/backend/notifications/views.py +++ b/backend/notifications/views.py @@ -36,3 +36,29 @@ def post(self, request: Request, user_id: str) -> Response: notifications.update(is_read=True) return Response(status=HTTP_200_OK) + + +# TODO: Remove this view +class TestingView(APIView): + permission_classes = [IsAuthenticated] + + def get(self, request: Request): + from notifications.signals import NotificationType, notification_create + + print("Hi") + + notification_create.send( + sender="", + type=NotificationType.SCORE_ADDED, + user=request.user, + arguments={"score": "10"}, + ) + + return Response(status=HTTP_200_OK) + + def post(self, request: Request): + from notifications.logic import send_mails + + send_mails() + + return Response(status=HTTP_200_OK) diff --git a/backend/requirements.txt b/backend/requirements.txt index f0eec8cc..610d422a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,4 +6,5 @@ drf-yasg==1.21.7 requests==2.31.0 cas-client==1.0.0 psycopg2-binary==2.9.9 -djangorestframework-simplejwt==5.3.1 \ No newline at end of file +djangorestframework-simplejwt==5.3.1 +celery[redis]==5.3.6 \ No newline at end of file diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 32355200..1e496534 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -65,11 +65,9 @@ ], "DEFAULT_AUTHENTICATION_CLASSES": [ "rest_framework_simplejwt.authentication.JWTAuthentication", - "rest_framework.authentication.SessionAuthentication" + "rest_framework.authentication.SessionAuthentication", ], - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.IsAuthenticated' - ] + "DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"], } SIMPLE_JWT = { @@ -130,3 +128,12 @@ }, }, ] + +EMAIL_HOST = "smtprelay.UGent.be" +EMAIL_PORT = 25 + +EMAIL_CUSTOM = { + "from": "ypovoli@ugent.be", + "subject": "[Ypovoli] New Notification", + "timeout": 2, +} From d7e9d0ee370cb666fa178d6365d0dc30b6f232ab Mon Sep 17 00:00:00 2001 From: Topvennie Date: Sun, 10 Mar 2024 22:23:32 +0100 Subject: [PATCH 02/13] feat(deployment): Added development environment / container --- .env | 9 +++++ .gitignore | 1 + README.md | 6 +++ backend/Dockerfile | 17 +++++++++ data/nginx/nginx.conf | 64 ++++++++++++++++++++++++++++++++ data/nginx/ssl/certificate.crt | 19 ++++++++++ data/nginx/ssl/nginx.conf | 0 data/nginx/ssl/private.key | 28 ++++++++++++++ development.sh | 3 ++ development.yml | 67 ++++++++++++++++++++++++++++++++++ frontend/Dockerfile | 12 ++++++ frontend/nginx.conf | 9 +++++ 12 files changed, 235 insertions(+) create mode 100644 .env create mode 100644 backend/Dockerfile create mode 100644 data/nginx/nginx.conf create mode 100644 data/nginx/ssl/certificate.crt create mode 100755 data/nginx/ssl/nginx.conf create mode 100644 data/nginx/ssl/private.key create mode 100755 development.sh create mode 100644 development.yml create mode 100644 frontend/Dockerfile create mode 100644 frontend/nginx.conf diff --git a/.env b/.env new file mode 100644 index 00000000..118bb629 --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +PUID=1000 +PGID=1000 +TZ="Europe/Brussels" + +DATADIR="./data" + +BACKEND_DIR="./backend" + +FRONTEND_DIR="./frontend" diff --git a/.gitignore b/.gitignore index f494b1b6..7c90a8ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .tool-versions +data/postgres/* \ No newline at end of file diff --git a/README.md b/README.md index 0c3a257d..6a987026 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ This application was developed within the framework of the course "Software Engi ## Development +Run `development.sh`. +It starts the development environment and attaches itself to the output of the backend. +The backend will auto reload when changing a file. + +If you change something to one of the docker files run `docker-compose -f development.yml up --build` to rebuild. + ### Backend Instructions for the setup of the Django backend are to be found in `backend/README.md`. diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..bce03b5a --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11.4 + +RUN apt update && apt install -y gettext libgettextpo-dev && pip install --upgrade pip + +WORKDIR /code + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY requirements.txt /code/ +RUN pip install -r requirements.txt + +COPY . /code/ + +RUN ./setup.sh + +CMD ["python", "manage.py", "runsslserver", "192.168.90.2:8080"] \ No newline at end of file diff --git a/data/nginx/nginx.conf b/data/nginx/nginx.conf new file mode 100644 index 00000000..af8d4362 --- /dev/null +++ b/data/nginx/nginx.conf @@ -0,0 +1,64 @@ +events { + worker_connections 1024; +} + +http { + upstream backend { + server backend:8080; + } + + upstream frontend { + server frontend:3000; + } + + server { + listen 80; + listen [::]:80; + + location / { + proxy_pass http://frontend; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + + } + + server { + + listen 8080; + listen [::]:8080; + + location / { + proxy_pass https://backend; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + } + + server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + + ssl_certificate certificate.crt; + ssl_certificate_key private.key; + + return 301 http://$host$request_uri; + } + + server { + listen 8080 ssl default_server; + listen [::]:8080 ssl default_server; + + ssl_certificate certificate.crt; + ssl_certificate_key private.key; + + location / { + proxy_pass https://backend; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_redirect off; + } + } +} diff --git a/data/nginx/ssl/certificate.crt b/data/nginx/ssl/certificate.crt new file mode 100644 index 00000000..2e061c41 --- /dev/null +++ b/data/nginx/ssl/certificate.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfkCFBhBz10TLA7SeDaEZKMIVotHAu8FMA0GCSqGSIb3DQEBCwUAMEUx +CzAJBgNVBAYTAkJFMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMzEwMjA1ODQ0WhcNMjUwMzEwMjA1 +ODQ0WjBFMQswCQYDVQQGEwJCRTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE +CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAxtUMnT5ahfM7VjK+ls+sr9+rvdjuuWtfJ7r49hAzucRFf8DC +Ew63bO/RpLFGW6vp4uBsL6y8zonQWjBvHnf9YUDBV+ZR+Pl/R+Zz6uB/yLHqBuT8 +InXFU4taptV5jPYx5/F6VpubE/W46gzXK66akgAoKUxlHZXdoxMbMBqjpUVz85Hx +BLM6N+LRcttl/6ittCPPjgkR7KF08AZu+jnm7NzJ16eF0VmgMAhKqozwSkegcQSb +vzDucmIDsLl/ZvFkP8L2tDNh1GNRcByPQ7pAR+VsIBkZOgAtkYyTMQP/mpHS7HKW +uSiT6o+2A8J7Y58nkPHpk1L7dVdWaeLB0ZmnAwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQCR220B4S08uxtOFll0q93QVpvyw5sOnumjeUOd1GRp2n/mgAzjA6/ANwgu +xXzdS/jCk6Bqv8gWqKDaFCvTC4437Ql5PRw8mho72K9cx/4T4TbTi7UffxU0BB7f +2XYSS0ORdAaSs3MzwnBBHTPM94EfbDPvfhWmDUwJT2QM1CAsHJ7KqMnYbWdR5j7h +XIy4M7NiE76QWSsjliN+s9f1CWFKtH4vZ+6nlu2uX473Hg6PX9AZD8MBC4juvRsS +wWBiF+x2498yMqzc+bYkEDqox1u9BfMfi8dbBu+QUBs4dpN9cdInEYcN5yTJBVHF +st2ChvK+w/ZptJ95fbDFiDhdL2mt +-----END CERTIFICATE----- diff --git a/data/nginx/ssl/nginx.conf b/data/nginx/ssl/nginx.conf new file mode 100755 index 00000000..e69de29b diff --git a/data/nginx/ssl/private.key b/data/nginx/ssl/private.key new file mode 100644 index 00000000..df1ca317 --- /dev/null +++ b/data/nginx/ssl/private.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDG1QydPlqF8ztW +Mr6Wz6yv36u92O65a18nuvj2EDO5xEV/wMITDrds79GksUZbq+ni4GwvrLzOidBa +MG8ed/1hQMFX5lH4+X9H5nPq4H/IseoG5PwidcVTi1qm1XmM9jHn8XpWm5sT9bjq +DNcrrpqSACgpTGUdld2jExswGqOlRXPzkfEEszo34tFy22X/qK20I8+OCRHsoXTw +Bm76Oebs3MnXp4XRWaAwCEqqjPBKR6BxBJu/MO5yYgOwuX9m8WQ/wva0M2HUY1Fw +HI9DukBH5WwgGRk6AC2RjJMxA/+akdLscpa5KJPqj7YDwntjnyeQ8emTUvt1V1Zp +4sHRmacDAgMBAAECggEAQRKL4NB93tXmZwUPhBruiNa6hdT/+BYQW9fgz+MokpUO +K8vhmEwaMuBf67cK8EiYsKRDM+0kE7Jdyo6MZ1vcxJ3lSQe7bzD0e4sMB+Q2XfAA +SAZcEEkb7gYvAmfeMoiGd8L7h2nAvK0QOiU+rHCl7L95ZV63vxGDqnG/1aP6R8Wa +BiK2ncHymUtsJc05lSrkbTo+rOiuR0AMPkBcw+egKb0Topyu9UNL9FwCiZvUg10o +82OtJqIXDJ4CMIU/70apJ59AR7h+1ekAgBZwJ0Pf00OFOxJUORIoKQg08u8JjoNo +duzsbf095b6lcNGB00sWqTnpqbkgEqjU2YU/N64eOQKBgQDlGg6b+C1ob6JL3QN0 +qra4fRIAzDWaT4UU1ePWovK65kG1t3lvh75e1bSRW3v3Uwy33DB+atI90VA/zsAb +aEFlWufnkGSl2Mr+0ommpPkl7ML01be4IjivXQzMV9JBhizpUNZ/TELCspcwhzjx +owtfDtcQwVks/dtjwt+euSXtGQKBgQDeLTPO+h+QLnS/H7+jJjAX4vIVhXIbSXZm +ROq6/6X8wCpIdf4x2/y38VkjS5ayeNMNq6joRzte1Q9eURCwekj8kbIdh04cKSSg +Wc3waWDMvMNw52drzmadfe2mW64X42Md0mzXH6F7RVH4ZwxCu2d8JAWaXBpJHoL6 +URlRBkIcewKBgACi3+OC/u1JUhQP2xCZ4MQGZORnrMZu7hmutmFENpRaS1hr2AR9 +RgQRZ9z3ehKnwmNIU0ImncraJ/TlaBcrZPMZG4fDGOR1A6tNfmBeGOsIC0qOxWHX +hnzGL2Dp5YWVD87eEJpt5cmQoWbbGUdigoeTDPnY75x2YAOY6PIR5Y8RAoGAJNVX +nnvHGc8p2bm4uqKNHJiqS7kY5r8yGthYFfJmIVX2bJbrMnbnGdOwVHKmpCX1z3Fj +Ckcs55bo+lj0LF3Jld3NqqmQ4IhNoyvgQXgm7SpqOGCUu8G3L2r+KDNQ1HMFLp+B +HdUHn3kpksX6uWF6UZFjQGj+jpq5WihxywX/ldsCgYAsVhw8tvkN5Dysu5VWt/XN +drj5/V9MRsvJ4cJ2fg3EA8iccTaoaDmrlpwZ+deSBQzKlxnzGFTsSUMNySjK3wAq +0tOvKN8FhJcSttbYUaTVyexMfpNihm/NIIXqsTs7jUtE0qlgrfoOqip6/FAgfsxE +KAeZx1IBFLr9C5UcDKFSFw== +-----END PRIVATE KEY----- diff --git a/development.sh b/development.sh new file mode 100755 index 00000000..bb64f9fd --- /dev/null +++ b/development.sh @@ -0,0 +1,3 @@ +docker-compose -f development.yml up -d + +docker-compose -f development.yml logs --follow --tail 50 backend \ No newline at end of file diff --git a/development.yml b/development.yml new file mode 100644 index 00000000..d72ee963 --- /dev/null +++ b/development.yml @@ -0,0 +1,67 @@ +version: "3.9" + +############################# NETWORKS + +networks: + selab_network: + name: selab_network + driver: bridge + ipam: + config: + - subnet: 192.168.90.0/24 + +############################# EXTENSIONS + +x-common-keys-selab: &common-keys-selab + networks: + - selab_network + security_opt: + - no-new-privileges:true + restart: unless-stopped + environment: + TZ: $TZ + PUID: $PUID + PGID: $PGID + env_file: + - .env + +############################# SERVICES + +services: + + nginx: + <<: *common-keys-selab + image: nginx:latest + container_name: nginx + ports: + - 80:80 + - 443:443 + - 8080:8080 + volumes: + - $DATADIR/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - $DATADIR/nginx/ssl:/etc/nginx/ + depends_on: + - backend + - frontend + + backend: + <<: *common-keys-selab + container_name: backend + build: + context: $BACKEND_DIR + dockerfile: Dockerfile + expose: + - 8000 + volumes: + - $BACKEND_DIR:/code + + frontend: + <<: *common-keys-selab + container_name: frontend + build: + context: $FRONTEND_DIR + dockerfile: Dockerfile + expose: + - 3000 + depends_on: + - backend \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 00000000..5254aa37 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,12 @@ +FROM node:16 as build-stage +WORKDIR /app +COPY package*.json ./ +RUN npm install +COPY ./ . +RUN npm run build + +FROM nginx as production-stage +EXPOSE 3000 +RUN mkdir /app +COPY nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=build-stage /app/dist /app \ No newline at end of file diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 00000000..0ae8c83e --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,9 @@ +server { + listen 3000; + + location / { + root /app; + index index.html index.htm; + try_files $uri $uri/ /index.html; + } +} \ No newline at end of file From 41abbbcc2c2144c5375a1e740fe859dc4d3b6aa3 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Sun, 10 Mar 2024 22:40:59 +0100 Subject: [PATCH 03/13] chore: http -> https --- README.md | 1 + data/nginx/nginx.conf | 32 ++++++++++---------------------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6a987026..a7503111 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ This application was developed within the framework of the course "Software Engi Run `development.sh`. It starts the development environment and attaches itself to the output of the backend. The backend will auto reload when changing a file. +Acces the server by going to `https://localhost:8080` for the backend and `https://localhost:443` for the frontend. If you change something to one of the docker files run `docker-compose -f development.yml up --build` to rebuild. diff --git a/data/nginx/nginx.conf b/data/nginx/nginx.conf index af8d4362..a3a6b6d0 100644 --- a/data/nginx/nginx.conf +++ b/data/nginx/nginx.conf @@ -12,25 +12,23 @@ http { } server { - listen 80; - listen [::]:80; + listen 80; + listen [::]:80; location / { - proxy_pass http://frontend; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $host; - proxy_redirect off; + return 301 https://$host$request_uri; } - } server { + listen 443 ssl; + listen [::]:443 ssl; - listen 8080; - listen [::]:8080; + ssl_certificate certificate.crt; + ssl_certificate_key private.key; location / { - proxy_pass https://backend; + proxy_pass https://frontend; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; proxy_redirect off; @@ -38,18 +36,8 @@ http { } server { - listen 443 ssl default_server; - listen [::]:443 ssl default_server; - - ssl_certificate certificate.crt; - ssl_certificate_key private.key; - - return 301 http://$host$request_uri; - } - - server { - listen 8080 ssl default_server; - listen [::]:8080 ssl default_server; + listen 8080 ssl; + listen [::]:8080 ssl; ssl_certificate certificate.crt; ssl_certificate_key private.key; From b765db6487c677c4fe520a63a278803ac836ea64 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Sun, 10 Mar 2024 22:56:02 +0100 Subject: [PATCH 04/13] chore: generate ssl keys --- .gitignore | 3 ++- data/nginx/ssl/certificate.crt | 19 ------------------- data/nginx/ssl/nginx.conf | 0 data/nginx/ssl/private.key | 28 ---------------------------- development.sh | 14 ++++++++++++++ 5 files changed, 16 insertions(+), 48 deletions(-) delete mode 100644 data/nginx/ssl/certificate.crt delete mode 100755 data/nginx/ssl/nginx.conf delete mode 100644 data/nginx/ssl/private.key diff --git a/.gitignore b/.gitignore index 7c90a8ac..3984195e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .tool-versions -data/postgres/* \ No newline at end of file +data/postgres/* +data/nginx/ssl/* \ No newline at end of file diff --git a/data/nginx/ssl/certificate.crt b/data/nginx/ssl/certificate.crt deleted file mode 100644 index 2e061c41..00000000 --- a/data/nginx/ssl/certificate.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDETCCAfkCFBhBz10TLA7SeDaEZKMIVotHAu8FMA0GCSqGSIb3DQEBCwUAMEUx -CzAJBgNVBAYTAkJFMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjQwMzEwMjA1ODQ0WhcNMjUwMzEwMjA1 -ODQ0WjBFMQswCQYDVQQGEwJCRTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAxtUMnT5ahfM7VjK+ls+sr9+rvdjuuWtfJ7r49hAzucRFf8DC -Ew63bO/RpLFGW6vp4uBsL6y8zonQWjBvHnf9YUDBV+ZR+Pl/R+Zz6uB/yLHqBuT8 -InXFU4taptV5jPYx5/F6VpubE/W46gzXK66akgAoKUxlHZXdoxMbMBqjpUVz85Hx -BLM6N+LRcttl/6ittCPPjgkR7KF08AZu+jnm7NzJ16eF0VmgMAhKqozwSkegcQSb -vzDucmIDsLl/ZvFkP8L2tDNh1GNRcByPQ7pAR+VsIBkZOgAtkYyTMQP/mpHS7HKW -uSiT6o+2A8J7Y58nkPHpk1L7dVdWaeLB0ZmnAwIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCR220B4S08uxtOFll0q93QVpvyw5sOnumjeUOd1GRp2n/mgAzjA6/ANwgu -xXzdS/jCk6Bqv8gWqKDaFCvTC4437Ql5PRw8mho72K9cx/4T4TbTi7UffxU0BB7f -2XYSS0ORdAaSs3MzwnBBHTPM94EfbDPvfhWmDUwJT2QM1CAsHJ7KqMnYbWdR5j7h -XIy4M7NiE76QWSsjliN+s9f1CWFKtH4vZ+6nlu2uX473Hg6PX9AZD8MBC4juvRsS -wWBiF+x2498yMqzc+bYkEDqox1u9BfMfi8dbBu+QUBs4dpN9cdInEYcN5yTJBVHF -st2ChvK+w/ZptJ95fbDFiDhdL2mt ------END CERTIFICATE----- diff --git a/data/nginx/ssl/nginx.conf b/data/nginx/ssl/nginx.conf deleted file mode 100755 index e69de29b..00000000 diff --git a/data/nginx/ssl/private.key b/data/nginx/ssl/private.key deleted file mode 100644 index df1ca317..00000000 --- a/data/nginx/ssl/private.key +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDG1QydPlqF8ztW -Mr6Wz6yv36u92O65a18nuvj2EDO5xEV/wMITDrds79GksUZbq+ni4GwvrLzOidBa -MG8ed/1hQMFX5lH4+X9H5nPq4H/IseoG5PwidcVTi1qm1XmM9jHn8XpWm5sT9bjq -DNcrrpqSACgpTGUdld2jExswGqOlRXPzkfEEszo34tFy22X/qK20I8+OCRHsoXTw -Bm76Oebs3MnXp4XRWaAwCEqqjPBKR6BxBJu/MO5yYgOwuX9m8WQ/wva0M2HUY1Fw -HI9DukBH5WwgGRk6AC2RjJMxA/+akdLscpa5KJPqj7YDwntjnyeQ8emTUvt1V1Zp -4sHRmacDAgMBAAECggEAQRKL4NB93tXmZwUPhBruiNa6hdT/+BYQW9fgz+MokpUO -K8vhmEwaMuBf67cK8EiYsKRDM+0kE7Jdyo6MZ1vcxJ3lSQe7bzD0e4sMB+Q2XfAA -SAZcEEkb7gYvAmfeMoiGd8L7h2nAvK0QOiU+rHCl7L95ZV63vxGDqnG/1aP6R8Wa -BiK2ncHymUtsJc05lSrkbTo+rOiuR0AMPkBcw+egKb0Topyu9UNL9FwCiZvUg10o -82OtJqIXDJ4CMIU/70apJ59AR7h+1ekAgBZwJ0Pf00OFOxJUORIoKQg08u8JjoNo -duzsbf095b6lcNGB00sWqTnpqbkgEqjU2YU/N64eOQKBgQDlGg6b+C1ob6JL3QN0 -qra4fRIAzDWaT4UU1ePWovK65kG1t3lvh75e1bSRW3v3Uwy33DB+atI90VA/zsAb -aEFlWufnkGSl2Mr+0ommpPkl7ML01be4IjivXQzMV9JBhizpUNZ/TELCspcwhzjx -owtfDtcQwVks/dtjwt+euSXtGQKBgQDeLTPO+h+QLnS/H7+jJjAX4vIVhXIbSXZm -ROq6/6X8wCpIdf4x2/y38VkjS5ayeNMNq6joRzte1Q9eURCwekj8kbIdh04cKSSg -Wc3waWDMvMNw52drzmadfe2mW64X42Md0mzXH6F7RVH4ZwxCu2d8JAWaXBpJHoL6 -URlRBkIcewKBgACi3+OC/u1JUhQP2xCZ4MQGZORnrMZu7hmutmFENpRaS1hr2AR9 -RgQRZ9z3ehKnwmNIU0ImncraJ/TlaBcrZPMZG4fDGOR1A6tNfmBeGOsIC0qOxWHX -hnzGL2Dp5YWVD87eEJpt5cmQoWbbGUdigoeTDPnY75x2YAOY6PIR5Y8RAoGAJNVX -nnvHGc8p2bm4uqKNHJiqS7kY5r8yGthYFfJmIVX2bJbrMnbnGdOwVHKmpCX1z3Fj -Ckcs55bo+lj0LF3Jld3NqqmQ4IhNoyvgQXgm7SpqOGCUu8G3L2r+KDNQ1HMFLp+B -HdUHn3kpksX6uWF6UZFjQGj+jpq5WihxywX/ldsCgYAsVhw8tvkN5Dysu5VWt/XN -drj5/V9MRsvJ4cJ2fg3EA8iccTaoaDmrlpwZ+deSBQzKlxnzGFTsSUMNySjK3wAq -0tOvKN8FhJcSttbYUaTVyexMfpNihm/NIIXqsTs7jUtE0qlgrfoOqip6/FAgfsxE -KAeZx1IBFLr9C5UcDKFSFw== ------END PRIVATE KEY----- diff --git a/development.sh b/development.sh index bb64f9fd..ed637eb4 100755 --- a/development.sh +++ b/development.sh @@ -1,3 +1,17 @@ +echo "Checking for existing SSL certificates..." + +if [ ! -f "data/nginx/ssl/private.key" ] || [ ! -f "data/nginx/ssl/certificate.crt" ]; then +echo "Generating SSL certificates..." + sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout data/nginx/ssl/private.key \ + -out data/nginx/ssl/certificate.crt \ + -subj "/C=BE/ST=/L=/O=/OU=/CN=" +else + echo "SSL certificates already exist, skipping generation." +fi + +echo "Starting services..." docker-compose -f development.yml up -d +echo "Following logs..." docker-compose -f development.yml logs --follow --tail 50 backend \ No newline at end of file From 176183ce66d89d16b5d07438e0845c8464aa8586 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Sun, 10 Mar 2024 23:07:05 +0100 Subject: [PATCH 05/13] chore: keep ssl directory --- data/nginx/ssl/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/nginx/ssl/.gitkeep diff --git a/data/nginx/ssl/.gitkeep b/data/nginx/ssl/.gitkeep new file mode 100644 index 00000000..e69de29b From eb0022837eb995171a0bf000e0e00f020226ea00 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 00:13:13 +0100 Subject: [PATCH 06/13] chore: Use celery & Redis --- .env | 4 ++++ .gitignore | 6 ++++-- backend/Dockerfile | 4 ---- backend/notifications/logic.py | 19 ++++++++++++++++++- backend/notifications/signals.py | 4 ++-- backend/requirements.txt | 3 ++- backend/ypovoli/__init__.py | 5 +++++ backend/ypovoli/celery.py | 22 ++++++++++++++++++++++ backend/ypovoli/settings.py | 21 +++++++++++++++++++++ data/nginx/nginx.conf | 8 ++++---- development.sh | 19 +++++++++++++++---- development.yml | 32 ++++++++++++++++++++++++++++++-- frontend/Dockerfile | 2 +- 13 files changed, 128 insertions(+), 21 deletions(-) create mode 100644 backend/ypovoli/celery.py diff --git a/.env b/.env index 118bb629..0d3f97be 100644 --- a/.env +++ b/.env @@ -7,3 +7,7 @@ DATADIR="./data" BACKEND_DIR="./backend" FRONTEND_DIR="./frontend" + +REDIS_IP="192.168.90.10" +REDIS_PORT=6379 +REDIS_PASSWORD="oqOsNX1PXGOX5soJtKkw" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3984195e..5dafea5e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .tool-versions -data/postgres/* -data/nginx/ssl/* \ No newline at end of file +data/* + +!data/nginx/nginx.conf +!data/nginx/ssl/.gitkeep diff --git a/backend/Dockerfile b/backend/Dockerfile index bce03b5a..0365d5f9 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -11,7 +11,3 @@ COPY requirements.txt /code/ RUN pip install -r requirements.txt COPY . /code/ - -RUN ./setup.sh - -CMD ["python", "manage.py", "runsslserver", "192.168.90.2:8080"] \ No newline at end of file diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py index 99104eb9..f7140134 100644 --- a/backend/notifications/logic.py +++ b/backend/notifications/logic.py @@ -2,7 +2,9 @@ from smtplib import SMTPException from typing import Dict, List +from celery import shared_task from django.core import mail +from django.core.cache import cache from django.utils.translation import gettext as _ from notifications.models import Notification from ypovoli.settings import EMAIL_CUSTOM @@ -17,6 +19,17 @@ def get_message_dict(notification: Notification) -> Dict[str, str]: } +# Call the function after 60 seconds and no more than once in that period +def schedule_send_mails(): + print("Hiii") + if not cache.get("notifications_send_mails"): + print("Not in cache yet") + cache.set("notifications_send_mails", True) + _send_mails.apply_async(countdown=60) + else: + print("Already in cache") + + # Try to send one email and set the result def _send_mail(mail: mail.EmailMessage, result: List[bool]): try: @@ -27,7 +40,11 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]): # Send all unsent emails -def send_mails(): +@shared_task +def _send_mails(): + print("Sending") + cache.set("notifications_send_mails", False) + notifications = Notification.objects.filter(is_sent=False) # No notifications to send diff --git a/backend/notifications/signals.py b/backend/notifications/signals.py index 6d7e993f..7a817e5a 100644 --- a/backend/notifications/signals.py +++ b/backend/notifications/signals.py @@ -6,7 +6,7 @@ from authentication.models import User from django.dispatch import Signal, receiver from django.urls import reverse -from notifications.logic import send_mails +from notifications.logic import schedule_send_mails from notifications.serializers import NotificationSerializer notification_create = Signal() @@ -33,7 +33,7 @@ def notification_creation( serializer.save() - send_mails() + schedule_send_mails() return True diff --git a/backend/requirements.txt b/backend/requirements.txt index 610d422a..d1dab654 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -7,4 +7,5 @@ requests==2.31.0 cas-client==1.0.0 psycopg2-binary==2.9.9 djangorestframework-simplejwt==5.3.1 -celery[redis]==5.3.6 \ No newline at end of file +celery[redis]==5.3.6 +django-redis==5.4.0 \ No newline at end of file diff --git a/backend/ypovoli/__init__.py b/backend/ypovoli/__init__.py index e69de29b..5568b6d7 100644 --- a/backend/ypovoli/__init__.py +++ b/backend/ypovoli/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/backend/ypovoli/celery.py b/backend/ypovoli/celery.py new file mode 100644 index 00000000..4195e638 --- /dev/null +++ b/backend/ypovoli/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ypovoli.settings") + +app = Celery("ypovoli") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f"Request: {self.request!r}") diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 1e496534..22b29a82 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ +import os from datetime import timedelta from pathlib import Path @@ -137,3 +138,23 @@ "subject": "[Ypovoli] New Notification", "timeout": 2, } + +REDIS_CUSTOM = { + "host": os.environ.get("REDIS_IP", "localhost"), + "port": os.environ.get("REDIS_PORT", 6379), + "password": os.environ.get("REDIS_PASSWORD", ""), + "db_django": 0, + "db_celery": 1, +} + +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_django']}", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + } +} + +CELERY_BROKER_URL = f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}" diff --git a/data/nginx/nginx.conf b/data/nginx/nginx.conf index a3a6b6d0..c46ebf37 100644 --- a/data/nginx/nginx.conf +++ b/data/nginx/nginx.conf @@ -24,8 +24,8 @@ http { listen 443 ssl; listen [::]:443 ssl; - ssl_certificate certificate.crt; - ssl_certificate_key private.key; + ssl_certificate ssl/certificate.crt; + ssl_certificate_key ssl/private.key; location / { proxy_pass https://frontend; @@ -39,8 +39,8 @@ http { listen 8080 ssl; listen [::]:8080 ssl; - ssl_certificate certificate.crt; - ssl_certificate_key private.key; + ssl_certificate ssl/certificate.crt; + ssl_certificate_key ssl/private.key; location / { proxy_pass https://backend; diff --git a/development.sh b/development.sh index ed637eb4..24697323 100755 --- a/development.sh +++ b/development.sh @@ -1,11 +1,12 @@ echo "Checking for existing SSL certificates..." if [ ! -f "data/nginx/ssl/private.key" ] || [ ! -f "data/nginx/ssl/certificate.crt" ]; then -echo "Generating SSL certificates..." + echo "Generating SSL certificates..." sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ -keyout data/nginx/ssl/private.key \ -out data/nginx/ssl/certificate.crt \ - -subj "/C=BE/ST=/L=/O=/OU=/CN=" + -subj "/C=BE/ST=/L=/O=/OU=/CN=" > /dev/null + echo "SSL certificates generated." else echo "SSL certificates already exist, skipping generation." fi @@ -13,5 +14,15 @@ fi echo "Starting services..." docker-compose -f development.yml up -d -echo "Following logs..." -docker-compose -f development.yml logs --follow --tail 50 backend \ No newline at end of file +echo "-------------------------------------" +echo "Following backend logs..." +echo "Press CTRL + C to stop all containers" +echo "-------------------------------------" + +docker-compose -f development.yml logs --follow --tail 50 backend + +echo "Cleaning up..." + +docker-compose -f development.yml down + +echo "Done." diff --git a/development.yml b/development.yml index d72ee963..58a532e3 100644 --- a/development.yml +++ b/development.yml @@ -39,7 +39,7 @@ services: - 8080:8080 volumes: - $DATADIR/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - - $DATADIR/nginx/ssl:/etc/nginx/ + - $DATADIR/nginx/ssl:/etc/nginx/ssl:ro depends_on: - backend - frontend @@ -50,11 +50,25 @@ services: build: context: $BACKEND_DIR dockerfile: Dockerfile + command: bash -c "./setup.sh && python manage.py runsslserver 192.168.90.2:8080" expose: - 8000 volumes: - $BACKEND_DIR:/code + celery: + <<: *common-keys-selab + container_name: celery + build: + context: $BACKEND_DIR + dockerfile: Dockerfile + command: celery -A ypovoli worker -l INFO + volumes: + - $BACKEND_DIR:/code + depends_on: + - backend + - redis + frontend: <<: *common-keys-selab container_name: frontend @@ -64,4 +78,18 @@ services: expose: - 3000 depends_on: - - backend \ No newline at end of file + - backend + + redis: + <<: *common-keys-selab + container_name: redis + image: redis:latest + networks: + selab_network: + ipv4_address: $REDIS_IP + expose: + - $REDIS_PORT + entrypoint: redis-server --appendonly yes --requirepass $REDIS_PASSWORD --maxmemory 512mb --maxmemory-policy allkeys-lru + volumes: + - $DATADIR/redis:/data + \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5254aa37..d3a568de 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,7 +5,7 @@ RUN npm install COPY ./ . RUN npm run build -FROM nginx as production-stage +FROM nginx as development-stage EXPOSE 3000 RUN mkdir /app COPY nginx.conf /etc/nginx/conf.d/default.conf From cd4efb82c95797efe312d284bac2833200c9bff8 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 00:46:35 +0100 Subject: [PATCH 07/13] chore: cleanup + dev environment --- backend/notifications/logic.py | 5 ----- backend/notifications/signals.py | 1 - backend/notifications/urls.py | 4 +--- backend/notifications/views.py | 27 --------------------------- backend/ypovoli/celery.py | 5 ----- data/nginx/nginx.conf | 4 ++-- development.yml | 5 ++++- frontend/Dockerfile | 28 +++++++++++++++++++--------- frontend/package.json | 1 + 9 files changed, 27 insertions(+), 53 deletions(-) diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py index f7140134..731b4281 100644 --- a/backend/notifications/logic.py +++ b/backend/notifications/logic.py @@ -21,13 +21,9 @@ def get_message_dict(notification: Notification) -> Dict[str, str]: # Call the function after 60 seconds and no more than once in that period def schedule_send_mails(): - print("Hiii") if not cache.get("notifications_send_mails"): - print("Not in cache yet") cache.set("notifications_send_mails", True) _send_mails.apply_async(countdown=60) - else: - print("Already in cache") # Try to send one email and set the result @@ -42,7 +38,6 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]): # Send all unsent emails @shared_task def _send_mails(): - print("Sending") cache.set("notifications_send_mails", False) notifications = Notification.objects.filter(is_sent=False) diff --git a/backend/notifications/signals.py b/backend/notifications/signals.py index 7a817e5a..24d7ea4e 100644 --- a/backend/notifications/signals.py +++ b/backend/notifications/signals.py @@ -12,7 +12,6 @@ notification_create = Signal() -# TODO: Remove send_mails call @receiver(notification_create) def notification_creation( type: NotificationType, diff --git a/backend/notifications/urls.py b/backend/notifications/urls.py index a9dacf25..23b79d7e 100644 --- a/backend/notifications/urls.py +++ b/backend/notifications/urls.py @@ -1,8 +1,6 @@ from django.urls import path -from notifications.views import NotificationView, TestingView +from notifications.views import NotificationView -# TODO: Remove test urlpatterns = [ path("tmp//", NotificationView.as_view(), name="notification-detail"), - path("test/", TestingView.as_view(), name="notification-test"), ] diff --git a/backend/notifications/views.py b/backend/notifications/views.py index c83d2da8..b5d1f4f6 100644 --- a/backend/notifications/views.py +++ b/backend/notifications/views.py @@ -11,7 +11,6 @@ from rest_framework.views import APIView -# TODO: Give admin access to everything class NotificationPermission(BasePermission): # The user can only access their own notifications # An admin can access all notifications @@ -36,29 +35,3 @@ def post(self, request: Request, user_id: str) -> Response: notifications.update(is_read=True) return Response(status=HTTP_200_OK) - - -# TODO: Remove this view -class TestingView(APIView): - permission_classes = [IsAuthenticated] - - def get(self, request: Request): - from notifications.signals import NotificationType, notification_create - - print("Hi") - - notification_create.send( - sender="", - type=NotificationType.SCORE_ADDED, - user=request.user, - arguments={"score": "10"}, - ) - - return Response(status=HTTP_200_OK) - - def post(self, request: Request): - from notifications.logic import send_mails - - send_mails() - - return Response(status=HTTP_200_OK) diff --git a/backend/ypovoli/celery.py b/backend/ypovoli/celery.py index 4195e638..5f2cac39 100644 --- a/backend/ypovoli/celery.py +++ b/backend/ypovoli/celery.py @@ -15,8 +15,3 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() - - -@app.task(bind=True, ignore_result=True) -def debug_task(self): - print(f"Request: {self.request!r}") diff --git a/data/nginx/nginx.conf b/data/nginx/nginx.conf index c46ebf37..f6a71bee 100644 --- a/data/nginx/nginx.conf +++ b/data/nginx/nginx.conf @@ -8,7 +8,7 @@ http { } upstream frontend { - server frontend:3000; + server frontend:5173; } server { @@ -28,7 +28,7 @@ http { ssl_certificate_key ssl/private.key; location / { - proxy_pass https://frontend; + proxy_pass http://frontend; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $host; proxy_redirect off; diff --git a/development.yml b/development.yml index 58a532e3..2530320b 100644 --- a/development.yml +++ b/development.yml @@ -75,8 +75,11 @@ services: build: context: $FRONTEND_DIR dockerfile: Dockerfile + command: bash -c "npm install && npm run host" expose: - - 3000 + - 5173 + volumes: + - $FRONTEND_DIR:/app depends_on: - backend diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d3a568de..ec6c3ba7 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,12 +1,22 @@ -FROM node:16 as build-stage +# FROM node:16 as build-stage +# WORKDIR /app +# COPY package*.json ./ +# RUN npm install +# COPY ./ . +# RUN npm run build + +# FROM nginx as development-stage +# EXPOSE 3000 +# RUN mkdir /app +# COPY nginx.conf /etc/nginx/conf.d/default.conf +# COPY --from=build-stage /app/dist /app + +FROM node:16 + WORKDIR /app + COPY package*.json ./ + RUN npm install -COPY ./ . -RUN npm run build - -FROM nginx as development-stage -EXPOSE 3000 -RUN mkdir /app -COPY nginx.conf /etc/nginx/conf.d/default.conf -COPY --from=build-stage /app/dist /app \ No newline at end of file + +COPY . /app/ diff --git a/frontend/package.json b/frontend/package.json index 57fc78e0..60994926 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "dev": "vite", + "host": "vite --host", "build": "vue-tsc && vite build", "preview": "vite preview" }, From ba1a28c26df71cb853bf3a6bf109c70a2e7c5315 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 01:08:23 +0100 Subject: [PATCH 08/13] chore: linting --- backend/ypovoli/settings.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 22b29a82..38931569 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -150,11 +150,15 @@ CACHES = { "default": { "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_django']}", + "LOCATION": f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/" + f"{REDIS_CUSTOM['db_django']}", "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", }, } } -CELERY_BROKER_URL = f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/{REDIS_CUSTOM['db_celery']}" +CELERY_BROKER_URL = ( + f"redis://:{REDIS_CUSTOM['password']}@{REDIS_CUSTOM['host']}:{REDIS_CUSTOM['port']}/" + f"{REDIS_CUSTOM['db_celery']}" +) From b1bda33aeabbce32520b8efc16f508e16eb670ce Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 01:13:59 +0100 Subject: [PATCH 09/13] chore: Remove unused nginx conf --- frontend/nginx.conf | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 frontend/nginx.conf diff --git a/frontend/nginx.conf b/frontend/nginx.conf deleted file mode 100644 index 0ae8c83e..00000000 --- a/frontend/nginx.conf +++ /dev/null @@ -1,9 +0,0 @@ -server { - listen 3000; - - location / { - root /app; - index index.html index.htm; - try_files $uri $uri/ /index.html; - } -} \ No newline at end of file From 696b7d3cfd76dd2b66079665d52fbf34e11e2d08 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 11:45:23 +0100 Subject: [PATCH 10/13] chore: linter --- backend/ypovoli/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 38931569..2d09c8f3 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -10,8 +10,8 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ -import os from datetime import timedelta +from os import environ from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -140,9 +140,9 @@ } REDIS_CUSTOM = { - "host": os.environ.get("REDIS_IP", "localhost"), - "port": os.environ.get("REDIS_PORT", 6379), - "password": os.environ.get("REDIS_PASSWORD", ""), + "host": environ.get("REDIS_IP", "localhost"), + "port": environ.get("REDIS_PORT", 6379), + "password": environ.get("REDIS_PASSWORD", ""), "db_django": 0, "db_celery": 1, } From c2024e3dfcf7467cfdfdd897b71186a3d0ccf3f5 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 16:12:47 +0100 Subject: [PATCH 11/13] chore: change single user to queryset --- backend/notifications/signals.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/notifications/signals.py b/backend/notifications/signals.py index 24d7ea4e..f8203f39 100644 --- a/backend/notifications/signals.py +++ b/backend/notifications/signals.py @@ -1,9 +1,10 @@ from __future__ import annotations from enum import Enum -from typing import Dict +from typing import Dict, List, Union from authentication.models import User +from django.db.models.query import QuerySet from django.dispatch import Signal, receiver from django.urls import reverse from notifications.logic import schedule_send_mails @@ -15,17 +16,22 @@ @receiver(notification_create) def notification_creation( type: NotificationType, - user: User, + queryset: QuerySet[User], arguments: Dict[str, str], **kwargs, # Required by django ) -> bool: - serializer = NotificationSerializer( - data={ - "template_id": type.value, - "user": reverse("user-detail", kwargs={"pk": user.id}), - "arguments": arguments, - } - ) + data: List[Dict[str, Union[str, int, Dict[str, str]]]] = [] + + for user in queryset: + data.append( + { + "template_id": type.value, + "user": reverse("user-detail", kwargs={"pk": user.id}), + "arguments": arguments, + } + ) + + serializer = NotificationSerializer(data=data, many=True) if not serializer.is_valid(): return False From 84d5d1e3d7cd6fb4a280aa2a86a87191a378443b Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 16:35:17 +0100 Subject: [PATCH 12/13] chore: max sent retries --- backend/notifications/logic.py | 33 +++++++++++++++++++++++++++++---- backend/ypovoli/settings.py | 1 + 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/backend/notifications/logic.py b/backend/notifications/logic.py index 731b4281..b5df6d18 100644 --- a/backend/notifications/logic.py +++ b/backend/notifications/logic.py @@ -1,6 +1,8 @@ import threading +from collections import defaultdict +from os import error from smtplib import SMTPException -from typing import Dict, List +from typing import DefaultDict, Dict, List from celery import shared_task from django.core import mail @@ -38,9 +40,12 @@ def _send_mail(mail: mail.EmailMessage, result: List[bool]): # Send all unsent emails @shared_task def _send_mails(): - cache.set("notifications_send_mails", False) - + # All notifications that need to be sent notifications = Notification.objects.filter(is_sent=False) + # Dictionary with the number of errors for each email + errors: DefaultDict[str, int] = cache.get( + "notifications_send_mails_errors", defaultdict(int) + ) # No notifications to send if notifications.count() == 0: @@ -72,9 +77,29 @@ def _send_mails(): thread.start() thread.join(timeout=EMAIL_CUSTOM["timeout"]) - # If the email was not sent, continue + # Email failed to send if thread.is_alive() or not result[0]: + # Increase the number of errors for the email + errors[notification.user.email] += 1 + # Mark notification as sent if the maximum number of errors is reached + if errors[notification.user.email] >= EMAIL_CUSTOM["max_errors"]: + errors.pop(notification.user.email) + notification.sent() + continue + # Email sent successfully + if notification.user.email in errors: + errors.pop(notification.user.email) + # Mark the notification as sent notification.sent() + + # Save the number of errors for each email + cache.set("notifications_send_mails_errors", errors) + + # Restart the process if there are any notifications left that were not sent + unsent_notifications = Notification.objects.filter(is_sent=False) + cache.set("notifications_send_mails", False) + if unsent_notifications.count() > 0: + schedule_send_mails() diff --git a/backend/ypovoli/settings.py b/backend/ypovoli/settings.py index 2d09c8f3..414d9409 100644 --- a/backend/ypovoli/settings.py +++ b/backend/ypovoli/settings.py @@ -137,6 +137,7 @@ "from": "ypovoli@ugent.be", "subject": "[Ypovoli] New Notification", "timeout": 2, + "max_errors": 3, } REDIS_CUSTOM = { From 3f249e044c979b1335eb59587497b9ef6159ed21 Mon Sep 17 00:00:00 2001 From: Topvennie Date: Mon, 11 Mar 2024 16:36:30 +0100 Subject: [PATCH 13/13] chore: dockerfile cleanup --- frontend/Dockerfile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/frontend/Dockerfile b/frontend/Dockerfile index ec6c3ba7..5792f9fc 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,16 +1,3 @@ -# FROM node:16 as build-stage -# WORKDIR /app -# COPY package*.json ./ -# RUN npm install -# COPY ./ . -# RUN npm run build - -# FROM nginx as development-stage -# EXPOSE 3000 -# RUN mkdir /app -# COPY nginx.conf /etc/nginx/conf.d/default.conf -# COPY --from=build-stage /app/dist /app - FROM node:16 WORKDIR /app