From 9a563f74cbf19bbef71a828172a5416caf89cd69 Mon Sep 17 00:00:00 2001 From: hklarner Date: Thu, 7 Sep 2023 16:52:46 +0200 Subject: [PATCH] replaces background tasks with celery tasks --- adhocracy-plus/config/celery.py | 9 ++++++ .../commands/errored_task_notification.py | 21 -------------- apps/notifications/emails.py | 2 +- apps/projects/tasks.py | 10 +++---- docs/installation_prod.md | 28 ++++++++++++------- tests/conftest.py | 18 ++++-------- 6 files changed, 38 insertions(+), 50 deletions(-) delete mode 100644 apps/contrib/management/commands/errored_task_notification.py diff --git a/adhocracy-plus/config/celery.py b/adhocracy-plus/config/celery.py index 443b43812a..e920f3da8d 100644 --- a/adhocracy-plus/config/celery.py +++ b/adhocracy-plus/config/celery.py @@ -19,3 +19,12 @@ def dummy_task(): print(result) return result + + +@celery_app.task(name="crash_task") +def crash_task(): + """ + This task is for testing purposes only. + """ + + 1 / 0 diff --git a/apps/contrib/management/commands/errored_task_notification.py b/apps/contrib/management/commands/errored_task_notification.py deleted file mode 100644 index baff154a01..0000000000 --- a/apps/contrib/management/commands/errored_task_notification.py +++ /dev/null @@ -1,21 +0,0 @@ -from background_task.models import CompletedTask -from django.core.management.base import BaseCommand -from django.urls import reverse - - -class Command(BaseCommand): - help = "Send notifications to inform admins about taks that errored" - - def handle(self, *args, **options): - broken_tasks = CompletedTask.objects.exclude(last_error="").order_by("-run_at") - - for task in broken_tasks: - url = reverse( - "admin:{}_{}_change".format( - task._meta.app_label, task._meta.model_name - ), - args=[task.id], - ) - self.stdout.write( - "Error in Task {}, see: {} \n".format(task.task_params, url) - ) diff --git a/apps/notifications/emails.py b/apps/notifications/emails.py index 228bc884fa..60194a6aa1 100644 --- a/apps/notifications/emails.py +++ b/apps/notifications/emails.py @@ -189,7 +189,7 @@ def send_no_object(cls, object, *args, **kwargs): ), "organisation_id": organisation.id, } - tasks.send_async_no_object( + tasks.send_async_no_object.delay( cls.__module__, cls.__name__, object_dict, args, kwargs ) return [] diff --git a/apps/projects/tasks.py b/apps/projects/tasks.py index 1d2647aa51..b13bd43934 100644 --- a/apps/projects/tasks.py +++ b/apps/projects/tasks.py @@ -1,10 +1,10 @@ import importlib -from background_task import background +from celery import shared_task -@background(schedule=1) +@shared_task def send_async_no_object(email_module_name, email_class_name, object, args, kwargs): - mod = importlib.import_module(email_module_name) - cls = getattr(mod, email_class_name) - return cls().dispatch(object, *args, **kwargs) + email_module = importlib.import_module(email_module_name) + email_class = getattr(email_module, email_class_name) + email_class().dispatch(object, *args, **kwargs) diff --git a/docs/installation_prod.md b/docs/installation_prod.md index 96e0e512cb..aa0cf36d43 100644 --- a/docs/installation_prod.md +++ b/docs/installation_prod.md @@ -86,11 +86,16 @@ SECURE_CONTENT_TYPE_NOSNIFF = True SESSION_COOKIE_HTTPONLY = True FILE_UPLOAD_PERMISSIONS = 0o644 + +# celery configuration +CELERY_BROKER_URL = "redis+socket://var/run/redis/redis.sock" +CELERY_RESULT_BACKEND = "redis+socket://var/run/redis/redis.sock" +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True ``` #### Populate database -This will create all required tables via so called **migrations** +This will create all required tables via so-called **migrations** ``` python manage.py migrate @@ -117,7 +122,7 @@ Cancel the server after testing via `ctrl`+`c` ### Run the server as system daemon -In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create unit files. +In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create service files. `/etc/systemd/system/adhocracy-plus.service`: @@ -139,26 +144,29 @@ StandardError=inherit WantedBy=default.target ``` -`/etc/systemd/system/adhocracy-plus-background-task.service`: +`/etc/systemd/system/adhocracy-plus-celery-worker.service`: ``` [Unit] -Description=adhocracy+ background task +Description=adhocracy+ celery worker After=network.target [Service] +Type=forking User=aplus WorkingDirectory=/home/aplus/adhocracy-plus -ExecStart=/home/aplus/.virtualenvs/aplus/bin/python manage.py process_tasks --settings adhocracy-plus.config.settings.production --sleep 5 +ExecStart=/home/aplus/.virtualenvs/aplus/bin/celery --app adhocracy-plus worker Restart=always RestartSec=3 -StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-background-task.log +StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-celery-worker.log StandardError=inherit [Install] WantedBy=default.target ``` +Depending on your celery configuration you will also need to start a message broker service like redis or rabbit-mq and configure celery accordingly in `local.py` (see above). If you use redis and the default installation it should already be running, call `service redis status` to check. + This will log all output to files in `/var/log/adhocracy-plus/`. You will also need to create that folder before starting the service (as `root` or using `sudo`): ``` @@ -170,14 +178,14 @@ Load and start units (as `root` or using `sudo`): ``` systemctl daemon-reload systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` Enable autostart on boot: ``` systemctl enable adhocracy-plus -systemctl enable adhocracy-plus-background-task +systemctl enable adhocracy-plus-celery-worker ``` ### Setting up a proxy webserver @@ -254,7 +262,7 @@ The landing page is managed via [wagtail](https://wagtail.io/). You can find the ``` systemctl stop adhocracy-plus -systemctl stop adhocracy-plus-background-task +systemctl stop adhocracy-plus-celery-worker ``` #### Switch to user @@ -310,5 +318,5 @@ python manage.py runserver ``` systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` diff --git a/tests/conftest.py b/tests/conftest.py index 0f025320a4..3e72ee5aeb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,30 +2,18 @@ import factory import pytest +from celery import Celery from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django.urls.base import get_resolver from PIL import Image from pytest_factoryboy import register from rest_framework.test import APIClient from adhocracy4.test import factories as a4_factories from adhocracy4.test.factories.maps import AreaSettingsFactory -from adhocracy4.test.helpers import patch_background_task_decorator from . import factories - -def pytest_configure(config): - # Patch email background_task decorators for all tests - patch_background_task_decorator("adhocracy4.emails.tasks") - patch_background_task_decorator("apps.projects.tasks") - - # Populate reverse dict with organisation patterns - resolver = get_resolver() - resolver.reverse_dict - - register(factories.UserFactory) register(factories.UserFactory, "user2") register(factories.AdminFactory, "admin") @@ -48,6 +36,10 @@ def pytest_configure(config): register(AreaSettingsFactory) +def pytest_configure(): + Celery(task_always_eager=True) + + @pytest.fixture def apiclient(): return APIClient()