diff --git a/adhocracy-plus/config/celery.py b/adhocracy-plus/config/celery.py index 443b43812..e920f3da8 100644 --- a/adhocracy-plus/config/celery.py +++ b/adhocracy-plus/config/celery.py @@ -19,3 +19,12 @@ def dummy_task(): print(result) return result + + +@celery_app.task(name="crash_task") +def crash_task(): + """ + This task is for testing purposes only. + """ + + 1 / 0 diff --git a/adhocracy-plus/config/settings/base.py b/adhocracy-plus/config/settings/base.py index 2f8bb8b6b..57c20d5b3 100644 --- a/adhocracy-plus/config/settings/base.py +++ b/adhocracy-plus/config/settings/base.py @@ -36,7 +36,6 @@ "easy_thumbnails", "ckeditor", "ckeditor_uploader", - "background_task", "parler", # Wagtail cms components "wagtail.contrib.forms", diff --git a/apps/contrib/management/commands/errored_task_notification.py b/apps/contrib/management/commands/errored_task_notification.py deleted file mode 100644 index baff154a0..000000000 --- a/apps/contrib/management/commands/errored_task_notification.py +++ /dev/null @@ -1,21 +0,0 @@ -from background_task.models import CompletedTask -from django.core.management.base import BaseCommand -from django.urls import reverse - - -class Command(BaseCommand): - help = "Send notifications to inform admins about taks that errored" - - def handle(self, *args, **options): - broken_tasks = CompletedTask.objects.exclude(last_error="").order_by("-run_at") - - for task in broken_tasks: - url = reverse( - "admin:{}_{}_change".format( - task._meta.app_label, task._meta.model_name - ), - args=[task.id], - ) - self.stdout.write( - "Error in Task {}, see: {} \n".format(task.task_params, url) - ) diff --git a/apps/notifications/emails.py b/apps/notifications/emails.py index 228bc884f..60194a6aa 100644 --- a/apps/notifications/emails.py +++ b/apps/notifications/emails.py @@ -189,7 +189,7 @@ def send_no_object(cls, object, *args, **kwargs): ), "organisation_id": organisation.id, } - tasks.send_async_no_object( + tasks.send_async_no_object.delay( cls.__module__, cls.__name__, object_dict, args, kwargs ) return [] diff --git a/apps/projects/tasks.py b/apps/projects/tasks.py index 1d2647aa5..d8b77d2d6 100644 --- a/apps/projects/tasks.py +++ b/apps/projects/tasks.py @@ -1,9 +1,9 @@ import importlib -from background_task import background +from celery import shared_task -@background(schedule=1) +@shared_task def send_async_no_object(email_module_name, email_class_name, object, args, kwargs): mod = importlib.import_module(email_module_name) cls = getattr(mod, email_class_name) diff --git a/docs/installation_prod.md b/docs/installation_prod.md index 96e0e512c..2692f9048 100644 --- a/docs/installation_prod.md +++ b/docs/installation_prod.md @@ -86,11 +86,16 @@ SECURE_CONTENT_TYPE_NOSNIFF = True SESSION_COOKIE_HTTPONLY = True FILE_UPLOAD_PERMISSIONS = 0o644 + +# celery configuration +CELERY_BROKER_URL = "redis://localhost:6379" +CELERY_RESULT_BACKEND = "redis://localhost:6379" +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True ``` #### Populate database -This will create all required tables via so called **migrations** +This will create all required tables via so-called **migrations** ``` python manage.py migrate @@ -117,7 +122,7 @@ Cancel the server after testing via `ctrl`+`c` ### Run the server as system daemon -In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create unit files. +In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create service files. `/etc/systemd/system/adhocracy-plus.service`: @@ -139,26 +144,29 @@ StandardError=inherit WantedBy=default.target ``` -`/etc/systemd/system/adhocracy-plus-background-task.service`: +`/etc/systemd/system/adhocracy-plus-celery-worker.service`: ``` [Unit] -Description=adhocracy+ background task +Description=adhocracy+ celery worker After=network.target [Service] +Type=forking User=aplus WorkingDirectory=/home/aplus/adhocracy-plus -ExecStart=/home/aplus/.virtualenvs/aplus/bin/python manage.py process_tasks --settings adhocracy-plus.config.settings.production --sleep 5 +ExecStart=/home/aplus/.virtualenvs/aplus/bin/celery --app adhocracy-plus worker Restart=always RestartSec=3 -StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-background-task.log +StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-celery-worker.log StandardError=inherit [Install] WantedBy=default.target ``` +Depending on your celery configuration you will also need to start a message broker service like redis or rabbit-mq and configure celery accordingly in `local.py` (see above). If you use redis and the default installation it should already be running, call `service redis status` to check. + This will log all output to files in `/var/log/adhocracy-plus/`. You will also need to create that folder before starting the service (as `root` or using `sudo`): ``` @@ -170,14 +178,14 @@ Load and start units (as `root` or using `sudo`): ``` systemctl daemon-reload systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` Enable autostart on boot: ``` systemctl enable adhocracy-plus -systemctl enable adhocracy-plus-background-task +systemctl enable adhocracy-plus-celery-worker ``` ### Setting up a proxy webserver @@ -254,7 +262,7 @@ The landing page is managed via [wagtail](https://wagtail.io/). You can find the ``` systemctl stop adhocracy-plus -systemctl stop adhocracy-plus-background-task +systemctl stop adhocracy-plus-celery-worker ``` #### Switch to user @@ -310,5 +318,5 @@ python manage.py runserver ``` systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` diff --git a/requirements/base.txt b/requirements/base.txt index 46bf1f2fd..f9a79c920 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,7 +18,6 @@ redis==5.0.0 Django==3.2.19 django-allauth==0.54.0 git+https://github.com/liqd/django-autoslug.git@liqd2212#egg=django-autoslug -django-background-tasks==1.2.5 django-ckeditor==6.5.1 django-filter==22.1 django-widget-tweaks==1.4.12 diff --git a/tests/conftest.py b/tests/conftest.py index 0f025320a..3e72ee5ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,30 +2,18 @@ import factory import pytest +from celery import Celery from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django.urls.base import get_resolver from PIL import Image from pytest_factoryboy import register from rest_framework.test import APIClient from adhocracy4.test import factories as a4_factories from adhocracy4.test.factories.maps import AreaSettingsFactory -from adhocracy4.test.helpers import patch_background_task_decorator from . import factories - -def pytest_configure(config): - # Patch email background_task decorators for all tests - patch_background_task_decorator("adhocracy4.emails.tasks") - patch_background_task_decorator("apps.projects.tasks") - - # Populate reverse dict with organisation patterns - resolver = get_resolver() - resolver.reverse_dict - - register(factories.UserFactory) register(factories.UserFactory, "user2") register(factories.AdminFactory, "admin") @@ -48,6 +36,10 @@ def pytest_configure(config): register(AreaSettingsFactory) +def pytest_configure(): + Celery(task_always_eager=True) + + @pytest.fixture def apiclient(): return APIClient()