diff --git a/Makefile b/Makefile index c908fd063..bd0c4e8fe 100644 --- a/Makefile +++ b/Makefile @@ -41,10 +41,13 @@ help: @echo " make po -- create new po files from the source" @echo " make mo -- create new mo files from the translated po files" @echo " make release -- build everything required for a release" - @echo " make start-postgres -- start the local postgres cluster" - @echo " make stop-postgres -- stops the local postgres cluster" - @echo " make create-postgres -- create the local postgres cluster (only works on ubuntu 20.04)" + @echo " make postgres-start -- start the local postgres cluster" + @echo " make postgres-stop -- stops the local postgres cluster" + @echo " make postgres-create -- create the local postgres cluster (only works on ubuntu 20.04)" @echo " make local-a4 -- patch to use local a4 (needs to have path ../adhocracy4)" + @echo " make celery-worker-start -- starts the celery worker in the foreground + @echo " make celery-worker-status -- lists all registered tasks and active worker nodes + @echo " make celery-worker-dummy-task -- calls the dummy task and prints result from redis @echo .PHONY: install @@ -174,16 +177,16 @@ release: $(VIRTUAL_ENV)/bin/python manage.py compilemessages -v0 $(VIRTUAL_ENV)/bin/python manage.py collectstatic --noinput -v0 -.PHONY: start-postgres -start-postgres: +.PHONY: postgres-start +postgres-start: sudo -u postgres PGDATA=pgsql PGPORT=5556 /usr/lib/postgresql/12/bin/pg_ctl start -.PHONY: stop-postgres -stop-postgres: +.PHONY: postgres-stop +postgres-stop: sudo -u postgres PGDATA=pgsql PGPORT=5556 /usr/lib/postgresql/12/bin/pg_ctl stop -.PHONY: create-postgres -create-postgres: +.PHONY: postgres-create +postgres-create: if [ -d "pgsql" ]; then \ echo "postgresql has already been initialized"; \ else \ @@ -201,3 +204,15 @@ local-a4: $(VIRTUAL_ENV)/bin/python manage.py migrate; \ npm link ../adhocracy4; \ fi + +.PHONY: celery-worker-start +celery-worker-start: + $(VIRTUAL_ENV)/bin/celery --app adhocracy-plus worker --loglevel INFO + +.PHONY: celery-worker-status +celery-worker-status: + $(VIRTUAL_ENV)/bin/celery --app adhocracy-plus inspect registered + +.PHONY: celery-worker-dummy-task +celery-worker-dummy-task: + $(VIRTUAL_ENV)/bin/celery --app adhocracy-plus call dummy_task | awk '{print "celery-task-meta-"$$0}' | xargs redis-cli get | python3 -m json.tool diff --git a/README.md b/README.md index 579fa0161..3633c0f48 100644 --- a/README.md +++ b/README.md @@ -11,43 +11,68 @@ adhocracy+ is designed to make online participation easy and accessible to every ## Installation for development -### Requirements: +### Requirements * nodejs (+ npm) * python 3.x (+ venv + pip) * libpq (only if postgres should be used) + * redis (only if celery is used) -### Installation: +### Installation git clone https://github.com/liqd/adhocracy-plus.git cd adhocracy-plus make install make fixtures -### Start virtual environment: +### Start virtual environment + source venv/bin/activate -### Check if tests work: +### Check if tests work make test -### Start a local server: +### Start a local server + make watch -### Use postgresql database for testing: +### Use postgresql database for testing + run the following command once: ``` -make create-postgres +make postgres-create ``` -to start the testserver with postgresql, run: +to start the test server with postgresql, run: ``` export DATABASE=postgresql -make start-postgres +make postgres-start make watch ``` Go to http://localhost:8004/ and login with admin@liqd.net | password +### Use Celery for task queues + +For a celery worker to pick up tasks you need to make sure that: +- the redis server is running +- the celery config parameter "always eager" is disabled (add `CELERY_TASK_ALWAYS_EAGER = False` to your `local.py`) + +To start a celery worker node in the foreground, call: +``` +make celery-worker-start +``` + +To inspect all registered tasks, list the running worker nodes, call: +``` +make celery-worker-status +``` + +To send a dummy task to the queue and report the result, call: +``` +make celery-worker-dummy-task +``` + ## Installation on a production system You like adhocracy+ and want to run your own version? An installation guide for production systems can be found [here](./docs/installation_prod.md). @@ -57,4 +82,5 @@ You like adhocracy+ and want to run your own version? An installation guide for If you found an issue, want to contribute, or would like to add your own features to your own version of adhocracy+, check out [contributing](./docs/contributing.md). ## Security + We care about security. So, if you find any issues concerning security, please send us an email at info [at] liqd [dot] net. diff --git a/adhocracy-plus/__init__.py b/adhocracy-plus/__init__.py index e69de29bb..9059e9fe1 100644 --- a/adhocracy-plus/__init__.py +++ b/adhocracy-plus/__init__.py @@ -0,0 +1,3 @@ +from .config.celery import celery_app + +__all__ = ("celery_app",) diff --git a/adhocracy-plus/config/celery.py b/adhocracy-plus/config/celery.py new file mode 100644 index 000000000..e920f3da8 --- /dev/null +++ b/adhocracy-plus/config/celery.py @@ -0,0 +1,30 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "adhocracy-plus.config.settings") + +celery_app = Celery() +celery_app.config_from_object("django.conf:settings", namespace="CELERY") +celery_app.autodiscover_tasks() + + +@celery_app.task(name="dummy_task") +def dummy_task(): + """ + This task is for testing purposes only. + """ + + result = "hello world" + print(result) + + return result + + +@celery_app.task(name="crash_task") +def crash_task(): + """ + This task is for testing purposes only. + """ + + 1 / 0 diff --git a/adhocracy-plus/config/settings/base.py b/adhocracy-plus/config/settings/base.py index c4cb16184..918427527 100644 --- a/adhocracy-plus/config/settings/base.py +++ b/adhocracy-plus/config/settings/base.py @@ -555,3 +555,9 @@ # Add insights for project if insight model exists INSIGHT_MODEL = "a4_candy_projects.ProjectInsight" + +# Celery configuration +CELERY_BROKER_URL = "redis://localhost:6379" +CELERY_RESULT_BACKEND = "redis://localhost:6379" +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True +CELERY_RESULT_EXTENDED = True diff --git a/adhocracy-plus/config/settings/dev.py b/adhocracy-plus/config/settings/dev.py index 21b3d7e14..4e2a8cf7d 100644 --- a/adhocracy-plus/config/settings/dev.py +++ b/adhocracy-plus/config/settings/dev.py @@ -19,23 +19,6 @@ INTERNAL_IPS = ("127.0.0.1", "localhost") -try: - from .local import * -except ImportError: - pass - -try: - INSTALLED_APPS += tuple(ADDITIONAL_APPS) -except NameError: - pass - -try: - CKEDITOR_CONFIGS["collapsible-image-editor"]["embed_provider"] = CKEDITOR_URL - CKEDITOR_CONFIGS["video-editor"]["embed_provider"] = CKEDITOR_URL -except NameError: - pass - - WAGTAILADMIN_BASE_URL = "http://localhost:8004" CAPTCHA_URL = "https://captcheck.netsyms.com/api.php" SITE_ID = 1 @@ -52,3 +35,27 @@ "OPTIONS": {}, } } + +CELERY_TASK_ALWAYS_EAGER = True + +# The local.py import happen at the end of this file so that it can overwrite +# any defaults in dev.py. +# Special cases are: +# 1) ADDITIONAL_APPS in local.py should be appended to INSTALLED_APPS. +# 2) CKEDITOR_URL should be inserted into CKEDITOR_CONFIGS in the correct location. + +try: + from .local import * +except ImportError: + pass + +try: + INSTALLED_APPS += tuple(ADDITIONAL_APPS) +except NameError: + pass + +try: + CKEDITOR_CONFIGS["collapsible-image-editor"]["embed_provider"] = CKEDITOR_URL + CKEDITOR_CONFIGS["video-editor"]["embed_provider"] = CKEDITOR_URL +except NameError: + pass diff --git a/apps/contrib/management/commands/errored_task_notification.py b/apps/contrib/management/commands/errored_task_notification.py deleted file mode 100644 index baff154a0..000000000 --- a/apps/contrib/management/commands/errored_task_notification.py +++ /dev/null @@ -1,21 +0,0 @@ -from background_task.models import CompletedTask -from django.core.management.base import BaseCommand -from django.urls import reverse - - -class Command(BaseCommand): - help = "Send notifications to inform admins about taks that errored" - - def handle(self, *args, **options): - broken_tasks = CompletedTask.objects.exclude(last_error="").order_by("-run_at") - - for task in broken_tasks: - url = reverse( - "admin:{}_{}_change".format( - task._meta.app_label, task._meta.model_name - ), - args=[task.id], - ) - self.stdout.write( - "Error in Task {}, see: {} \n".format(task.task_params, url) - ) diff --git a/apps/notifications/emails.py b/apps/notifications/emails.py index 228bc884f..60194a6aa 100644 --- a/apps/notifications/emails.py +++ b/apps/notifications/emails.py @@ -189,7 +189,7 @@ def send_no_object(cls, object, *args, **kwargs): ), "organisation_id": organisation.id, } - tasks.send_async_no_object( + tasks.send_async_no_object.delay( cls.__module__, cls.__name__, object_dict, args, kwargs ) return [] diff --git a/apps/projects/tasks.py b/apps/projects/tasks.py index 1d2647aa5..b13bd4393 100644 --- a/apps/projects/tasks.py +++ b/apps/projects/tasks.py @@ -1,10 +1,10 @@ import importlib -from background_task import background +from celery import shared_task -@background(schedule=1) +@shared_task def send_async_no_object(email_module_name, email_class_name, object, args, kwargs): - mod = importlib.import_module(email_module_name) - cls = getattr(mod, email_class_name) - return cls().dispatch(object, *args, **kwargs) + email_module = importlib.import_module(email_module_name) + email_class = getattr(email_module, email_class_name) + email_class().dispatch(object, *args, **kwargs) diff --git a/changelog/7601.md b/changelog/7601.md new file mode 100644 index 000000000..5bda85439 --- /dev/null +++ b/changelog/7601.md @@ -0,0 +1,4 @@ +## Added + +- adds support for celery task queues with a redis message broker +- adds makefile commands for starting and status checking of celery worker processes diff --git a/docs/celery.md b/docs/celery.md new file mode 100644 index 000000000..d806291cc --- /dev/null +++ b/docs/celery.md @@ -0,0 +1,53 @@ + +## Background + +We want to upgrade Django from the current version to at least 4. But our current approach to running background tasks, namely `django-background-tasks` is no longer supported in Django 4. Hence, we decided to switch to celery for distributed tasks. + + +## Developer Notes + +### configuration + +The celery configuration file is `adhocracy-plus/config/celery.py`. + +Currently, we make use of the following config parameters: +- `broker_url = "redis://localhost:6379"` +- `result_backend = "redis"` +- `broker_connection_retry_on_startup = True` +- `result_extended = True` + +The celery app is configured from the django settings file and namespaced variables. The defaults are defined in `config/settings/base.py` but can be overriden by `config/settings/local.py`. + +Note that the default settings in`dev.py` enable the "always eager mode" (`CELERY_TASK_ALWAYS_EAGER = True`) in which the django server will run the shared tasks itself. This is to keep the development setup as lightweight as possible. If you want to develop or test using a celery worker make sure that you add the following to your `local.py`: +``` +CELERY_TASK_ALWAYS_EAGER = False +``` + +### tasks + +Celery is set up to autodiscover tasks. To register a task import the shared task decorator from celery and apply it to your task function. + +```python +from celery import shared_task + +@shared_task +def add_two_numbers(): + return 1 + 1 +``` + +For testing purposes we have added a dummy task the prints and returns the string `"hello world"`. The dummy task can be called form the celery CLI via +``` +$ celery --app adhocracy-plus call dummy_task +b5351175-335d-4be0-b1fa-06278a613ccf +``` + + + +### makefile + +We added three makefile commands: + +- `celery-worker-start` to start a worker node in the foreground +- `celery-worker-status` to inspect registered tasks and running worker nodes +- `celery-worker-dummy-task` to call the dummy task + diff --git a/docs/installation_prod.md b/docs/installation_prod.md index 96e0e512c..aa0cf36d4 100644 --- a/docs/installation_prod.md +++ b/docs/installation_prod.md @@ -86,11 +86,16 @@ SECURE_CONTENT_TYPE_NOSNIFF = True SESSION_COOKIE_HTTPONLY = True FILE_UPLOAD_PERMISSIONS = 0o644 + +# celery configuration +CELERY_BROKER_URL = "redis+socket://var/run/redis/redis.sock" +CELERY_RESULT_BACKEND = "redis+socket://var/run/redis/redis.sock" +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True ``` #### Populate database -This will create all required tables via so called **migrations** +This will create all required tables via so-called **migrations** ``` python manage.py migrate @@ -117,7 +122,7 @@ Cancel the server after testing via `ctrl`+`c` ### Run the server as system daemon -In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create unit files. +In order to start up the software as a regular system daemon, similar to a database or webserver, we need to create service files. `/etc/systemd/system/adhocracy-plus.service`: @@ -139,26 +144,29 @@ StandardError=inherit WantedBy=default.target ``` -`/etc/systemd/system/adhocracy-plus-background-task.service`: +`/etc/systemd/system/adhocracy-plus-celery-worker.service`: ``` [Unit] -Description=adhocracy+ background task +Description=adhocracy+ celery worker After=network.target [Service] +Type=forking User=aplus WorkingDirectory=/home/aplus/adhocracy-plus -ExecStart=/home/aplus/.virtualenvs/aplus/bin/python manage.py process_tasks --settings adhocracy-plus.config.settings.production --sleep 5 +ExecStart=/home/aplus/.virtualenvs/aplus/bin/celery --app adhocracy-plus worker Restart=always RestartSec=3 -StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-background-task.log +StandardOutput=append:/var/log/adhocracy-plus/adhocracy-plus-celery-worker.log StandardError=inherit [Install] WantedBy=default.target ``` +Depending on your celery configuration you will also need to start a message broker service like redis or rabbit-mq and configure celery accordingly in `local.py` (see above). If you use redis and the default installation it should already be running, call `service redis status` to check. + This will log all output to files in `/var/log/adhocracy-plus/`. You will also need to create that folder before starting the service (as `root` or using `sudo`): ``` @@ -170,14 +178,14 @@ Load and start units (as `root` or using `sudo`): ``` systemctl daemon-reload systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` Enable autostart on boot: ``` systemctl enable adhocracy-plus -systemctl enable adhocracy-plus-background-task +systemctl enable adhocracy-plus-celery-worker ``` ### Setting up a proxy webserver @@ -254,7 +262,7 @@ The landing page is managed via [wagtail](https://wagtail.io/). You can find the ``` systemctl stop adhocracy-plus -systemctl stop adhocracy-plus-background-task +systemctl stop adhocracy-plus-celery-worker ``` #### Switch to user @@ -310,5 +318,5 @@ python manage.py runserver ``` systemctl start adhocracy-plus -systemctl start adhocracy-plus-background-task +systemctl start adhocracy-plus-celery-worker ``` diff --git a/package.json b/package.json index f9c2024b3..c14358f56 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "dependencies": { "@fortawesome/fontawesome-free": "5.15.4", "@maplibre/maplibre-gl-leaflet": "0.0.19", - "adhocracy4": "git+https://github.com/liqd/adhocracy4#8a5484d0b77db4bb168e5831c9026a9bddae3e14", + "adhocracy4": "git+https://github.com/liqd/adhocracy4#db03f3478931a39013fa50de183d5042a6cc284b", "autoprefixer": "10.4.14", "bootstrap": "5.2.3", "css-loader": "6.8.1", diff --git a/requirements/base.txt b/requirements/base.txt index a1eb1eceb..0f41dd49c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # A4 -git+https://github.com/liqd/adhocracy4.git@8a5484d0b77db4bb168e5831c9026a9bddae3e14#egg=adhocracy4 +git+https://github.com/liqd/adhocracy4.git@db03f3478931a39013fa50de183d5042a6cc284b#egg=adhocracy4 # Additional requirements brotli==1.0.9 @@ -26,3 +26,5 @@ easy-thumbnails[svg]==2.8.5 jsonfield==3.1.0 python-dateutil==2.8.2 rules==3.3 +celery==5.3.1 +redis==5.0.0 diff --git a/tests/conftest.py b/tests/conftest.py index 0f025320a..3e72ee5ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,30 +2,18 @@ import factory import pytest +from celery import Celery from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django.urls.base import get_resolver from PIL import Image from pytest_factoryboy import register from rest_framework.test import APIClient from adhocracy4.test import factories as a4_factories from adhocracy4.test.factories.maps import AreaSettingsFactory -from adhocracy4.test.helpers import patch_background_task_decorator from . import factories - -def pytest_configure(config): - # Patch email background_task decorators for all tests - patch_background_task_decorator("adhocracy4.emails.tasks") - patch_background_task_decorator("apps.projects.tasks") - - # Populate reverse dict with organisation patterns - resolver = get_resolver() - resolver.reverse_dict - - register(factories.UserFactory) register(factories.UserFactory, "user2") register(factories.AdminFactory, "admin") @@ -48,6 +36,10 @@ def pytest_configure(config): register(AreaSettingsFactory) +def pytest_configure(): + Celery(task_always_eager=True) + + @pytest.fixture def apiclient(): return APIClient()