From ab3d95ad8165633a605dbdb3c3240491c25c1e42 Mon Sep 17 00:00:00 2001 From: hklarner Date: Tue, 5 Sep 2023 15:55:31 +0200 Subject: [PATCH] pull request review changes --- Makefile | 14 +++++++++++ README.md | 4 +++ adhocracy-plus/__init__.py | 3 +++ adhocracy-plus/config/celery.py | 24 +++++++++--------- adhocracy-plus/config/settings/base.py | 6 ++++- apps/__init__.py | 2 -- changelog/7601.md | 2 +- docs/celery.md | 34 ++++++++++++++++++++------ requirements/base.txt | 2 -- 9 files changed, 64 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 03bf2a1638..6dd5a01ddc 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ help: @echo " make stop-postgres -- stops the local postgres cluster" @echo " make create-postgres -- create the local postgres cluster (only works on ubuntu 20.04)" @echo " make local-a4 -- patch to use local a4 (needs to have path ../adhocracy4)" + @echo " make celery-worker-start -- starts the celery worker in the background + @echo " make celery-worker-stop -- sends shutdown signal to celery worker + @echo " make celery-worker-status -- lists all registered tasks and active worker nodes @echo .PHONY: install @@ -201,3 +204,14 @@ local-a4: $(VIRTUAL_ENV)/bin/python manage.py migrate; \ npm link ../adhocracy4; \ fi + +.PHONY: celery-worker-start +celery-worker-start: + $(VIRTUAL_ENV)/bin/celery --app adhocracy-plus worker + + +.PHONY: celery-worker-status +celery-worker-status: + $(VIRTUAL_ENV)/bin/celery --app adhocracy-plus inspect registered + + diff --git a/README.md b/README.md index 579fa01613..33e31acb6b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ adhocracy+ is designed to make online participation easy and accessible to every * nodejs (+ npm) * python 3.x (+ venv + pip) * libpq (only if postgres should be used) + * redis (only if celery is used) ### Installation: @@ -32,9 +33,11 @@ adhocracy+ is designed to make online participation easy and accessible to every make test ### Start a local server: + make watch ### Use postgresql database for testing: + run the following command once: ``` make create-postgres @@ -57,4 +60,5 @@ You like adhocracy+ and want to run your own version? An installation guide for If you found an issue, want to contribute, or would like to add your own features to your own version of adhocracy+, check out [contributing](./docs/contributing.md). ## Security + We care about security. So, if you find any issues concerning security, please send us an email at info [at] liqd [dot] net. diff --git a/adhocracy-plus/__init__.py b/adhocracy-plus/__init__.py index e69de29bb2..9059e9fe1c 100644 --- a/adhocracy-plus/__init__.py +++ b/adhocracy-plus/__init__.py @@ -0,0 +1,3 @@ +from .config.celery import celery_app + +__all__ = ("celery_app",) diff --git a/adhocracy-plus/config/celery.py b/adhocracy-plus/config/celery.py index 221715acce..a963640c3a 100644 --- a/adhocracy-plus/config/celery.py +++ b/adhocracy-plus/config/celery.py @@ -4,20 +4,18 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "adhocracy-plus.config.settings") +celery_app = Celery() +celery_app.config_from_object("django.conf:settings", namespace="CELERY") +celery_app.autodiscover_tasks() -class Config: - broker_url = "redis://localhost:6379" - result_backend = "redis" - broker_connection_retry_on_startup = True +@celery_app.task +def dummy_task(): + """ + This task is for testing purposes only. + """ -celery = Celery(main="adhocracy-plus") -celery.config_from_object(Config) -celery.autodiscover_tasks() + result = "hello world" + print(result) - -@celery.task -def celery_dummy_task(): - print("hello") - - return "world" + return result diff --git a/adhocracy-plus/config/settings/base.py b/adhocracy-plus/config/settings/base.py index 16dc2204f5..db1a9e1a3c 100644 --- a/adhocracy-plus/config/settings/base.py +++ b/adhocracy-plus/config/settings/base.py @@ -111,7 +111,6 @@ "apps.polls", "apps.topicprio", "apps.debate", - # Celery "celery", ) @@ -557,3 +556,8 @@ # Add insights for project if insight model exists INSIGHT_MODEL = "a4_candy_projects.ProjectInsight" + +# Celery configuration +CELERY_BROKER_URL = "redis://localhost:6379" +CELERY_RESULT_BACKEND = "redis" +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True diff --git a/apps/__init__.py b/apps/__init__.py index e048e2eb60..eea436a379 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -1,5 +1,3 @@ -import importlib import logging logger = logging.getLogger(__name__) -celery = getattr(importlib.import_module("adhocracy-plus.config.celery"), "celery") diff --git a/changelog/7601.md b/changelog/7601.md index 98610d6ae7..5bda854398 100644 --- a/changelog/7601.md +++ b/changelog/7601.md @@ -1,4 +1,4 @@ ## Added - adds support for celery task queues with a redis message broker - +- adds makefile commands for starting and status checking of celery worker processes diff --git a/docs/celery.md b/docs/celery.md index a2ea276ff8..cc59817281 100644 --- a/docs/celery.md +++ b/docs/celery.md @@ -6,16 +6,34 @@ We want to upgrade Django from the current version to at least 4. But our curren ## Developer Notes -The celery configuration file is `adhocracy-plus/config/celery.py`. The celery app is identically configured for all environments and for simplicity we use a `Config` class inside `celery.py` instead of namespaced variables in Django settings files. +### configuration -Celery is set up to autodiscover tasks. To define a celery task simply import the celery app and use it to decorate the task function. Since the celery config file is located in a directory that uses a hyphen (`adhocracy-plus`) we use the `importlib` to import the app. For convenience, you can import celery from apps: +The celery configuration file is `adhocracy-plus/config/celery.py`. -```python -from apps import celery +Currently, we make use of only three config parameters: +- `broker_url = "redis://localhost:6379"` +- `result_backend = "redis"` +- `broker_connection_retry_on_startup = True` + +The celery app is configured from the django settings file and namespaced variables. The defaults are defined in `config/settings/base.py` but can be overriden by `config/settings/local.py`. + +### tasks -@celery.task -def celery_dummy_task(): - print("hello") +Celery is set up to autodiscover tasks. To register a task import the shared task decorator from celery and apply it to your task function. + +```python +from celery import shared_task - return "world" +@shared_task +def dummy_task(): + return "hello world" ``` + + +### makefile + +We added three makefile commands: + +- `celery-worker-start` to create worker processes +- `celery-worker-stop` to shut down worker processes +- `celery-worker-status` to inspect registered tasks and running worker nodes diff --git a/requirements/base.txt b/requirements/base.txt index 48cfdfee7c..4cb22908ba 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -26,7 +26,5 @@ easy-thumbnails[svg]==2.8.5 jsonfield==3.1.0 python-dateutil==2.8.2 rules==3.3 - -# celery celery==5.3.1 redis==5.0.0