pull request review changes

Makefile

@@ -45,6 +45,9 @@ help:
 	@echo "  make stop-postgres				-- stops the local postgres cluster"
 	@echo "  make create-postgres			-- create the local postgres cluster (only works on ubuntu 20.04)"
 	@echo "  make local-a4					-- patch to use local a4 (needs to have path ../adhocracy4)"
+	@echo "  make celery-worker-start		-- starts the celery worker in the background
+	@echo "  make celery-worker-stop		-- sends shutdown signal to celery worker
+	@echo "  make celery-worker-status		-- lists all registered tasks and active worker nodes
 	@echo
 
 .PHONY: install
@@ -201,3 +204,14 @@ local-a4:
 		$(VIRTUAL_ENV)/bin/python manage.py migrate; \
 		npm link ../adhocracy4; \
 	fi
+
+.PHONY: celery-worker-start
+celery-worker-start:
+	$(VIRTUAL_ENV)/bin/celery --app adhocracy-plus worker
+
+
+.PHONY: celery-worker-status
+celery-worker-status:
+	$(VIRTUAL_ENV)/bin/celery --app adhocracy-plus inspect registered
+
+

README.md

@@ -16,6 +16,7 @@ adhocracy+ is designed to make online participation easy and accessible to every
  * nodejs (+ npm)
  * python 3.x (+ venv + pip)
  * libpq (only if postgres should be used)
+ * redis (only if celery is used)
 
 ### Installation:
 
@@ -32,9 +33,11 @@ adhocracy+ is designed to make online participation easy and accessible to every
     make test
 
 ### Start a local server:
+
     make watch
 
 ### Use postgresql database for testing:
+
 run the following command once:
 ```
 make create-postgres
@@ -57,4 +60,5 @@ You like adhocracy+ and want to run your own version? An installation guide for
 If you found an issue, want to contribute, or would like to add your own features to your own version of adhocracy+, check out [contributing](./docs/contributing.md).
 
 ## Security
+
 We care about security. So, if you find any issues concerning security, please send us an email at info [at] liqd [dot] net.

adhocracy-plus/__init__.py

@@ -0,0 +1,3 @@
+from .config.celery import celery_app
+
+__all__ = ("celery_app",)

adhocracy-plus/config/celery.py

@@ -4,20 +4,18 @@
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "adhocracy-plus.config.settings")
 
+celery_app = Celery()
+celery_app.config_from_object("django.conf:settings", namespace="CELERY")
+celery_app.autodiscover_tasks()
 
-class Config:
-    broker_url = "redis://localhost:6379"
-    result_backend = "redis"
-    broker_connection_retry_on_startup = True
 
+@celery_app.task
+def dummy_task():
+    """
+    This task is for testing purposes only.
+    """
 
-celery = Celery(main="adhocracy-plus")
-celery.config_from_object(Config)
-celery.autodiscover_tasks()
+    result = "hello world"
+    print(result)
 
-
-@celery.task
-def celery_dummy_task():
-    print("hello")
-
-    return "world"
+    return result

adhocracy-plus/config/settings/base.py

@@ -111,7 +111,6 @@
     "apps.polls",
     "apps.topicprio",
     "apps.debate",
-    # Celery
     "celery",
 )
 
@@ -557,3 +556,8 @@
 
 # Add insights for project if insight model exists
 INSIGHT_MODEL = "a4_candy_projects.ProjectInsight"
+
+# Celery configuration
+CELERY_BROKER_URL = "redis://localhost:6379"
+CELERY_RESULT_BACKEND = "redis"
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True

apps/__init__.py

@@ -1,5 +1,3 @@
-import importlib
 import logging
 
 logger = logging.getLogger(__name__)
-celery = getattr(importlib.import_module("adhocracy-plus.config.celery"), "celery")

changelog/7601.md

@@ -1,4 +1,4 @@
 ## Added
 
 - adds support for celery task queues with a redis message broker
-
+- adds makefile commands for starting and status checking of celery worker processes

docs/celery.md

@@ -6,16 +6,34 @@ We want to upgrade Django from the current version to at least 4. But our curren
 
 ## Developer Notes
 
-The celery configuration file is `adhocracy-plus/config/celery.py`. The celery app is identically configured for all environments and for simplicity we use a `Config` class inside `celery.py` instead of namespaced variables in Django settings files.
+### configuration
 
-Celery is set up to autodiscover tasks. To define a celery task simply import the celery app and use it to decorate the task function. Since the celery config file is located in a directory that uses a hyphen (`adhocracy-plus`) we use the `importlib` to import the app. For convenience, you can import celery from apps:
+The celery configuration file is `adhocracy-plus/config/celery.py`.
 
-```python
-from apps import celery
+Currently, we make use of only three config parameters:
+- `broker_url = "redis://localhost:6379"`
+- `result_backend = "redis"`
+- `broker_connection_retry_on_startup = True`
+
+The celery app is configured from the django settings file and namespaced variables. The defaults are defined in `config/settings/base.py` but can be overriden by `config/settings/local.py`.
+
+### tasks
 
-@celery.task
-def celery_dummy_task():
-    print("hello")
+Celery is set up to autodiscover tasks. To register a task import the shared task decorator from celery and apply it to your task function.
+
+```python
+from celery import shared_task
 
-    return "world"
+@shared_task
+def dummy_task():
+    return "hello world"
 ```
+
+
+### makefile
+
+We added three makefile commands:
+
+- `celery-worker-start` to create worker processes
+- `celery-worker-stop` to shut down worker processes
+- `celery-worker-status` to inspect registered tasks and running worker nodes

requirements/base.txt

@@ -26,7 +26,5 @@ easy-thumbnails[svg]==2.8.5
 jsonfield==3.1.0
 python-dateutil==2.8.2
 rules==3.3
-
-# celery
 celery==5.3.1
 redis==5.0.0