[7601] Install and configure Celery

Makefile

@@ -41,10 +41,13 @@ help:
 	@echo "  make po						-- create new po files from the source"
 	@echo "  make mo						-- create new mo files from the translated po files"
 	@echo "  make release					-- build everything required for a release"
-	@echo "  make start-postgres			-- start the local postgres cluster"
-	@echo "  make stop-postgres				-- stops the local postgres cluster"
-	@echo "  make create-postgres			-- create the local postgres cluster (only works on ubuntu 20.04)"
+	@echo "  make postgres-start			-- start the local postgres cluster"
+	@echo "  make postgres-stop				-- stops the local postgres cluster"
+	@echo "  make postgres-create			-- create the local postgres cluster (only works on ubuntu 20.04)"
 	@echo "  make local-a4					-- patch to use local a4 (needs to have path ../adhocracy4)"
+	@echo "  make celery-worker-start		-- starts the celery worker in the foreground
+	@echo "  make celery-worker-status		-- lists all registered tasks and active worker nodes
+	@echo "  make celery-worker-dummy-task	-- calls the dummy task and prints result from redis
 	@echo
 
 .PHONY: install
@@ -174,16 +177,16 @@ release:
 	$(VIRTUAL_ENV)/bin/python manage.py compilemessages -v0
 	$(VIRTUAL_ENV)/bin/python manage.py collectstatic --noinput -v0
 
-.PHONY: start-postgres
-start-postgres:
+.PHONY: postgres-start
+postgres-start:
 	sudo -u postgres PGDATA=pgsql PGPORT=5556 /usr/lib/postgresql/12/bin/pg_ctl start
 
-.PHONY: stop-postgres
-stop-postgres:
+.PHONY: postgres-stop
+postgres-stop:
 	sudo -u postgres PGDATA=pgsql PGPORT=5556 /usr/lib/postgresql/12/bin/pg_ctl stop
 
-.PHONY: create-postgres
-create-postgres:
+.PHONY: postgres-create
+postgres-create:
 	if [ -d "pgsql" ]; then \
 		echo "postgresql has already been initialized"; \
 	else \
@@ -201,3 +204,15 @@ local-a4:
 		$(VIRTUAL_ENV)/bin/python manage.py migrate; \
 		npm link ../adhocracy4; \
 	fi
+
+.PHONY: celery-worker-start
+celery-worker-start:
+	$(VIRTUAL_ENV)/bin/celery --app adhocracy-plus worker --loglevel INFO
+
+.PHONY: celery-worker-status
+celery-worker-status:
+	$(VIRTUAL_ENV)/bin/celery --app adhocracy-plus inspect registered
+
+.PHONY: celery-worker-dummy-task
+celery-worker-dummy-task:
+	$(VIRTUAL_ENV)/bin/celery --app adhocracy-plus call dummy_task | awk '{print "celery-task-meta-"$$0}' | xargs redis-cli get | python3 -m json.tool

README.md

@@ -11,43 +11,68 @@ adhocracy+ is designed to make online participation easy and accessible to every
 
 ## Installation for development
 
-### Requirements:
+### Requirements
 
  * nodejs (+ npm)
  * python 3.x (+ venv + pip)
  * libpq (only if postgres should be used)
+ * redis (only if celery is used)
 
-### Installation:
+### Installation
 
     git clone https://github.com/liqd/adhocracy-plus.git
     cd adhocracy-plus
     make install
     make fixtures
 
-### Start virtual environment:
+### Start virtual environment
+
     source venv/bin/activate
 
-### Check if tests work:
+### Check if tests work
 
     make test
 
-### Start a local server:
+### Start a local server
+
     make watch
 
-### Use postgresql database for testing:
+### Use postgresql database for testing
+
 run the following command once:
 ```
-make create-postgres
+make postgres-create
 ```
-to start the testserver with postgresql, run:
+to start the test server with postgresql, run:
 ```
 export DATABASE=postgresql
-make start-postgres
+make postgres-start
 make watch
 ```
 
 Go to http://localhost:8004/ and login with [email protected] | password
 
+### Use Celery for task queues
+
+For a celery worker to pick up tasks you need to make sure that:
+- the redis server is running
+- the celery config parameter "always eager" is disabled (add `CELERY_TASK_ALWAYS_EAGER = False` to your `local.py`)
+
+To start a celery worker node in the foreground, call:
+```
+make celery-worker-start
+```
+
+To inspect all registered tasks, list the running worker nodes, call:
+```
+make celery-worker-status
+```
+
+To send a dummy task to the queue and report the result, call:
+```
+make celery-worker-dummy-task
+```
+
 ## Installation on a production system
 
 You like adhocracy+ and want to run your own version? An installation guide for production systems can be found [here](./docs/installation_prod.md).
@@ -57,4 +82,5 @@ You like adhocracy+ and want to run your own version? An installation guide for
 If you found an issue, want to contribute, or would like to add your own features to your own version of adhocracy+, check out [contributing](./docs/contributing.md).
 
 ## Security
+
 We care about security. So, if you find any issues concerning security, please send us an email at info [at] liqd [dot] net.

adhocracy-plus/__init__.py

@@ -0,0 +1,3 @@
+from .config.celery import celery_app
+
+__all__ = ("celery_app",)

adhocracy-plus/config/celery.py

@@ -0,0 +1,30 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "adhocracy-plus.config.settings")
+
+celery_app = Celery()
+celery_app.config_from_object("django.conf:settings", namespace="CELERY")
+celery_app.autodiscover_tasks()
+
+
+@celery_app.task(name="dummy_task")
+def dummy_task():
+    """
+    This task is for testing purposes only.
+    """
+
+    result = "hello world"
+    print(result)
+
+    return result
+
+
+@celery_app.task(name="crash_task")
+def crash_task():
+    """
+    This task is for testing purposes only.
+    """
+
+    1 / 0

adhocracy-plus/config/settings/base.py

@@ -555,3 +555,9 @@
 
 # Add insights for project if insight model exists
 INSIGHT_MODEL = "a4_candy_projects.ProjectInsight"
+
+# Celery configuration
+CELERY_BROKER_URL = "redis://localhost:6379"
+CELERY_RESULT_BACKEND = "redis://localhost:6379"
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
+CELERY_RESULT_EXTENDED = True

adhocracy-plus/config/settings/dev.py

@@ -19,23 +19,6 @@
 
     INTERNAL_IPS = ("127.0.0.1", "localhost")
 
-try:
-    from .local import *
-except ImportError:
-    pass
-
-try:
-    INSTALLED_APPS += tuple(ADDITIONAL_APPS)
-except NameError:
-    pass
-
-try:
-    CKEDITOR_CONFIGS["collapsible-image-editor"]["embed_provider"] = CKEDITOR_URL
-    CKEDITOR_CONFIGS["video-editor"]["embed_provider"] = CKEDITOR_URL
-except NameError:
-    pass
-
-
 WAGTAILADMIN_BASE_URL = "http://localhost:8004"
 CAPTCHA_URL = "https://captcheck.netsyms.com/api.php"
 SITE_ID = 1
@@ -52,3 +35,27 @@
             "OPTIONS": {},
         }
     }
+
+CELERY_TASK_ALWAYS_EAGER = True
+
+# The local.py import happen at the end of this file so that it can overwrite
+# any defaults in dev.py.
+# Special cases are:
+# 1) ADDITIONAL_APPS in local.py should be appended to INSTALLED_APPS.
+# 2) CKEDITOR_URL should be inserted into CKEDITOR_CONFIGS in the correct location.
+
+try:
+    from .local import *
+except ImportError:
+    pass
+
+try:
+    INSTALLED_APPS += tuple(ADDITIONAL_APPS)
+except NameError:
+    pass
+
+try:
+    CKEDITOR_CONFIGS["collapsible-image-editor"]["embed_provider"] = CKEDITOR_URL
+    CKEDITOR_CONFIGS["video-editor"]["embed_provider"] = CKEDITOR_URL
+except NameError:
+    pass

apps/notifications/emails.py

@@ -189,7 +189,7 @@ def send_no_object(cls, object, *args, **kwargs):
             ),
             "organisation_id": organisation.id,
         }
-        tasks.send_async_no_object(
+        tasks.send_async_no_object.delay(
             cls.__module__, cls.__name__, object_dict, args, kwargs
         )
         return []

apps/projects/tasks.py

@@ -1,10 +1,10 @@
 import importlib
 
-from background_task import background
+from celery import shared_task
 
 
-@background(schedule=1)
+@shared_task
 def send_async_no_object(email_module_name, email_class_name, object, args, kwargs):
-    mod = importlib.import_module(email_module_name)
-    cls = getattr(mod, email_class_name)
-    return cls().dispatch(object, *args, **kwargs)
+    email_module = importlib.import_module(email_module_name)
+    email_class = getattr(email_module, email_class_name)
+    email_class().dispatch(object, *args, **kwargs)

changelog/7601.md

@@ -0,0 +1,4 @@
+## Added
+
+- adds support for celery task queues with a redis message broker
+- adds makefile commands for starting and status checking of celery worker processes

docs/celery.md

@@ -0,0 +1,53 @@
+
+## Background
+
+We want to upgrade Django from the current version to at least 4. But our current approach to running background tasks, namely `django-background-tasks` is no longer supported in Django 4. Hence, we decided to switch to celery for distributed tasks.
+
+
+## Developer Notes
+
+### configuration
+
+The celery configuration file is `adhocracy-plus/config/celery.py`.
+
+Currently, we make use of the following config parameters:
+- `broker_url = "redis://localhost:6379"`
+- `result_backend = "redis"`
+- `broker_connection_retry_on_startup = True`
+- `result_extended = True`
+
+The celery app is configured from the django settings file and namespaced variables. The defaults are defined in `config/settings/base.py` but can be overriden by `config/settings/local.py`.
+
+Note that the default settings in`dev.py` enable the "always eager mode" (`CELERY_TASK_ALWAYS_EAGER = True`) in which the django server will run the shared tasks itself. This is to keep the development setup as lightweight as possible. If you want to develop or test using a celery worker make sure that you add the following to your `local.py`:
+```
+CELERY_TASK_ALWAYS_EAGER = False
+```
+
+### tasks
+
+Celery is set up to autodiscover tasks. To register a task import the shared task decorator from celery and apply it to your task function.
+
+```python
+from celery import shared_task
+
+@shared_task
+def add_two_numbers():
+    return 1 + 1
+```
+
+For testing purposes we have added a dummy task the prints and returns the string `"hello world"`. The dummy task can be called form the celery CLI via
+```
+$ celery --app adhocracy-plus call dummy_task
+b5351175-335d-4be0-b1fa-06278a613ccf
+```
+
+
+
+### makefile
+
+We added three makefile commands:
+
+- `celery-worker-start` to start a worker node in the foreground
+- `celery-worker-status` to inspect registered tasks and running worker nodes
+- `celery-worker-dummy-task` to call the dummy task
+