[change] Added dedicated celery queue for firmware upgrader #273

Related to #273

README.md

@@ -762,35 +762,56 @@ Below are listed all the variables you can customize (you may also want to take
     openwisp2_redis_port: 6379
     openwisp2_redis_cache_url: "redis://{{ openwisp2_redis_host }}:{{ openwisp2_redis_port }}/1"
     # celery concurrency for the default queue, by default it is set to 1
-    # Setting it to "null" will make concurrency equal to number of CPUs
+    # Setting it to "null" will make concurrency equal to number of CPUs if autoscaling is not used
     openwisp2_celery_concurrency: null
-    # alternatively to the previous option, the celery autoscale option can be set if needed
+    # alternative to the previous option, the celery autoscale option can be set if needed
     # for more info, consult the documentation of celery regarding "autoscaling"
-    openwisp2_celery_autoscale: null
+    # by default it is set to "null" (no autoscaling)
+    openwisp2_celery_autoscale: 1,4
     # prefetch multiplier for the default queue,
     # the default value is calculated automatically by celery
     openwisp2_celery_prefetch_multiplier: null
     # celery queuing mode for the default queue,
     # leaving the default will work for most cases
     openwisp2_celery_optimization: default
-    # whether the dedicated worker for the celery network queue is enabled
+    # whether the dedicated worker for the celery "network" queue is enabled
     # must be turned on unless there's another server running a worker for this queue
     openwisp2_celery_network: true
-    # concurrency option for the network queue (a worker is dedicated solely to network operations)
-    # the default is 1. Setting it to "null" will make concurrency equal to number of CPUs
+    # concurrency option for the "network" queue (a worker is dedicated solely to network operations)
+    # the default is 1. Setting it to "null" will make concurrency equal to number of CPUs if autoscaling is not used.
     openwisp2_celery_network_concurrency: null
-    # alternatively to the previous option, the celery autoscale option can be set if needed
+    # alternative to the previous option, the celery autoscale option can be set if needed
     # for more info, consult the documentation of celery regarding "autoscaling"
+    # by default it is set to "null" (no autoscaling)
     openwisp2_celery_network_autoscale: 4,8
-    # prefetch multiplier for the network queue,
+    # prefetch multiplier for the "network" queue,
     # the default is 1, which mean no prefetching,
     # because the network tasks are long running and is better
     # to distribute the tasks to multiple processes
     openwisp2_celery_network_prefetch_multiplier: 1
-    # celery queuing mode for the network queue,
+    # celery queuing mode for the "network" queue,
     # fair mode is used in this case, which means
     # tasks will be equally distributed among workers
     openwisp2_celery_network_optimization: fair
+    # whether the dedicated worker for the celery "firmware_upgrader" queue is enabled
+    # must be turned on unless there's another server running a worker for this queue
+    openwisp2_celery_firmware_upgrader: true
+    # concurrency option for the "firmware_upgrader" queue (a worker is dedicated solely to firmware upgrade operations)
+    # the default is 1. Setting it to "null" will make concurrency equal to number of CPUs if autoscaling is not used
+    openwisp2_celery_firmware_upgrader_concurrency: null
+    # alternative to the previous option, the celery autoscale option can be set if needed
+    # for more info, consult the documentation of celery regarding "autoscaling"
+    # by default it is set to "null" (no autoscaling)
+    openwisp2_celery_firmware_upgrader_autoscale: 4,8
+    # prefetch multiplier for the "firmware_upgrader" queue,
+    # the default is 1, which mean no prefetching,
+    # because the firmware upgrade tasks are long running and is better
+    # to distribute the tasks to multiple processes
+    openwisp2_celery_firmware_upgrader_prefetch_multiplier: 1
+    # celery queuing mode for the "firmware_upgrader" queue,
+    # fair mode is used in this case, which means
+    # tasks will be equally distributed among workers
+    openwisp2_celery_firmware_upgrader_optimization: fair
     # whether the default celery task routes should be written to the settings.py file
     # turn this off if you're defining custom task routing rules
     openwisp2_celery_task_routes_defaults: true

defaults/main.yml

@@ -98,6 +98,11 @@ openwisp2_celery_network_concurrency: 1
 openwisp2_celery_network_autoscale: null
 openwisp2_celery_network_prefetch_multiplier: 1
 openwisp2_celery_network_optimization: fair
+openwisp2_celery_firmware_upgrader: true
+openwisp2_celery_firmware_upgrader_concurrency: 1
+openwisp2_celery_firmware_upgrader_autoscale: null
+openwisp2_celery_firmware_upgrader_prefetch_multiplier: 1
+openwisp2_celery_firmware_upgrader_optimization: fair
 openwisp2_celery_task_routes_defaults: true
 openwisp2_celery_broker_url: redis://{{ openwisp2_redis_host }}:{{ openwisp2_redis_port }}/3
 openwisp2_celery_task_acks_late: true

tasks/supervisor.yml

@@ -36,6 +36,14 @@
   when: openwisp2_celery_network
   notify: reload supervisor
 
+- name: supervisor celery firmware_upgrader
+  template:
+    src: supervisor/celery_firmware_upgrader.j2
+    dest: "{{ supervisor_path | format('celery_firmware_upgrader') }}"
+    mode: 0644
+  when: openwisp2_celery_firmware_upgrader
+  notify: reload supervisor
+
 - name: supervisor celerybeat
   template:
     src: supervisor/celerybeat.j2

templates/openwisp2/settings.py

@@ -169,11 +169,14 @@
 
 {% if openwisp2_celery_task_routes_defaults %}
 CELERY_TASK_ROUTES = {
+{% if openwisp2_celery_network %}
     # network operations, executed in the "network" queue
     'openwisp_controller.connection.tasks.*': {'queue': 'network'},
-{% if openwisp2_firmware_upgrader %}
-    'openwisp_firmware_upgrader.tasks.upgrade_firmware': {'queue': 'network'},
-    'openwisp_firmware_upgrader.tasks.batch_upgrade_operation': {'queue': 'network'},
+{% endif %}
+{% if openwisp2_firmware_upgrader and openwisp2_celery_firmware_upgrader %}
+    # firmware upgrade operations, executed in the "firmware_upgrader" queue
+    'openwisp_firmware_upgrader.tasks.upgrade_firmware': {'queue': 'firmware_upgrader'},
+    'openwisp_firmware_upgrader.tasks.batch_upgrade_operation': {'queue': 'firmware_upgrader'},
 {% endif %}
     # all other tasks are routed to the default queue (named "celery")
 }

templates/supervisor/celery_firmware_upgrader.j2

@@ -0,0 +1,11 @@
+[program:celery_firmware_upgrader]
+user={{ www_user }}
+directory={{ openwisp2_path }}
+command={{ openwisp2_path }}/env/bin/celery -A openwisp2 worker -l info --queue firmware_upgrader -O {{ openwisp2_celery_firmware_upgrader_optimization }}{% if openwisp2_celery_firmware_upgrader_concurrency %} --concurrency={{ openwisp2_celery_firmware_upgrader_concurrency }}{% elif openwisp2_celery_firmware_upgrader_autoscale %} --autoscale {{ openwisp2_celery_firmware_upgrader_autoscale }}{% endif %}{% if openwisp2_celery_firmware_upgrader_prefetch_multiplier %} --prefetch-multiplier {{ openwisp2_celery_firmware_upgrader_prefetch_multiplier }}{% endif %} -n firmware_upgrader@%%h
+autostart=true
+autorestart=true
+stopsignal=INT
+redirect_stderr=true
+stdout_logfile={{ openwisp2_path }}/log/celery-firmware-upgrader.log
+stdout_logfile_maxbytes=30MB
+stdout_logfile_backups=5