Merge pull request packit#91 from jpopelka/worker

[deployment] Celery worker

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -1,4 +1,7 @@
 *.egg-info
 tests/data/acl/SOURCES
+
 deployment/roles/deploy/files/secrets
-deployment/roles/deploy/vars/main.yml
+
+deployment/roles/deploy/vars/*.yml
+!deployment/roles/deploy/vars/local.yml

Makefile

@@ -7,15 +7,11 @@ PACKAGE ?= rpm
 BRANCH ?= c8s
 DIR ?= git.centos.org
 IMAGE_NAME := dist2src
+IMAGE_WORKER_NAME ?= docker.io/usercont/dist2src-worker
 CONTAINER_ENGINE ?= $(shell command -v podman 2> /dev/null || echo docker)
 CONTAINER_CMD ?= /bin/bash
 TEST_TARGET ?= ./tests
 COLOR ?= yes
-ANSIBLE_PYTHON := /usr/bin/python3
-AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter=$(ANSIBLE_PYTHON)
-# "By default, Ansible runs as if --tags all had been specified."
-# https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html#special-tags
-TAGS ?= all
 
 usage:
 	@echo "Run 'make convert' to run the convert or 'make clean' to clean up things."
@@ -73,4 +69,7 @@ check-in-container:
 # example:
 # DEPLOYMENT=local make deploy TAGS=worker
 deploy:
-	$(AP) deployment/deploy.yml --tags $(TAGS)
+	make -C deployment deploy
+
+worker:
+	$(CONTAINER_ENGINE) build -t $(IMAGE_WORKER_NAME) -f files/Containerfile.worker .

deployment/Makefile

@@ -0,0 +1,10 @@
+ANSIBLE_PYTHON := /usr/bin/python3
+AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter=$(ANSIBLE_PYTHON)
+# "By default, Ansible runs as if --tags all had been specified."
+# https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html#special-tags
+TAGS ?= all
+
+# example:
+# DEPLOYMENT=local make deploy TAGS=worker
+deploy:
+	$(AP) deploy.yml --tags $(TAGS)

deployment/README.md

@@ -2,6 +2,8 @@
 
 See [roles/deploy/README.md](roles/deploy/README.md) for instructions, short version:
 
-- create (or symlink) [roles/deploy/files/secrets/](roles/deploy/files/secrets)
-- define `api_key` in [roles/deploy/vars/main.yml](roles/deploy/vars/main.yml) (.gitignore-d) and run `make deploy`
-- OR define `api_key` in [roles/deploy/vars/local.yml](roles/deploy/vars/local.yml) and run `DEPLOYMENT=local make deploy`
+- create (or symlink) `roles/deploy/files/secrets/{myenvironment}`
+- define `api_key` in `roles/deploy/vars/{myenvironment}.yml`
+- run `DEPLOYMENT={myenvironment} make deploy`
+
+Where `{myenvironment}` might be for example 'prod' or 'local'.

deployment/docker-compose.yml

@@ -0,0 +1,68 @@
+version: "2"
+
+services:
+  redis:
+    image: registry.fedoraproject.org/f31/redis
+    container_name: redis
+    ports:
+      - 6379:6379
+    user: "123123"
+
+  redis-commander:
+    container_name: redis-commander
+    hostname: redis-commander
+    image: rediscommander/redis-commander:latest
+    environment:
+      REDIS_HOST: redis
+      REDIS_PORT: "6379"
+    ports:
+      - 8081:8081
+    depends_on:
+      - redis
+    user: "123123"
+
+  flower:
+    image: mher/flower
+    container_name: flower
+    ports:
+      - 5555:5555
+    depends_on:
+      - redis
+    environment:
+      FLOWER_DEBUG: "True"
+      CELERY_BROKER_URL: redis://redis:6379/0
+    user: "123123"
+
+  worker:
+    container_name: worker
+    build:
+      context: ..
+      dockerfile: files/Containerfile.worker
+    image: docker.io/usercont/dist2src-worker
+    command: /usr/bin/run_worker.sh
+    tty: true
+    depends_on:
+      - redis
+    environment:
+      DEPLOYMENT: dev
+      APP: dist2src.worker.tasks
+      CELERY_TASK_NAME: task.dist2src.process_message
+    volumes:
+      - ../dist2src/worker:/usr/local/lib/python3.7/site-packages/dist2src/worker:ro,z
+    user: "123123"
+
+  centosmsg:
+    container_name: centosmsg
+    image: docker.io/usercont/packit-service-centosmsg:prod
+    depends_on:
+      - redis
+    environment:
+      DEPLOYMENT: dev
+      LOG_LEVEL: DEBUG
+      REDIS_SERVICE_HOST: redis
+      CELERY_TASK_NAME: task.dist2src.process_message
+      #DISABLE_SENDING_CELERY_TASKS: disable
+    volumes:
+      - ./roles/deploy/files/secrets/dev/centos-server-ca.cert:/secrets/centos-server-ca.cert:ro,Z
+      - ./roles/deploy/files/secrets/dev/centos.cert:/secrets/centos.cert:ro,Z
+    user: "123123"

deployment/roles/deploy/README.md

@@ -5,11 +5,12 @@ This role deploys dist2src-update service to Openshift.
 ## Role Variables
 
 Default variables are set in [defaults/main.yml](defaults/main.yml).
-At least `api_key` needs to be re-defined in a yaml file in [vars/](vars/).
-There's for example [vars/local.yml](vars/local.yml) which will be used if you set
-`DEPLOYMENT=local` environment variable.
-If you don't specify `DEPLOYMENT`, [vars/main.yml](vars/main.yml) (which is .gitignore-d)
-will be used by default.
+At least `api_key` needs to be re-defined in a yaml file in [vars/](vars).
+If you're for example deploying to prod environment, create `vars/prod.yml`,
+set `api_key` there and run `DEPLOYMENT=prod make deploy`.
+Don't worry, all `vars/*.yml` files are .gitignore-d.
+There's also [vars/local.yml](vars/local.yml), which will be used
+if you run `DEPLOYMENT=local make deploy`.
 
 ## Secrets
 

deployment/roles/deploy/defaults/main.yml

@@ -21,5 +21,8 @@ image_centosmsg: docker.io/usercont/packit-service-centosmsg:prod
 # How many workers
 worker_replicas: 1
 
+# sent by centosmsg pod and processed by worker(s)
+celery_task_name: task.dist2src.process_message
+
 container_engine: "{{ lookup('pipe', 'command -v podman 2> /dev/null || echo docker') }}"
-deployment: "{{ lookup('env', 'DEPLOYMENT') or 'main' }}"
+deployment: "{{ lookup('env', 'DEPLOYMENT') }}"

deployment/roles/deploy/files/redis-dc.yml

@@ -29,6 +29,9 @@ spec:
             limits:
               memory: "80Mi"
               cpu: "100m"
+          envFrom:
+            - configMapRef:
+                name: env-config
       volumes:
         - name: redis-pv
           persistentVolumeClaim:

deployment/roles/deploy/tasks/deploy-configmap.yml

@@ -0,0 +1,15 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+---
+- name: Deploy ConfigMap
+  k8s:
+    namespace: "{{ project }}"
+    resource_definition: "{{ item }}"
+    host: "{{ host }}"
+    api_key: "{{ api_key }}"
+    validate_certs: "{{ validate_certs }}"
+  with_template:
+    - configmap.yml.j2
+  tags:
+    - always

deployment/roles/deploy/tasks/main.yml

@@ -2,11 +2,15 @@
 # SPDX-License-Identifier: MIT
 
 ---
+- fail: msg="DEPLOYMENT is not set"
+  when: deployment|length == 0
 - include_vars: "{{ deployment }}.yml"
+
 - include_tasks: ./check-api-token.yml
 - include_tasks: ./push-dev-images.yml
   when: push_dev_images
 - include_tasks: ./deploy-secrets.yml
+- include_tasks: ./deploy-configmap.yml
 - include_tasks: ./deploy-redis.yml
 - include_tasks: ./deploy-worker.yml
 - include_tasks: ./deploy-centosmsg.yml

deployment/roles/deploy/templates/centos-secrets.yml.j2

@@ -8,5 +8,5 @@ metadata:
   name: centos-secrets
 type: Opaque
 data:
-  centos-server-ca.cert: "{{ lookup('file', 'secrets/centos-server-ca.cert') | b64encode }}"
-  centos.cert: "{{ lookup('file', 'secrets/centos.cert') | b64encode }}"
+  centos-server-ca.cert: "{{ lookup('file', 'secrets/{{ deployment }}/centos-server-ca.cert') | b64encode }}"
+  centos.cert: "{{ lookup('file', 'secrets/{{ deployment }}/centos.cert') | b64encode }}"

deployment/roles/deploy/templates/centosmsg-dc.yml.j2

@@ -25,19 +25,17 @@ spec:
           image: dist2src-centosmsg
           env:
             - name: MQTT_HOST
-              value: mqtt.git.centos.org
+              value: {{ "mqtt.git.centos.org" if deployment == "prod" else "mqtt.stg.centos.org" }}
             - name: MQTT_TOPICS
-              value: git.centos.org/#
+              value: {{ "git.centos.org/#" if deployment == "prod" else "git.stg.centos.org/#" }}
             - name: MQTT_SUBTOPICS
               value: git.receive
-            - name: REDIS_SERVICE_HOST
-              value: redis
-            # change when implementation is done
-            - name: CELERY_TASK_NAME
-              value: ""
             # temporary, delete when workers are configured and implementation is done
             - name: DISABLE_SENDING_CELERY_TASKS
               value: disable
+          envFrom:
+            - configMapRef:
+                name: env-config
           volumeMounts:
             - name: centos-secrets
               mountPath: /secrets

deployment/roles/deploy/templates/configmap.yml.j2

@@ -0,0 +1,14 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+# https://docs.openshift.com/container-platform/4.5/builds/builds-configmaps.html
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: env-config
+type: Opaque
+data:
+  DEPLOYMENT: "{{ deployment }}"
+  CELERY_TASK_NAME: "{{ celery_task_name }}"
+  REDIS_SERVICE_HOST: redis

deployment/roles/deploy/templates/secret-sentry.yml.j2

@@ -8,4 +8,4 @@ metadata:
   name: sentry
 type: Opaque
 data:
-  dsn: "{{ lookup('file', 'secrets/sentry_dsn') | b64encode }}"
+  dsn: "{{ lookup('file', 'secrets/{{ deployment }}/sentry_dsn') | b64encode }}"

deployment/roles/deploy/templates/worker-sts.yml.j2

@@ -38,9 +38,16 @@ spec:
           image: dist2src-worker:latest
           env:
             - name: APP
-              value: dist2src_service.worker.tasks
-          command:
-            - "/usr/bin/run_worker.sh"
+              value: dist2src.worker.tasks
+            - name: SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: sentry
+                  key: dsn
+          envFrom:
+            - configMapRef:
+                name: env-config
+          command: ["/usr/bin/run_worker.sh"]
           resources:
             limits:
               memory: "384Mi"

dist2src/worker/__init__.py

@@ -0,0 +1,2 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT

dist2src/worker/celerizer.py

@@ -0,0 +1,33 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+from os import getenv
+
+from celery import Celery
+from dist2src.worker.sentry import configure_sentry
+from lazy_object_proxy import Proxy
+
+
+class Celerizer:
+    def __init__(self):
+        self._celery_app = None
+
+    @property
+    def celery_app(self):
+        if self._celery_app is None:
+            host = getenv("REDIS_SERVICE_HOST", "redis")
+            password = getenv("REDIS_PASSWORD", "")
+            port = getenv("REDIS_SERVICE_PORT", "6379")
+            db = getenv("REDIS_SERVICE_DB", "0")
+            broker_url = f"redis://:{password}@{host}:{port}/{db}"
+
+            self._celery_app = Celery(broker=broker_url)
+        return self._celery_app
+
+
+def get_celery_application():
+    configure_sentry(runner_type="worker")
+    return Celerizer().celery_app
+
+
+celery_app: Celery = Proxy(get_celery_application)

dist2src/worker/decorators.py

@@ -0,0 +1,28 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+from logging import getLogger
+
+logger = getLogger(__name__)
+
+
+class only_once(object):
+    """
+    Use as a function decorator to run function only once.
+    """
+
+    def __init__(self, func):
+        self.func = func
+        self.configured = False
+
+    def __call__(self, *args, **kwargs):
+        if self.configured:
+            logger.debug(f"Function {self.func.__name__} already called. Skipping.")
+            return
+
+        self.configured = True
+        logger.debug(
+            f"Function {self.func.__name__} called for the first time with "
+            f"args: {args} and kwargs: {kwargs}"
+        )
+        return self.func(*args, **kwargs)

dist2src/worker/processor.py

@@ -0,0 +1,13 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+from logging import getLogger
+from typing import Optional
+
+logger = getLogger(__name__)
+
+
+class Processor:
+    def process_message(self, event: dict, **kwargs) -> Optional[dict]:
+        logger.info(f"Processing message with {event}")
+        return None

dist2src/worker/sentry.py

@@ -0,0 +1,30 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+from logging import getLogger
+from os import getenv
+
+from dist2src.worker.decorators import only_once
+
+logger = getLogger(__name__)
+
+
+@only_once
+def configure_sentry(runner_type: str) -> None:
+    logger.debug("Setting up Sentry")
+
+    if not getenv("SENTRY_DSN") or not getenv("DEPLOYMENT"):
+        logger.warning("$SENTRY_DSN or $DEPLOYMENT not set")
+        return
+
+    # so that we don't have to have sentry sdk installed locally
+    from sentry_sdk import init, configure_scope
+    from sentry_sdk.integrations.celery import CeleryIntegration
+
+    init(
+        dsn=getenv("SENTRY_DSN"),
+        integrations=[CeleryIntegration()],
+        environment=getenv("DEPLOYMENT"),
+    )
+    with configure_scope() as scope:
+        scope.set_tag("runner-type", runner_type)

dist2src/worker/tasks.py

@@ -0,0 +1,13 @@
+# Copyright Contributors to the Packit project.
+# SPDX-License-Identifier: MIT
+
+from os import getenv
+from typing import Optional
+
+from dist2src.worker.celerizer import celery_app
+from dist2src.worker.processor import Processor
+
+
+@celery_app.task(name=getenv("CELERY_TASK_NAME"))
+def process_message(event: dict, **kwargs) -> Optional[dict]:
+    return Processor().process_message(event=event)