From eb3869cb236627d35ba9eab22bb9cffe95ba2239 Mon Sep 17 00:00:00 2001 From: Vinicius Aguiar Date: Fri, 2 Feb 2024 15:57:05 -0300 Subject: [PATCH] feat: add task worker deployment env variables (#556) --- .github/workflows/cd-dev.yaml | 17 ++++++++++++----- .github/workflows/cd-prod.yaml | 26 ++++++++++++++++++++------ .github/workflows/cd-staging.yaml | 17 ++++++++++++----- bd_api/apps/api/v1/tasks.py | 5 ++++- bd_api/utils.py | 19 +++++++++++++++++++ 5 files changed, 67 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cd-dev.yaml b/.github/workflows/cd-dev.yaml index 7fafa532..b53f493e 100644 --- a/.github/workflows/cd-dev.yaml +++ b/.github/workflows/cd-dev.yaml @@ -100,18 +100,20 @@ jobs: value: "https://development.basedosdados.org" - name: "BASE_URL_API" value: "https://development.api.basedosdados.org" - - name: "GOOGLE_DIRECTORY_SUBJECT" - value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" - - name: "GOOGLE_DIRECTORY_GROUP_KEY" - value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "LOGGER_LEVEL" value: "${{ secrets.LOGGER_LEVEL }}" - name: "LOGGER_IGNORE" value: "${{ secrets.LOGGER_IGNORE }}" - name: "LOGGER_SERIALIZE" value: "${{ secrets.LOGGER_SERIALIZE }}" + - name: "WORKER" + value: "main" - name: "REDIS_DB" value: "0" + - name: "GOOGLE_DIRECTORY_SUBJECT" + value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" + - name: "GOOGLE_DIRECTORY_GROUP_KEY" + value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "STRIPE_LIVE_MODE" value: "" - name: "STRIPE_LIVE_SECRET_KEY" @@ -148,4 +150,9 @@ jobs: - name: Deploy using Helm run: | - helm upgrade --install api-dev charts/basedosdados-api/. -n website -f values.yaml --wait + helm upgrade \ + --wait \ + --install \ + --namespace website \ + --values values.yaml \ + api-dev charts/basedosdados-api/. diff --git a/.github/workflows/cd-prod.yaml b/.github/workflows/cd-prod.yaml index 38044812..1d90a6e4 100644 --- a/.github/workflows/cd-prod.yaml +++ b/.github/workflows/cd-prod.yaml @@ -61,7 +61,7 @@ jobs: name: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}" tag: "stable" pullPolicy: "Always" - replicas: 2 + replicas: 1 resources: limits: cpu: 1500m @@ -74,18 +74,20 @@ jobs: value: "https://basedosdados.org" - name: "BASE_URL_API" value: "https://api.basedosdados.org" - - name: "GOOGLE_DIRECTORY_SUBJECT" - value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" - - name: "GOOGLE_DIRECTORY_GROUP_KEY" - value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "LOGGER_LEVEL" value: "${{ secrets.LOGGER_LEVEL }}" - name: "LOGGER_IGNORE" value: "${{ secrets.LOGGER_IGNORE }}" - name: "LOGGER_SERIALIZE" value: "${{ secrets.LOGGER_SERIALIZE }}" + - name: "WORKER" + value: "main" - name: "REDIS_DB" value: "0" + - name: "GOOGLE_DIRECTORY_SUBJECT" + value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" + - name: "GOOGLE_DIRECTORY_GROUP_KEY" + value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "STRIPE_LIVE_MODE" value: "True" - name: "STRIPE_LIVE_SECRET_KEY" @@ -122,4 +124,16 @@ jobs: - name: Deploy using Helm run: | - helm upgrade --install api-prod charts/basedosdados-api/. -n website -f values.yaml --wait + helm upgrade \ + --wait \ + --install \ + --namespace website \ + --values values.yaml \ + api-prod charts/basedosdados-api/. + helm upgrade \ + --wait \ + --install \ + --namespace website \ + --values values.yaml \ + --set "api.env[5].value=aux" \ + api-prod-aux charts/basedosdados-api/. diff --git a/.github/workflows/cd-staging.yaml b/.github/workflows/cd-staging.yaml index 8adedd56..47befe50 100644 --- a/.github/workflows/cd-staging.yaml +++ b/.github/workflows/cd-staging.yaml @@ -74,18 +74,20 @@ jobs: value: "https://staging.basedosdados.org" - name: "BASE_URL_API" value: "https://staging.api.basedosdados.org" - - name: "GOOGLE_DIRECTORY_SUBJECT" - value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" - - name: "GOOGLE_DIRECTORY_GROUP_KEY" - value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "LOGGER_LEVEL" value: "${{ secrets.LOGGER_LEVEL }}" - name: "LOGGER_IGNORE" value: "${{ secrets.LOGGER_IGNORE }}" - name: "LOGGER_SERIALIZE" value: "${{ secrets.LOGGER_SERIALIZE }}" + - name: "WORKER" + value: "main" - name: "REDIS_DB" value: "0" + - name: "GOOGLE_DIRECTORY_SUBJECT" + value: "${{ secrets.GOOGLE_DIRECTORY_SUBJECT }}" + - name: "GOOGLE_DIRECTORY_GROUP_KEY" + value: "${{ secrets.GOOGLE_DIRECTORY_GROUP_KEY }}" - name: "STRIPE_LIVE_MODE" value: "" - name: "STRIPE_LIVE_SECRET_KEY" @@ -122,4 +124,9 @@ jobs: - name: Deploy using Helm run: | - helm upgrade --install api-staging charts/basedosdados-api/. -n website -f values.yaml --wait + helm upgrade \ + --wait \ + --install \ + --namespace website \ + --values values.yaml \ + api-staging charts/basedosdados-api/. diff --git a/bd_api/apps/api/v1/tasks.py b/bd_api/apps/api/v1/tasks.py index 3c5719b6..ff5e04b7 100644 --- a/bd_api/apps/api/v1/tasks.py +++ b/bd_api/apps/api/v1/tasks.py @@ -9,7 +9,7 @@ from bd_api.apps.api.v1.models import Table from bd_api.custom.client import get_gbq_client, get_gcs_client, send_discord_message -from bd_api.utils import production_task +from bd_api.utils import main_task, production_task logger = logger.bind(module="api.v1") @@ -20,6 +20,7 @@ @periodic_task(crontab(day_of_week="0", hour="3", minute="0")) +@main_task @production_task def update_table_metadata_task(table_pks: list[str] = None): """Update the metadata of selected tables in the database""" @@ -112,12 +113,14 @@ def format_msg(msg: list[str]) -> str: @periodic_task(crontab(day_of_week="1-6", hour="5", minute="0")) +@main_task @production_task def update_search_index_task(): call_command("update_index", batchsize=100) @periodic_task(crontab(day_of_week="0", hour="5", minute="0")) +@main_task @production_task def rebuild_search_index_task(): call_command("rebuild_index", interactive=False, batchsize=100) diff --git a/bd_api/utils.py b/bd_api/utils.py index bced05a3..71b49f20 100644 --- a/bd_api/utils.py +++ b/bd_api/utils.py @@ -2,6 +2,7 @@ from functools import wraps from os import getenv +WORKER = getenv("WORKER", "aux") API_URL = getenv("BASE_URL_API", "https://localhost:8080") SETTINGS = getenv("DJANGO_SETTINGS_MODULE", "bd_api.settings") @@ -13,6 +14,13 @@ def is_remote(): return False +def is_main(): + """Check if it is main environment""" + if "main" in WORKER: + return True + return False + + def is_dev(): """Check if it is remote development environment""" if is_remote() and "development" in API_URL: @@ -58,6 +66,17 @@ def get_frontend_url(): return "localhost:3000" +def main_task(func): + """Decorator that avoids function call if it isn't main""" + + @wraps(func) + def wrapper(*args, **kwargs): + if is_main(): + return func(*args, **kwargs) + + return wrapper + + def production_task(func): """Decorator that avoids function call if it isn't production"""