From 1177f32f76e0f63b7b1af37cee05f60106162632 Mon Sep 17 00:00:00 2001 From: michaeljguarino Date: Fri, 6 Oct 2023 14:36:02 -0400 Subject: [PATCH] Make kubernetes executor default for airflow This is just an overall simpler architecture (but more resource intensive), so probably the saner default. Also removes complexity of an internal redis. --- airflow/helm/airflow/Chart.yaml | 2 +- airflow/helm/airflow/values.yaml | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/airflow/helm/airflow/Chart.yaml b/airflow/helm/airflow/Chart.yaml index 16c320859..ec6c4a246 100644 --- a/airflow/helm/airflow/Chart.yaml +++ b/airflow/helm/airflow/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: airflow description: A Helm chart for airflow deployable on plural type: application -version: 0.3.39 +version: 0.3.40 appVersion: "2.6.3" sources: - https://github.com/pluralsh/plural-artifacts/airflow/helm/airflow diff --git a/airflow/helm/airflow/values.yaml b/airflow/helm/airflow/values.yaml index 1ac7c7bf9..aa17574d9 100644 --- a/airflow/helm/airflow/values.yaml +++ b/airflow/helm/airflow/values.yaml @@ -12,7 +12,7 @@ httpConfig: password: dummy secrets: - createPluralRedisSecret: false + createPluralRedisSecret: true redis_password: example test-base: @@ -257,7 +257,7 @@ airflow: flower: enabled: false - + rbac: create: true @@ -268,7 +268,7 @@ airflow: image: registry: gcr.io/pluralsh tag: 6.0.9-debian-10-r0 - enabled: true + enabled: false existingSecret: airflow-redis-password existingSecretPasswordKey: redis-password @@ -310,11 +310,20 @@ airflow: passwordSecretKey: redis-password airflow: - executor: CeleryExecutor + executor: KubernetesExecutor image: repository: ghcr.io/pluralsh/containers/apache/airflow tag: 2.6.3-python3.10-plural1.4.4 + kubernetesPodTemplate: + resources: + requests: + cpu: 100m + memory: 500Mi + limits: + cpu: 500m + memory: 1.5Gi + config: ## security AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" @@ -383,7 +392,7 @@ airflow: workers: ## if the airflow workers StatefulSet should be deployed ## - enabled: true + enabled: false replicas: 2