From 1f2abfd63a1039063fea23361552eeb58a7217ee Mon Sep 17 00:00:00 2001 From: Jinlin Yang <86577891+jinlintt@users.noreply.github.com> Date: Wed, 20 Dec 2023 18:20:34 -0800 Subject: [PATCH] feat (datahub-actions): Expose executor id, gms protocol, and debug flag through values file. Update schema registry url (#410) * Expose executor id, gms protocol, debug flag through values file. Update schema registry url * Update readme and chart version * Provide a default value for debug.enabled --- charts/datahub/Chart.yaml | 4 ++-- charts/datahub/README.md | 2 ++ .../acryl-datahub-actions/Chart.yaml | 2 +- .../templates/_helpers.tpl | 21 +++++++++++++++++++ .../templates/deployment.yaml | 16 +++++++++++--- .../acryl-datahub-actions/values.yaml | 7 ++++++- charts/datahub/values.yaml | 1 + 7 files changed, 46 insertions(+), 7 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 01e1dc409..8206c59a4 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.3.19 +version: 0.3.20 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.12.1 @@ -30,7 +30,7 @@ dependencies: repository: file://./subcharts/datahub-ingestion-cron condition: datahub-ingestion-cron.enabled - name: acryl-datahub-actions - version: 0.2.141 + version: 0.2.142 repository: file://./subcharts/acryl-datahub-actions condition: acryl-datahub-actions.enabled maintainers: diff --git a/charts/datahub/README.md b/charts/datahub/README.md index 023eecda8..f923cbc8e 100644 --- a/charts/datahub/README.md +++ b/charts/datahub/README.md @@ -85,6 +85,8 @@ helm install datahub datahub/datahub --values <> | global.datahub_standalone_consumers_enabled | boolean | true | Enable standalone consumers for kafka | | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics | | global.datahub.appVersion | string | `"1.0"` | App version for annotation | +| global.datahub.gms.protocol | string | `"http"` | Protocol of GMS service | +| global.datahub.gms.host | string | `"datahub-datahub-gms" | Host of GMS service | | global.datahub.gms.port | string | `"8080"` | Port of GMS service | | global.datahub.monitoring.portName | string | `jmx` | Name of Kube port for monitoring | | global.elasticsearch.host | string | `"elasticsearch-master"` | Elasticsearch host name (endpoint) | diff --git a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml index 18272395d..952a8c159 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.141 +version: 0.2.142 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.0.11 diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/_helpers.tpl b/charts/datahub/subcharts/acryl-datahub-actions/templates/_helpers.tpl index 3ca075049..2e0ca72e2 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/_helpers.tpl +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/_helpers.tpl @@ -61,3 +61,24 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Datahub GMS protocol +*/}} +{{- define "acryl-datahub-actions.datahubGmsProtocol" -}} +{{ ((.Values.datahub).gms).protocol | default .Values.global.datahub.gms.protocol }} +{{- end -}} + +{{/* +Datahub GMS host +*/}} +{{- define "acryl-datahub-actions.datahubGmsHost" -}} +{{ (((.Values.datahub).gms).host | default ((.Values.global.datahub).gms).host) | default (printf "%s-%s" .Release.Name "datahub-gms") | trunc 63 | trimSuffix "-"}} +{{- end -}} + +{{/* +Datahub GMS port +*/}} +{{- define "acryl-datahub-actions.datahubGmsPort" -}} +{{ ((.Values.datahub).gms).port | default .Values.global.datahub.gms.port }} +{{- end -}} diff --git a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml index 37f4a622c..6ea1ce2a1 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml @@ -77,20 +77,30 @@ spec: args: {{ .Values.image.args | toJson }} {{- end }} env: + - name: DATAHUB_GMS_PROTOCOL + value: {{ include "acryl-datahub-actions.datahubGmsProtocol" . }} - name: DATAHUB_GMS_HOST - value: {{ (((.Values.datahub).gms).host | default ((.Values.global.datahub).gms).host) | default (printf "%s-%s" .Release.Name "datahub-gms") | trunc 63 | trimSuffix "-"}} + value: {{ include "acryl-datahub-actions.datahubGmsHost" . }} - name: DATAHUB_GMS_PORT - value: "{{ ((.Values.datahub).gms).port | default .Values.global.datahub.gms.port }}" + value: {{ include "acryl-datahub-actions.datahubGmsPort" . | quote }} # Deprecated in favour of DATAHUB_* variables - name: GMS_HOST value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} - name: GMS_PORT value: "{{ .Values.global.datahub.gms.port }}" + {{- if .Values.actions.executorId }} + - name: EXECUTOR_ID + value: "{{ .Values.actions.executorId }}" + {{- end }} + {{- if .Values.debug.enabled }} + - name: DATAHUB_DEBUG + value: "true" + {{- end }} - name: KAFKA_BOOTSTRAP_SERVER value: "{{ .Values.global.kafka.bootstrap.server }}" {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} - name: SCHEMA_REGISTRY_URL - value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }} + value: {{ printf "%s://%s:%s/schema-registry/api/" (include "acryl-datahub-actions.datahubGmsProtocol" .) (include "acryl-datahub-actions.datahubGmsHost" .) (include "acryl-datahub-actions.datahubGmsPort" .) | quote }} {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }} - name: SCHEMA_REGISTRY_URL value: "{{ .Values.global.kafka.schemaregistry.url }}" diff --git a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml index 65e1cdcaa..e2aca2f15 100644 --- a/charts/datahub/subcharts/acryl-datahub-actions/values.yaml +++ b/charts/datahub/subcharts/acryl-datahub-actions/values.yaml @@ -77,14 +77,19 @@ affinity: {} actions: kafkaAutoOffsetPolicy: "latest" + # Configure a custom executor id that will be set as the EXECUTOR_ID environment variable + # executorId: "" # mount the k8s secret as a volume in the container, each key name is mounted as a file on the mount path /etc/datahub/ingestion-secret-files # ingestionSecretFiles: # name: ${K8S_SECRET_NAME} # defaultMode: "0444" -global: +debug: + # Set enabled to true will set the DATAHUB_DEBUG env var to true + enabled: false +global: kafka: bootstrap: server: "broker:9092" diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 942c44689..25aaf0da3 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -580,6 +580,7 @@ global: datahub: version: v0.12.1 gms: + protocol: "http" port: "8080" nodePort: "30001"