feat (datahub-actions): Expose executor id, gms protocol, and debug flag through values file. Update schema registry url

charts/datahub/Chart.yaml

@@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.3.19
+version: 0.3.20
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.12.1
@@ -30,7 +30,7 @@ dependencies:
     repository: file://./subcharts/datahub-ingestion-cron
     condition: datahub-ingestion-cron.enabled
   - name: acryl-datahub-actions
-    version: 0.2.141
+    version: 0.2.142
     repository: file://./subcharts/acryl-datahub-actions
     condition: acryl-datahub-actions.enabled
 maintainers:

charts/datahub/README.md

@@ -85,6 +85,8 @@ helm install datahub datahub/datahub --values <<path-to-values-file>>
 | global.datahub_standalone_consumers_enabled                    | boolean | true                                                                                             | Enable standalone consumers for kafka                                                                                                                   |
 | global.datahub_analytics_enabled                               | boolean | true                                                                                             | Enable datahub usage analytics                                                                                                                          |
 | global.datahub.appVersion                                      | string  | `"1.0"`                                                                                          | App version for annotation                                                                                                                              |
+| global.datahub.gms.protocol                                    | string  | `"http"`                                                                                         | Protocol of GMS service                                                                                                                                 |
+| global.datahub.gms.host                                        | string  | `"datahub-datahub-gms"                                                                           | Host of GMS service                                                                                                                                     |
 | global.datahub.gms.port                                        | string  | `"8080"`                                                                                         | Port of GMS service                                                                                                                                     |
 | global.datahub.monitoring.portName                             | string  | `jmx`                                                                                            | Name of Kube port for monitoring                                                                                                                        |
 | global.elasticsearch.host                                      | string  | `"elasticsearch-master"`                                                                         | Elasticsearch host name (endpoint)                                                                                                                      |

charts/datahub/subcharts/acryl-datahub-actions/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.141
+version: 0.2.142
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.0.11

charts/datahub/subcharts/acryl-datahub-actions/templates/_helpers.tpl

@@ -61,3 +61,24 @@ Create the name of the service account to use
     {{ default "default" .Values.serviceAccount.name }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Datahub GMS protocol
+*/}}
+{{- define "acryl-datahub-actions.datahubGmsProtocol" -}}
+{{ ((.Values.datahub).gms).protocol | default .Values.global.datahub.gms.protocol }}
+{{- end -}}
+
+{{/*
+Datahub GMS host
+*/}}
+{{- define "acryl-datahub-actions.datahubGmsHost" -}}
+{{ (((.Values.datahub).gms).host | default ((.Values.global.datahub).gms).host) | default (printf "%s-%s" .Release.Name "datahub-gms") | trunc 63 | trimSuffix "-"}}
+{{- end -}}
+
+{{/*
+Datahub GMS port
+*/}}
+{{- define "acryl-datahub-actions.datahubGmsPort" -}}
+{{ ((.Values.datahub).gms).port | default .Values.global.datahub.gms.port }}
+{{- end -}}

charts/datahub/subcharts/acryl-datahub-actions/templates/deployment.yaml

@@ -77,20 +77,30 @@ spec:
           args: {{ .Values.image.args | toJson }}
           {{- end }}
           env:
+            - name: DATAHUB_GMS_PROTOCOL
+              value: {{ include "acryl-datahub-actions.datahubGmsProtocol" . }}
             - name: DATAHUB_GMS_HOST
-              value: {{ (((.Values.datahub).gms).host | default ((.Values.global.datahub).gms).host) | default (printf "%s-%s" .Release.Name "datahub-gms") | trunc 63 | trimSuffix "-"}}
+              value: {{ include "acryl-datahub-actions.datahubGmsHost" . }}
             - name: DATAHUB_GMS_PORT
-              value: "{{ ((.Values.datahub).gms).port | default .Values.global.datahub.gms.port }}"
+              value: {{ include "acryl-datahub-actions.datahubGmsPort" . | quote }}
             # Deprecated in favour of DATAHUB_* variables
             - name: GMS_HOST
               value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
             - name: GMS_PORT
               value: "{{ .Values.global.datahub.gms.port }}"
+            {{- if .Values.actions.executorId }}
+            - name: EXECUTOR_ID
+              value: "{{ .Values.actions.executorId }}"
+            {{- end }}
+            {{- if .Values.debug.enabled }}
+            - name: DATAHUB_DEBUG
+              value: "true"
+            {{- end }}
             - name: KAFKA_BOOTSTRAP_SERVER
               value: "{{ .Values.global.kafka.bootstrap.server }}"
             {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }}
             - name: SCHEMA_REGISTRY_URL
-              value: {{ printf "http://%s-%s:%s/schema-registry/api/" .Release.Name "datahub-gms" .Values.global.datahub.gms.port }}
+              value: {{ printf "%s://%s:%s/schema-registry/api/" (include "acryl-datahub-actions.datahubGmsProtocol" .) (include "acryl-datahub-actions.datahubGmsHost" .) (include "acryl-datahub-actions.datahubGmsPort" .) | quote }}
             {{- else if eq .Values.global.kafka.schemaregistry.type "KAFKA" }}
             - name: SCHEMA_REGISTRY_URL
               value: "{{ .Values.global.kafka.schemaregistry.url }}"

charts/datahub/subcharts/acryl-datahub-actions/values.yaml

@@ -77,14 +77,19 @@ affinity: {}
 
 actions:
   kafkaAutoOffsetPolicy: "latest"
+  # Configure a custom executor id that will be set as the EXECUTOR_ID environment variable
+  # executorId: ""
 
 # mount the k8s secret as a volume in the container, each key name is mounted as a file on the mount path /etc/datahub/ingestion-secret-files
 # ingestionSecretFiles:
 #   name: ${K8S_SECRET_NAME}
 #   defaultMode: "0444"
 
-global:
+debug:
+  # Set enabled to true will set the DATAHUB_DEBUG env var to true
+  enabled: false
 
+global:
   kafka:
     bootstrap:
       server: "broker:9092"

charts/datahub/values.yaml

@@ -580,6 +580,7 @@ global:
   datahub:
     version: v0.12.1
     gms:
+      protocol: "http"
       port: "8080"
       nodePort: "30001"