Merge branch 'master' into fix-mce-consumer-group-id

acryldata · Nov 29, 2024 · 37b56b8 · 37b56b8
2 parents d719742 + 2a0dc8c
commit 37b56b8
Show file tree

Hide file tree

Showing 11 changed files with 115 additions and 8 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,13 +4,13 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.37
+version: 0.4.39
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1
 dependencies:
   - name: datahub-gms
-    version: 0.2.175
+    version: 0.2.176
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend

diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.175
+version: 0.2.176
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1
diff --git a/charts/datahub/subcharts/datahub-gms/README.md b/charts/datahub/subcharts/datahub-gms/README.md
@@ -2,7 +2,7 @@ datahub-gms
 ===========
 A Helm chart for DataHub's datahub-gms component
 
-Current chart version is `0.2.0`
+Current chart version is `0.2.176`
 
 ## Chart Values
 

diff --git a/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl b/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl
@@ -74,3 +74,22 @@ Return the appropriate apiVersion for ingress.
 {{- print "networking.k8s.io/v1beta1" -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Return the appropriate apiVersion for HorizontalPodAutoscaler.
+*/}}
+{{- define "datahub-gms.hpa.apiVersion" -}}
+  {{- if and (.Capabilities.APIVersions.Has "autoscaling/v2") (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) -}}
+    {{- print "autoscaling/v2" -}}
+  {{- else -}}
+    {{- print "autoscaling/v2beta1" -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+Create image registry, name and tag for a datahub component
+*/}}
+{{- define "datahub.image" -}}
+{{- $registry := .image.registry | default .imageRegistry -}}
+{{ $registry }}/{{ .image.repository }}:{{ required "Global or specific tag is required" (.image.tag | default .version) -}}
+{{- end -}}
diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml
@@ -506,6 +506,12 @@ spec:
             - name: FORMS_HOOK_CONSUMER_GROUP_SUFFIX
               value: "{{ .consumerGroupSuffix }}"
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-gms/templates/hpa.yaml b/charts/datahub/subcharts/datahub-gms/templates/hpa.yaml
@@ -0,0 +1,52 @@
+{{- if .Values.hpa.enabled }}
+{{- $apiVersion := include "datahub-gms.hpa.apiVersion" . -}}
+apiVersion: {{ $apiVersion }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "datahub-gms.fullname" . }}
+  labels:
+    {{- include "datahub-gms.labels" . | nindent 4 }}
+    {{- range $key, $val := .Values.extraLabels }}
+    {{ $key }}: {{ $val | quote }}
+    {{- end }}
+spec:
+  {{- if and (.Values.global.strict_mode) (.Values.hpa.enabled) (.Values.global.datahub.managed_ingestion.enabled) (not .Values.global.datahub_standalone_consumers_enabled) }}
+  {{- fail "\nHPA cannot be used in non standalone mode. Please enable standalone consumers if you wish to run multiple GMS instances and managed ingestion." }}
+  {{- else }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "datahub-gms.fullname" . }}
+  minReplicas: {{ .Values.hpa.minReplicas }}
+  maxReplicas: {{ .Values.hpa.maxReplicas }}
+  {{- with .Values.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  metrics:
+  {{- with .Values.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+  {{- with .Values.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}
diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml
@@ -4,6 +4,15 @@
 
 replicaCount: 1
 
+hpa:
+  # Can only be enabled if global.datahub_standalone_consumers_enabled
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 2
+  behavior: {}
+  targetCPUUtilizationPercentage: 100
+  targetMemoryUtilizationPercentage:
+
 revisionHistoryLimit: 10
 
 image:
@@ -256,6 +265,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     cache:
       search:
         ## Enable general search caching
@@ -275,6 +287,11 @@ global:
           ttlSeconds: 86400
           lightningThreshold: 300
 
+    metadataChangeProposal:
+      consumer:
+        batch:
+          enabled: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml
@@ -341,6 +341,12 @@ spec:
             {{- end }}
             {{- end }}
             {{- end }}
+            {{- with .Values.global.datahub.strictUrnValidation }}
+            - name: STRICT_URN_VALIDATION_ENABLED
+              value: "{{ . }}"
+            {{- end }}
+            - name: MCP_CONSUMER_BATCH_ENABLED
+              value: {{ .Values.global.datahub.metadataChangeProposal.consumer.batch.enabled | quote }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/values.yaml b/charts/datahub/subcharts/datahub-mce-consumer/values.yaml
@@ -184,6 +184,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml
@@ -151,7 +151,7 @@ spec:
               value: {{ $result | toJson | quote }}
             {{- with $v.revision_env }}
             - name: {{ . }}
-              value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
             {{- end }}
             {{- end }}
             {{- end }}
@@ -334,7 +334,7 @@ spec:
               value: {{ $result | toJson | quote }}
             {{- with $v.revision_env }}
             - name: {{ . }}
-              value: '{"version":"{{ $.Chart.Version }}-{{ $.Release.Revision }}"}'
+              value: '{"version":"{{ $.Values.global.datahub.version }}-{{ $result | toJson | sha256sum | trunc 7 }}"}'
             {{- end }}
             {{- end }}
             {{- end }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -415,15 +415,16 @@ datahubSystemUpdate:
       values:
         ingestion:
           name: datahub-gc
-          # override global.datahub.managed_ingestion.defaultCliVersion
-          version: "0.14.1.7rc2"
+          # # override global.datahub.managed_ingestion.defaultCliVersion
+          # version: "0.14.1.7rc2"
         # schedule:
           # # override global.datahub.timezone
           # timezone: "UTC"
           # # override dailyCronWindow
           # interval: "0 1 * * *"
         cleanup_expired_tokens: "false"
         truncate_indices: "true"
+        truncate_indices_retention_days: 30
         dataprocess_cleanup:
           retention_days: 30
           delete_empty_data_jobs: "true"
@@ -819,6 +820,9 @@ global:
     ## Enables diff mode for graph writes, uses a different code path that produces a diff from previous to next to write relationships instead of wholesale deleting edges and reading
     enableGraphDiffMode: true
 
+    ## Enable stricter URN validation logic
+    strictUrnValidation: false
+
     ## Values specific to the unified search and browse feature.
     search_and_browse:
       show_search_v2: true  # If on, show the new search filters experience as of v0.10.5