feat(datahub-gms): Enable autoscaling via HPA

.github/workflows/lint-test.yaml

@@ -1,9 +1,11 @@
 name: Lint and Test Charts
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
+      - 7onn/autoscale-gms
     paths-ignore:
       - '**.md'
       - 'LICENSE'

charts/datahub/Chart.yaml

@@ -4,13 +4,13 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.34
+version: 0.4.35
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.1
 dependencies:
   - name: datahub-gms
-    version: 0.2.175
+    version: 0.2.176
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.175
+version: 0.2.176
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.14.1

charts/datahub/subcharts/datahub-gms/README.md

@@ -2,7 +2,7 @@ datahub-gms
 ===========
 A Helm chart for DataHub's datahub-gms component
 
-Current chart version is `0.2.0`
+Current chart version is `0.2.176`
 
 ## Chart Values
 

charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl

@@ -74,3 +74,22 @@ Return the appropriate apiVersion for ingress.
 {{- print "networking.k8s.io/v1beta1" -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Return the appropriate apiVersion for HorizontalPodAutoscaler.
+*/}}
+{{- define "datahub-gms.hpa.apiVersion" -}}
+  {{- if and (.Capabilities.APIVersions.Has "autoscaling/v2") (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) -}}
+    {{- print "autoscaling/v2" -}}
+  {{- else -}}
+    {{- print "autoscaling/v2beta1" -}}
+  {{- end -}}
+{{- end -}}
+
+{{/*
+Create image registry, name and tag for a datahub component
+*/}}
+{{- define "datahub.image" -}}
+{{- $registry := .image.registry | default .imageRegistry -}}
+{{ $registry }}/{{ .image.repository }}:{{ required "Global or specific tag is required" (.image.tag | default .version) -}}
+{{- end -}}

charts/datahub/subcharts/datahub-gms/templates/hpa.yaml

@@ -0,0 +1,52 @@
+{{- if .Values.hpa.enabled }}
+{{- $apiVersion := include "datahub-gms.hpa.apiVersion" . -}}
+apiVersion: {{ $apiVersion }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "datahub-gms.fullname" . }}
+  labels:
+    {{- include "datahub-gms.labels" . | nindent 4 }}
+    {{- range $key, $val := .Values.extraLabels }}
+    {{ $key }}: {{ $val | quote }}
+    {{- end }}
+spec:
+  {{- if and (.Values.global.strict_mode) (.Values.hpa.enabled) (.Values.global.datahub.managed_ingestion.enabled) (not .Values.global.datahub_standalone_consumers_enabled) }}
+  {{- fail "\nHPA cannot be used in non standalone mode. Please enable standalone consumers if you wish to run multiple GMS instances and managed ingestion." }}
+  {{- else }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "datahub-gms.fullname" . }}
+  minReplicas: {{ .Values.hpa.minReplicas }}
+  maxReplicas: {{ .Values.hpa.maxReplicas }}
+  {{- with .Values.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  metrics:
+  {{- with .Values.hpa.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+  {{- with .Values.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if (eq $apiVersion "autoscaling/v2") }}
+        target:
+          type: Utilization
+          averageUtilization: {{ . }}
+        {{- else }}
+        targetAverageUtilization: {{ . }}
+        {{- end }}
+  {{- end }}
+{{- end }}
+{{- end }}

charts/datahub/subcharts/datahub-gms/values.yaml

@@ -4,6 +4,15 @@
 
 replicaCount: 1
 
+hpa:
+  # Can only be enabled if global.datahub_standalone_consumers_enabled
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 2
+  behavior: {}
+  targetCPUUtilizationPercentage: 100
+  targetMemoryUtilizationPercentage:
+
 revisionHistoryLimit: 10
 
 image:

charts/datahub/values.yaml

@@ -446,7 +446,7 @@ global:
   strict_mode: true
   graph_service_impl: elasticsearch
   datahub_analytics_enabled: true
-  datahub_standalone_consumers_enabled: false
+  datahub_standalone_consumers_enabled: true
   imageRegistry: "docker.io"
 
   elasticsearch: