From 58773fa52f7eaa378ee8e17e7db360a9f17f8842 Mon Sep 17 00:00:00 2001 From: Tom M G Date: Sun, 3 Nov 2024 15:32:34 +0100 Subject: [PATCH 1/5] feat(datahub-gms): Enable autoscaling via HPA --- charts/datahub/Chart.yaml | 4 +- .../datahub/subcharts/datahub-gms/Chart.yaml | 2 +- .../datahub/subcharts/datahub-gms/README.md | 2 +- .../datahub-gms/templates/_helpers.tpl | 11 ++++ .../subcharts/datahub-gms/templates/hpa.yaml | 52 +++++++++++++++++++ .../datahub/subcharts/datahub-gms/values.yaml | 9 ++++ charts/datahub/values.yaml | 4 +- 7 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 charts/datahub/subcharts/datahub-gms/templates/hpa.yaml diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index e3d11e913..5c8582b5e 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,13 +4,13 @@ description: A Helm chart for DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.4.34 +version: 0.4.35 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.14.1 dependencies: - name: datahub-gms - version: 0.2.175 + version: 0.2.176 repository: file://./subcharts/datahub-gms condition: datahub-gms.enabled - name: datahub-frontend diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml index fb9033768..89766c6b3 100644 --- a/charts/datahub/subcharts/datahub-gms/Chart.yaml +++ b/charts/datahub/subcharts/datahub-gms/Chart.yaml @@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.2.175 +version: 0.2.176 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: v0.14.1 diff --git a/charts/datahub/subcharts/datahub-gms/README.md b/charts/datahub/subcharts/datahub-gms/README.md index 7446f16fd..6129c7813 100644 --- a/charts/datahub/subcharts/datahub-gms/README.md +++ b/charts/datahub/subcharts/datahub-gms/README.md @@ -2,7 +2,7 @@ datahub-gms =========== A Helm chart for DataHub's datahub-gms component -Current chart version is `0.2.0` +Current chart version is `0.2.176` ## Chart Values diff --git a/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl b/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl index 8bdf26f22..0944430c1 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl +++ b/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl @@ -74,3 +74,14 @@ Return the appropriate apiVersion for ingress. {{- print "networking.k8s.io/v1beta1" -}} {{- end -}} {{- end -}} + +{{/* +Return the appropriate apiVersion for HorizontalPodAutoscaler. +*/}} +{{- define "datahub-gms.hpa.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "autoscaling/v2") (semverCompare ">=1.23-0" .Capabilities.KubeVersion.Version) -}} + {{- print "autoscaling/v2" -}} + {{- else -}} + {{- print "autoscaling/v2beta1" -}} + {{- end -}} +{{- end -}} diff --git a/charts/datahub/subcharts/datahub-gms/templates/hpa.yaml b/charts/datahub/subcharts/datahub-gms/templates/hpa.yaml new file mode 100644 index 000000000..02dc28a54 --- /dev/null +++ b/charts/datahub/subcharts/datahub-gms/templates/hpa.yaml @@ -0,0 +1,52 @@ +{{- if .Values.hpa.enabled }} +{{- $apiVersion := include "datahub-gms.hpa.apiVersion" . -}} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "datahub-gms.fullname" . }} + labels: + {{- include "datahub-gms.labels" . | nindent 4 }} + {{- range $key, $val := .Values.extraLabels }} + {{ $key }}: {{ $val | quote }} + {{- end }} +spec: + {{- if and (.Values.global.strict_mode) (.Values.hpa.enabled) (.Values.global.datahub.managed_ingestion.enabled) (not .Values.global.datahub_standalone_consumers_enabled) }} + {{- fail "\nHPA cannot be used in non standalone mode. Please enable standalone consumers if you wish to run multiple GMS instances and managed ingestion." }} + {{- else }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "datahub-gms.fullname" . }} + minReplicas: {{ .Values.hpa.minReplicas }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + {{- with .Values.hpa.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} + metrics: + {{- with .Values.hpa.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} + {{- with .Values.hpa.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if (eq $apiVersion "autoscaling/v2") }} + target: + type: Utilization + averageUtilization: {{ . }} + {{- else }} + targetAverageUtilization: {{ . }} + {{- end }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/datahub/subcharts/datahub-gms/values.yaml b/charts/datahub/subcharts/datahub-gms/values.yaml index 42fbf84b5..d29fa03d8 100644 --- a/charts/datahub/subcharts/datahub-gms/values.yaml +++ b/charts/datahub/subcharts/datahub-gms/values.yaml @@ -4,6 +4,15 @@ replicaCount: 1 +hpa: + # Can only be enabled if global.datahub_standalone_consumers_enabled + enabled: false + minReplicas: 1 + maxReplicas: 2 + behavior: {} + targetCPUUtilizationPercentage: 100 + targetMemoryUtilizationPercentage: + revisionHistoryLimit: 10 image: diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index b4c0c9e47..29855b1b5 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -2,6 +2,8 @@ # Copy this chart and change configuration as needed. datahub-gms: enabled: true + hpa: + enabled: true image: repository: acryldata/datahub-gms # tag: "v0.11.0 # defaults to .global.datahub.version @@ -446,7 +448,7 @@ global: strict_mode: true graph_service_impl: elasticsearch datahub_analytics_enabled: true - datahub_standalone_consumers_enabled: false + datahub_standalone_consumers_enabled: true imageRegistry: "docker.io" elasticsearch: From 4d88b8f37a6c77fc34866f3591e2c227348ff2ab Mon Sep 17 00:00:00 2001 From: Tom M G Date: Sun, 3 Nov 2024 15:35:44 +0100 Subject: [PATCH 2/5] trigger chart lint --- .github/workflows/lint-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index c4721be95..ce209a509 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -4,6 +4,7 @@ on: push: branches: - master + - 7onn/autoscale-gms paths-ignore: - '**.md' - 'LICENSE' From 1715d2b829ebdea58061674fdcfc079960d6bf3b Mon Sep 17 00:00:00 2001 From: Tom M G Date: Sun, 3 Nov 2024 15:36:31 +0100 Subject: [PATCH 3/5] trigger workflow --- .github/workflows/lint-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index ce209a509..f6ea1e134 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -1,6 +1,7 @@ name: Lint and Test Charts on: + workflow_dispatch: push: branches: - master From 0ba87ccf5cc371418ae172b15996491584592bc3 Mon Sep 17 00:00:00 2001 From: Tom M G Date: Sun, 3 Nov 2024 15:54:47 +0100 Subject: [PATCH 4/5] Add missing datahub.image template for datahub-gms --- .../datahub/subcharts/datahub-gms/templates/_helpers.tpl | 8 ++++++++ charts/datahub/values.yaml | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl b/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl index 0944430c1..5db1f0034 100644 --- a/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl +++ b/charts/datahub/subcharts/datahub-gms/templates/_helpers.tpl @@ -85,3 +85,11 @@ Return the appropriate apiVersion for HorizontalPodAutoscaler. {{- print "autoscaling/v2beta1" -}} {{- end -}} {{- end -}} + +{{/* +Create image registry, name and tag for a datahub component +*/}} +{{- define "datahub.image" -}} +{{- $registry := .image.registry | default .imageRegistry -}} +{{ $registry }}/{{ .image.repository }}:{{ required "Global or specific tag is required" (.image.tag | default .version) -}} +{{- end -}} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 29855b1b5..01d2defcb 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -2,8 +2,6 @@ # Copy this chart and change configuration as needed. datahub-gms: enabled: true - hpa: - enabled: true image: repository: acryldata/datahub-gms # tag: "v0.11.0 # defaults to .global.datahub.version From 779b0a9b2f7dff507d88fef8b2f6b160b8aaacca Mon Sep 17 00:00:00 2001 From: tom Date: Sun, 3 Nov 2024 16:06:34 +0100 Subject: [PATCH 5/5] Apply suggestions from code review --- .github/workflows/lint-test.yaml | 2 -- charts/datahub/values.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index f6ea1e134..c4721be95 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -1,11 +1,9 @@ name: Lint and Test Charts on: - workflow_dispatch: push: branches: - master - - 7onn/autoscale-gms paths-ignore: - '**.md' - 'LICENSE' diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 01d2defcb..b4c0c9e47 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -446,7 +446,7 @@ global: strict_mode: true graph_service_impl: elasticsearch datahub_analytics_enabled: true - datahub_standalone_consumers_enabled: true + datahub_standalone_consumers_enabled: false imageRegistry: "docker.io" elasticsearch: