From 34440a035bc0e013a06340404b6d39f0ba9dd04d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:02:16 -0600 Subject: [PATCH] Support nonblocking system update mode (#432) * add logic to split system-update into blocking and nonblocking steps --- charts/datahub/Chart.yaml | 2 +- .../datahub-system-update-job.yml | 160 +++++++++++++++++- charts/datahub/values.yaml | 14 ++ 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml index 21e6cbd54..1e9819071 100644 --- a/charts/datahub/Chart.yaml +++ b/charts/datahub/Chart.yaml @@ -4,7 +4,7 @@ description: A Helm chart for LinkedIn DataHub type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.3.29 +version: 0.3.30 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. appVersion: 0.12.1 diff --git a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml index 72782c88a..df6976a78 100644 --- a/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml +++ b/charts/datahub/templates/datahub-upgrade/datahub-system-update-job.yml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-datahub-system-update-job + name: {{ .Release.Name }}-system-update labels: {{- include "datahub.labels" . | nindent 4 }} {{- with .Values.datahubSystemUpdate.annotations }} @@ -64,9 +64,14 @@ spec: args: {{ .Values.datahubSystemUpdate.image.args | toRawJson }} {{- else }} args: + {{- if .Values.datahubSystemUpdate.nonblocking.enabled }} + - "-u" + - "SystemUpdateBlocking" + {{- else }} - "-u" - "SystemUpdate" {{- end }} + {{- end }} env: - name: DATAHUB_REVISION value: {{ .Release.Revision | quote }} @@ -149,4 +154,157 @@ spec: tolerations: {{- toYaml . | nindent 12 }} {{- end }} +{{- if .Values.datahubSystemUpdate.nonblocking.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-system-update-nonblk + labels: + {{- include "datahub.labels" . | nindent 4 }} + {{- with .Values.datahubSystemUpdate.nonblocking.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + {{- if or .Values.global.podLabels .Values.datahubSystemUpdate.podAnnotations}} + metadata: + {{- with .Values.datahubSystemUpdate.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.global.podLabels }} + labels: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + spec: + {{- with .Values.global.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.datahubSystemUpdate.serviceAccount }} + serviceAccountName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- with .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + secret: + defaultMode: 0444 + secretName: {{ .name }} + {{- end }} + {{- with .Values.datahubSystemUpdate.extraVolumes }} + {{- toYaml . | nindent 8}} + {{- end }} + restartPolicy: Never + securityContext: + {{- toYaml .Values.datahubSystemUpdate.podSecurityContext | nindent 8 }} + {{- with .Values.datahubSystemUpdate.extraInitContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: datahub-system-update-job + image: "{{ .Values.datahubSystemUpdate.image.repository }}:{{ required "Global or specific tag is required" (.Values.datahubSystemUpdate.image.tag | default .Values.global.datahub.version) }}" + imagePullPolicy: {{ .Values.datahubSystemUpdate.image.pullPolicy | default "IfNotPresent" }} + {{- if .Values.datahubSystemUpdate.image.command }} + command: {{ .Values.datahubSystemUpdate.image.command | toRawJson }} + {{- end }} + {{- if .Values.datahubSystemUpdate.nonblocking.image.args }} + args: {{ .Values.datahubSystemUpdate.nonblocking.image.args | toRawJson }} + {{- else }} + args: + - "-u" + - "SystemUpdateNonBlocking" + {{- end }} + env: + - name: DATAHUB_REVISION + value: {{ .Release.Revision | quote }} + {{- include "datahub.upgrade.env" . | nindent 12}} + - name: DATAHUB_ANALYTICS_ENABLED + value: {{ .Values.global.datahub_analytics_enabled | quote }} + {{- if eq .Values.global.kafka.schemaregistry.type "INTERNAL" }} + - name: SCHEMA_REGISTRY_SYSTEM_UPDATE + value: "true" + - name: SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS + value: "true" + - name: SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION + value: "true" + {{- end }} + {{- with .Values.global.kafka.schemaregistry.type }} + - name: SCHEMA_REGISTRY_TYPE + value: "{{ . }}" + {{- end }} + {{- with .Values.global.kafka.schemaregistry.glue }} + - name: AWS_GLUE_SCHEMA_REGISTRY_REGION + value: "{{ .region }}" + {{- with .registry }} + - name: AWS_GLUE_SCHEMA_REGISTRY_NAME + value: "{{ . }}" + {{- end }} + {{- end }} + - name: ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES + value: {{ .Values.global.elasticsearch.index.upgrade.cloneIndices | quote }} + {{- with .Values.global.elasticsearch.index.enableMappingsReindex }} + - name: ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.index.enableSettingsReindex }} + - name: ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.index.settingsOverrides }} + - name: ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.index.entitySettingsOverrides }} + - name: ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.index.refreshIntervalSeconds }} + - name: ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS + value: {{ . | quote }} + {{- end }} + {{- with .Values.global.elasticsearch.index.upgrade.allowDocCountMismatch }} + - name: ELASTICSEARCH_BUILD_INDICES_ALLOW_DOC_COUNT_MISMATCH + value: {{ . | quote }} + {{- end }} + {{- with .Values.datahubSystemUpdate.extraEnvs }} + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.datahubSystemUpdate.securityContext | nindent 12 }} + volumeMounts: + {{- with .Values.global.credentialsAndCertsSecrets }} + - name: datahub-certs-dir + mountPath: {{ .path | default "/mnt/certs" }} + {{- end }} + {{- with .Values.datahubSystemUpdate.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.datahubSystemUpdate.resources | nindent 12 }} + {{- with .Values.datahubSystemUpdate.extraSidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with default .Values.global.nodeSelector .Values.datahubSystemUpdate.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.datahubSystemUpdate.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with default .Values.global.tolerations .Values.datahubSystemUpdate.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} {{- end -}} diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml index 82bb83ff6..7c9061c1d 100644 --- a/charts/datahub/values.yaml +++ b/charts/datahub/values.yaml @@ -368,6 +368,20 @@ datahubSystemUpdate: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "-4" helm.sh/hook-delete-policy: before-hook-creation + # !! Requires version v0.13.0 or greater + # Split the system update into 2 jobs, one that is blocking the rest of + # the deployment and the other which is non-blocking. Once the blocking + # steps are completed, the non-blocking job runs while the rest of the + # system is starting. + nonblocking: + enabled: false + # When mode = 'nonblocking' the nonblocking job should not include the above helm.sh/hook annotations + annotations: {} + image: + # Add custom command / arguments to this job. Useful if you need a custom startup or shutdown script + # to run + # command: customCommand + # args: [] podAnnotations: {} resources: limits: