feat(mce): add mce throttling options (#477)

acryldata · Jun 20, 2024 · 41306bd · 41306bd
1 parent 6e6fa86
commit 41306bd
Show file tree

Hide file tree

Showing 6 changed files with 136 additions and 5 deletions.
diff --git a/charts/datahub/Chart.yaml b/charts/datahub/Chart.yaml
@@ -4,13 +4,13 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.17
+version: 0.4.18
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.13.2
 dependencies:
   - name: datahub-gms
-    version: 0.2.169
+    version: 0.2.170
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
@@ -22,7 +22,7 @@ dependencies:
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer
-    version: 0.2.161
+    version: 0.2.162
     repository: file://./subcharts/datahub-mce-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-ingestion-cron

diff --git a/charts/datahub/subcharts/datahub-gms/Chart.yaml b/charts/datahub/subcharts/datahub-gms/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for LinkedIn DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.169
+version: 0.2.170
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.13.1
diff --git a/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml b/charts/datahub/subcharts/datahub-gms/templates/deployment.yaml
@@ -392,6 +392,60 @@ spec:
               value: {{ .Values.global.datahub.enableGraphDiffMode | quote }}
             - name: GRAPHQL_QUERY_INTROSPECTION_ENABLED
               value: {{ .Values.global.datahub.gms.graphql.introspection.enabled | quote }}
+            {{- with .Values.global.datahub.mcp.throttle }}
+            {{- with .updateIntervalMs }}
+            - name: MCP_THROTTLE_UPDATE_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- if .versioned.enabled }}
+            - name: MCP_VERSIONED_THROTTLE_ENABLED
+              value: 'true'
+            {{- with .versioned.threshold }}
+            - name: MCP_VERSIONED_THRESHOLD
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.maxAttempts }}
+            - name: MCP_VERSIONED_MAX_ATTEMPTS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.initialIntervalMs }}
+            - name: MCP_VERSIONED_INITIAL_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.multiplier }}
+            - name: MCP_VERSIONED_MULTIPLIER
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.maxIntervalMs }}
+            - name: MCP_VERSIONED_MAX_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- end }}
+            {{- if .timeseries.enabled }}
+            - name: MCP_TIMESERIES_THROTTLE_ENABLED
+              value: 'true'
+            {{- with .timeseries.threshold }}
+            - name: MCP_TIMESERIES_THRESHOLD
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.maxAttempts }}
+            - name: MCP_TIMESERIES_MAX_ATTEMPTS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.initialIntervalMs }}
+            - name: MCP_TIMESERIES_INITIAL_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.multiplier }}
+            - name: MCP_TIMESERIES_MULTIPLIER
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.maxIntervalMs }}
+            - name: MCP_TIMESERIES_MAX_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml b/charts/datahub/subcharts/datahub-mce-consumer/Chart.yaml
@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.161
+version: 0.2.162
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.13.1
diff --git a/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml b/charts/datahub/subcharts/datahub-mce-consumer/templates/deployment.yaml
@@ -270,6 +270,60 @@ spec:
               value: {{ .Values.global.datahub.alwaysEmitChangeLog | quote }}
             - name: GRAPH_SERVICE_DIFF_MODE_ENABLED
               value: {{ .Values.global.datahub.enableGraphDiffMode | quote }}
+            {{- with .Values.global.datahub.mcp.throttle }}
+            {{- with .updateIntervalMs }}
+            - name: MCP_THROTTLE_UPDATE_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- if .versioned.enabled }}
+            - name: MCP_VERSIONED_THROTTLE_ENABLED
+              value: 'true'
+            {{- with .versioned.threshold }}
+            - name: MCP_VERSIONED_THRESHOLD
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.maxAttempts }}
+            - name: MCP_VERSIONED_MAX_ATTEMPTS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.initialIntervalMs }}
+            - name: MCP_VERSIONED_INITIAL_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.multiplier }}
+            - name: MCP_VERSIONED_MULTIPLIER
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .versioned.maxIntervalMs }}
+            - name: MCP_VERSIONED_MAX_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- end }}
+            {{- if .timeseries.enabled }}
+            - name: MCP_TIMESERIES_THROTTLE_ENABLED
+              value: 'true'
+            {{- with .timeseries.threshold }}
+            - name: MCP_TIMESERIES_THRESHOLD
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.maxAttempts }}
+            - name: MCP_TIMESERIES_MAX_ATTEMPTS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.initialIntervalMs }}
+            - name: MCP_TIMESERIES_INITIAL_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.multiplier }}
+            - name: MCP_TIMESERIES_MULTIPLIER
+              value: {{ . | quote }}
+            {{- end }}
+            {{- with .timeseries.maxIntervalMs }}
+            - name: MCP_TIMESERIES_MAX_INTERVAL_MS
+              value: {{ . | quote }}
+            {{- end }}
+            {{- end }}
+            {{- end }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

diff --git a/charts/datahub/values.yaml b/charts/datahub/values.yaml
@@ -749,6 +749,29 @@ global:
       show_browse_v2: true  # If on, show the new browse experience as of v0.10.5
       backfill_browse_v2: true  # If on, run the backfill upgrade job that generates default browse paths for relevant entities
 
+    ## v0.13.4+
+    mcp:
+      throttle:
+        # updateIntervalMs: 60000
+        ## Versioned MCL topic
+        versioned:
+          ## Whether to throttle MCP processing based on MCL backlog
+          enabled: true
+        #  threshold: 4000
+        #  maxAttempts: 1000
+        #  initialIntervalMs: 100
+        #  multiplier: 10
+        #  maxIntervalMs:  30000
+        # Timeseries MCL topic
+        timeseries:
+          ## Whether to throttle MCP processing based on MCL backlog
+          enabled: false
+        #  threshold: 4000
+        #  maxAttempts: 1000
+        #  initialIntervalMs: 100
+        #  multiplier: 10
+        #  maxIntervalMs: 30000
+
 #  hostAliases:
 #    - ip: "192.168.0.104"
 #      hostnames: