feat(mcl-hooks): add configuration for mcl hooks

charts/datahub/Chart.yaml

@@ -4,21 +4,21 @@ description: A Helm chart for DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.4.20
+version: 0.4.21
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: 0.14.0
 dependencies:
   - name: datahub-gms
-    version: 0.2.170
+    version: 0.2.171
     repository: file://./subcharts/datahub-gms
     condition: datahub-gms.enabled
   - name: datahub-frontend
     version: 0.2.158
     repository: file://./subcharts/datahub-frontend
     condition: datahub-frontend.enabled
   - name: datahub-mae-consumer
-    version: 0.2.159
+    version: 0.2.160
     repository: file://./subcharts/datahub-mae-consumer
     condition: global.datahub_standalone_consumers_enabled
   - name: datahub-mce-consumer

charts/datahub/subcharts/datahub-gms/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for DataHub's datahub-gms component
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.170
+version: 0.2.171
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.13.1

charts/datahub/subcharts/datahub-gms/templates/deployment.yaml

@@ -446,6 +446,42 @@ spec:
             {{- end }}
             {{- end }}
             {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.siblings }}
+            - name: ENABLE_SIBLING_HOOK
+              value: "{{ .enabled }}"
+            - name: SIBLINGS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.updateIndices }}
+            - name: ENABLE_UPDATE_INDICES_HOOK
+              value: "{{ .enabled }}"
+            - name: UPDATE_INDICES_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.ingestionScheduler }}
+            - name: ENABLE_INGESTION_SCHEDULER_HOOK
+              value: "{{ .enabled }}"
+            - name: INGESTION_SCHEDULER_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.incidents }}
+            - name: ENABLE_INCIDENTS_HOOK
+              value: "{{ .enabled }}"
+            - name: INCIDENTS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.entityChangeEvents }}
+            - name: ENABLE_ENTITY_CHANGE_EVENTS_HOOK
+              value: "{{ .enabled }}"
+            - name: ECE_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.forms }}
+            - name: FORMS_HOOK_ENABLED
+              value: "{{ .enabled }}"
+            - name: FORMS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

charts/datahub/subcharts/datahub-mae-consumer/Chart.yaml

@@ -12,7 +12,7 @@ description: A Helm chart for Kubernetes
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.2.159
+version: 0.2.160
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
 appVersion: v0.13.1

charts/datahub/subcharts/datahub-mae-consumer/templates/deployment.yaml

@@ -257,6 +257,42 @@ spec:
               value: {{ .Values.global.datahub.alwaysEmitChangeLog | quote }}
             - name: GRAPH_SERVICE_DIFF_MODE_ENABLED
               value: {{ .Values.global.datahub.enableGraphDiffMode | quote }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.siblings }}
+            - name: ENABLE_SIBLING_HOOK
+              value: "{{ .enabled }}"
+            - name: SIBLINGS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.updateIndices }}
+            - name: ENABLE_UPDATE_INDICES_HOOK
+              value: "{{ .enabled }}"
+            - name: UPDATE_INDICES_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.ingestionScheduler }}
+            - name: ENABLE_INGESTION_SCHEDULER_HOOK
+              value: "{{ .enabled }}"
+            - name: INGESTION_SCHEDULER_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.incidents }}
+            - name: ENABLE_INCIDENTS_HOOK
+              value: "{{ .enabled }}"
+            - name: INCIDENTS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.entityChangeEvents }}
+            - name: ENABLE_ENTITY_CHANGE_EVENTS_HOOK
+              value: "{{ .enabled }}"
+            - name: ECE_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
+            {{- with .Values.global.kafka.metadataChangeLog.hooks.forms }}
+            - name: FORMS_HOOK_ENABLED
+              value: "{{ .enabled }}"
+            - name: FORMS_HOOK_CONSUMER_GROUP_SUFFIX
+              value: "{{ .consumerGroupSuffix }}"
+            {{- end }}
           {{- with .Values.extraEnvs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

charts/datahub/values.yaml

@@ -593,6 +593,26 @@ global:
       metadata_change_log_timeseries_topic_name: "MetadataChangeLog_Timeseries_v1"
       platform_event_topic_name: "PlatformEvent_v1"
       datahub_upgrade_history_topic_name: "DataHubUpgradeHistory_v1"
+    metadataChangeLog:
+      hooks:
+        siblings:
+          enabled: true
+          consumerGroupSuffix: ''
+        updateIndices:
+          enabled: true
+          consumerGroupSuffix: ''
+        ingestionScheduler:
+          enabled: true
+          consumerGroupSuffix: ''
+        incidents:
+          enabled: true
+          consumerGroupSuffix: ''
+        entityChangeEvents:
+          enabled: true
+          consumerGroupSuffix: ''
+        forms:
+          enabled: true
+          consumerGroupSuffix: ''
     maxMessageBytes: "5242880"  # 5MB
     producer:
       compressionType: none