adding feature to use configmaps to configure kafka components

charts/kafka-kraft-on-k8s/templates/_helpers.tpl

@@ -0,0 +1,25 @@
+{{/* Define template to generate controller.quorum.voters for broker mode */}}
+{{- define "kafka.controllerQuorumVoters" -}}
+{{- $totalReplicas := .Values.kafka.configmap.properties.controllerReplicas -}}
+{{- $service := .Values.kafka.configmap.properties.controllerService -}}
+{{- $namespace := .Values.kafka.configmap.properties.namespace -}}
+{{- $voters := list -}}
+{{- range $i := until (int $totalReplicas) }}
+  {{- $voter := printf "%d@controller-%d.%s.%s.svc:9093" $i $i $service $namespace -}}
+  {{- $voters = append $voters $voter -}}
+{{- end }}
+{{- join "," $voters -}}
+{{- end -}}
+
+{{/* Define template to generate controller.quorum.voters for combined broker and controller mode */}}
+{{- define "kafka.controllerQuorumVotersCombined" -}}
+{{- $totalReplicas := .Values.kafka.configmap.properties.brokerReplicas -}}
+{{- $service := .Values.kafka.configmap.properties.service -}}
+{{- $namespace := .Values.kafka.configmap.properties.namespace -}}
+{{- $voters := list -}}
+{{- range $i := until (int $totalReplicas) }}
+  {{- $voter := printf "%d@kafka-%d.%s.%s.svc:9093" $i $i $service $namespace -}}
+  {{- $voters = append $voters $voter -}}
+{{- end }}
+{{- join "," $voters -}}
+{{- end -}}

charts/kafka-kraft-on-k8s/templates/configmap_connect.yaml

@@ -0,0 +1,35 @@
+{{- if and .Values.connect.enabled .Values.connect.configmap.enabled (not .Values.connect.env.ENVS_ENABLED) }}
+{{- $portsList := .Values.kafka.ports }}
+{{- $portMap := index $portsList 0 }}
+{{- $KafkaPort := $portMap.port }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.connect.configmap.name }}
+data:
+  connect-distributed.properties: |-
+    bootstrap.servers={{ .Values.connect.configmap.properties.service }}.{{ .Values.connect.configmap.properties.namespace }}.svc:{{ $KafkaPort }}
+    group.id={{ .Values.connect.configmap.properties.groupId }}
+    key.converter={{ .Values.connect.configmap.properties.keyConverter }}
+    key.converter.schemas.enable={{ .Values.connect.configmap.properties.keyConverterSchemasEnable }}
+    value.converter={{ .Values.connect.configmap.properties.valueConverter }}
+    value.converter.schemas.enable={{ .Values.connect.configmap.properties.valueConverterSchemasEnable }}
+    offset.storage.topic={{ .Values.connect.configmap.properties.offsetStorageTopic }}
+    offset.storage.replication.factor={{ .Values.connect.configmap.properties.offsetStorageReplicationFactor }}
+    offset.storage.partitions={{ .Values.connect.configmap.properties.offsetStoragePartitions }}
+    offset.storage.cleanup.policy={{ .Values.connect.configmap.properties.offsetStorageCleanupPolicy }}
+    config.storage.topic={{ .Values.connect.configmap.properties.configStorageTopic }}
+    config.storage.replication.factor={{ .Values.connect.configmap.properties.configStorageReplicationFactor }}
+    status.storage.topic={{ .Values.connect.configmap.properties.statusStorageTopic }}
+    status.storage.replication.factor={{ .Values.connect.configmap.properties.statusStorageReplicationFactor }}
+    status.storage.partitions={{ .Values.connect.configmap.properties.statusStoragePartitions }}
+    listeners={{ .Values.connect.configmap.properties.listeners }}
+    plugin.path={{ .Values.connect.configmap.properties.pluginPath }}
+{{- else }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.connect.configmap.name }}
+data:
+  message: "INFO: envs are enabled. properties are not set via configmap."
+{{- end }}

charts/kafka-kraft-on-k8s/templates/configmap_controller.yaml

@@ -0,0 +1,24 @@
+{{- if and .Values.controller.enabled .Values.controller.configmap.enabled (not .Values.controller.env.ENABLED) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.controller.configmap.name }}
+data:
+  {{- $processRoles := .Values.controller.configmap.properties.processRoles | trim }}
+  {{- if eq $processRoles "controller" }}
+  controller.properties: |-
+    process.roles={{ .Values.controller.configmap.properties.processRoles }}
+    log.dirs={{ .Values.controller.configmap.properties.logdirs }}/{{ add .Values.controller.configmap.properties.nodeIdOffset (int (index (splitList "-" .Release.Name) 0)) }}
+    listeners={{ .Values.controller.configmap.properties.listeners }}
+    controller.listener.names={{ .Values.controller.configmap.properties.controllerListenerNames }}
+    controller.listener.security.protocol.map={{ .Values.controller.configmap.properties.controllerListenerSecurityProtocolMap }}
+    controller.quorum.voters={{ include "kafka.controllerQuorumVoters" . }}
+{{- else }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.controller.configmap.name }}
+data:
+  message: "INFO: envs are enabled. properties are not set via configmap."
+{{- end }}
+{{- end }}

charts/kafka-kraft-on-k8s/templates/configmap_kafka.yaml

@@ -0,0 +1,43 @@
+{{- if and .Values.kafka.enabled .Values.kafka.configmap.enabled (not .Values.kafka.env.ENABLED) }}
+{{- $portsList := .Values.controller.ports }}
+{{- $portMap := index $portsList 0 }}
+{{- $ControllerPort := $portMap.port }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.kafka.configmap.name }}
+data:
+  {{- $processRoles := .Values.kafka.configmap.properties.processRoles | trim }}
+  {{- if eq $processRoles "broker" }}
+  broker.properties: |-
+    process.roles={{ .Values.kafka.configmap.properties.processRoles }}
+    listeners={{ .Values.kafka.configmap.properties.listeners }}
+    advertised.listeners={{ .Values.kafka.configmap.properties.advertisedListeners }}
+    log.dirs={{ .Values.kafka.configmap.properties.logdirs }}/{{ add .Values.kafka.configmap.properties.nodeIdOffset (int (index (splitList "-" .Release.Name) 0)) }}
+    num.partitions={{ .Values.kafka.configmap.properties.kafkaNumPartitions }}
+    controller.listener.names={{ .Values.kafka.configmap.properties.controllerListenerNames }}
+    inter.broker.listener.name={{ .Values.kafka.configmap.properties.interBrokerListenerName }}
+    listener.security.protocol.map={{ .Values.kafka.configmap.properties.controllerListenerSecurityProtocolMap }}
+    controller.quorum.voters={{ include "kafka.controllerQuorumVoters" . }}
+  {{- else if and (regexMatch "broker" .Values.kafka.configmap.properties.processRoles) (regexMatch "controller" .Values.kafka.configmap.properties.processRoles) }}
+  server.properties: |-
+    process.roles={{ .Values.kafka.configmap.properties.processRoles }}
+    listeners={{ .Values.kafka.configmap.properties.listeners }}
+    advertised.listeners={{ .Values.kafka.configmap.properties.advertisedListeners }}
+    log.dirs={{ .Values.kafka.configmap.properties.logdirs }}/{{ add .Values.kafka.configmap.properties.nodeIdOffset (int (index (splitList "-" .Release.Name) 0)) }}
+    num.partitions={{ .Values.kafka.configmap.properties.kafkaNumPartitions }}
+    default.replication.factor={{ .Values.kafka.configmap.properties.defaultReplicationFactor }}
+    auto.create.topics.enable={{ .Values.kafka.configmap.properties.autoCreateTopicsEnable }}
+    controller.listener.names={{ .Values.kafka.configmap.properties.controllerListenerNames }}
+    inter.broker.listener.name={{ .Values.kafka.configmap.properties.interBrokerListenerName }}
+    listener.security.protocol.map={{ .Values.kafka.configmap.properties.controllerListenerSecurityProtocolMap }}
+    controller.quorum.voters={{ include "kafka.controllerQuorumVotersCombined" . }}
+  {{- end }}
+{{- else }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.kafka.configmap.name }}
+data:
+  message: "INFO: envs are enabled. properties are not set via configmap."
+{{- end }}

charts/kafka-kraft-on-k8s/templates/stable_connect.yaml → charts/kafka-kraft-on-k8s/templates/sts_connect.yaml

@@ -3,6 +3,9 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: {{ .Values.connect.name }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "3"
   labels:
     {{- range $key, $value := .Values.connect.labels }}
     {{ $key }}: {{ $value }}
@@ -80,4 +83,15 @@ spec:
               cpu: {{ .Values.connect.resources.requests.cpu }}
               memory: {{ .Values.connect.resources.requests.memory }}
             {{- end }}
+          volumeMounts:
+            {{- range .Values.connect.volumeMounts }}
+            - name: {{ .name | quote }}
+              mountPath: {{ .mountPath }}
+            {{- end }}
+      volumes:
+        {{- range .Values.connect.volumes }}
+        - name: {{ .name | quote }}
+          configMap:
+            name: {{ $.Values.connect.configmap.name | quote }}
+        {{- end }}
 {{- end }}

charts/kafka-kraft-on-k8s/templates/stable_controller.yaml → charts/kafka-kraft-on-k8s/templates/sts_controller.yaml

@@ -3,6 +3,9 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: {{ .Values.controller.name }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "1"
   labels:
     {{- range $key, $value := .Values.controller.labels }}
     {{ $key }}: {{ $value }}
@@ -85,6 +88,12 @@ spec:
             - name: {{ .name | quote }}
               mountPath: {{ .mountPath }}
             {{- end }}
+      volumes:
+        {{- range .Values.controller.volumes }}
+        - name: {{ .name | quote }}
+          configMap:
+            name: {{ $.Values.controller.configmap.name | quote }}
+        {{- end }}
 
   volumeClaimTemplates:
     {{- range .Values.controller.volumeClaimTemplates }}

charts/kafka-kraft-on-k8s/templates/stable_kafka.yaml → charts/kafka-kraft-on-k8s/templates/sts_kafka.yaml

@@ -3,6 +3,9 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: {{ .Values.kafka.name }}
+  annotations:
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "2"
   labels:
     {{- range $key, $value := .Values.kafka.labels }}
     {{ $key }}: {{ $value }}
@@ -60,13 +63,13 @@ spec:
           livenessProbe:
             tcpSocket:
               port: {{ $port.port }}
-            initialDelaySeconds: 5
-            periodSeconds: 3
+            initialDelaySeconds: 15
+            periodSeconds: 10
           readinessProbe:
             tcpSocket:
               port: {{ $port.port }}
-            initialDelaySeconds: 5
-            periodSeconds: 3
+            initialDelaySeconds: 15
+            periodSeconds: 10
             {{- end }}
           {{- end }}
           resources:
@@ -85,6 +88,12 @@ spec:
             - name: {{ .name | quote }}
               mountPath: {{ .mountPath }}
             {{- end }}
+      volumes:
+        {{- range .Values.kafka.volumes }}
+        - name: {{ .name | quote }}
+          configMap:
+            name: {{ $.Values.kafka.configmap.name | quote }}
+        {{- end }}
 
   volumeClaimTemplates:
     {{- range .Values.kafka.volumeClaimTemplates }}