[#222] Support Kafka based messaging in Hono.

This adds the support for the Kafka based messaging to the Hono chart. The property `messagingNetworkType` is used to select which type of configuration is added to the adapters and services (either `amqp` or `kafka`). The property `kafkaMessagingClusterExample.enabled` indicates whether the example Kafka cluster should be deployed and used. Signed-off-by: Abel Buechner-Mihaljevic <[email protected]>
eclipse · May 18, 2021 · 1e16c9a · 1e16c9a
1 parent cd05725
commit 1e16c9a
Show file tree

Hide file tree

Showing 9 changed files with 150 additions and 8 deletions.
diff --git a/charts/hono/README.md b/charts/hono/README.md
@@ -518,3 +518,18 @@ helm install --dependency-update -n hono --set honoImagesType=quarkus-native ecl
 ```
 
 of Quarkus based services images.
+
+
+# Use Kafka based Messaging
+
+The chart can be configured to use Kafka as the messaging network instead of AMQP 1.0.
+The property `messagingNetworkType` is used to select the type of the messaging network.
+
+The following command provides a quickstart for Kafka based messaging:
+
+```bash
+# in directory packages/charts/hono
+helm install --dependency-update -n hono --values ./profileKafkaMessaging.yaml eclipse-hono .
+```
+It enables the deployment of an example Kafka cluster, disables the deployment of the AMQP 1.0 messaging network 
+and configures adapters and services to use Kafka based messaging:
diff --git a/charts/hono/profileKafkaMessaging.yaml b/charts/hono/profileKafkaMessaging.yaml
@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License 2.0 which is available at
+# http://www.eclipse.org/legal/epl-2.0
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+messagingNetworkType: kafka
+
+kafkaMessagingClusterExample:
+  enabled: true
+
+amqpMessagingNetworkExample:
+  enabled: false
diff --git a/charts/hono/requirements.yaml b/charts/hono/requirements.yaml
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -23,3 +23,7 @@ dependencies:
     repository: "https://charts.bitnami.com/bitnami"
     version: ~7.14.7
     condition: mongodb.createInstance
+  - name: kafka
+    repository: "https://charts.bitnami.com/bitnami"
+    version: ~12.17.6
+    condition: kafkaMessagingClusterExample.enabled
diff --git a/charts/hono/templates/_helpers.tpl b/charts/hono/templates/_helpers.tpl
@@ -171,6 +171,25 @@ healthCheck:
 {{- end }}
 
 
+{{/*
+Configuration for the messaging network clients.
+It configures for either AMQP based or Kafka based messaging.
+The scope passed in is expected to be a dict with keys
+- (mandatory) "dot": the root scope (".") and
+- (mandatory) "component": the name of the component
+*/}}
+{{- define "hono.messagingNetworkClientConfig" -}}
+{{- $args := dict "dot" .dot "component" .component -}}
+{{- if eq .dot.Values.messagingNetworkType "amqp" -}}
+  {{- include "hono.amqpMessagingNetworkClientConfig" $args }}
+{{- else if eq .dot.Values.messagingNetworkType "kafka" -}}
+  {{- include "hono.kafkaMessagingConfig" $args }}
+{{- else }}
+  {{- required "Property messagingNetworkType MUST be either 'amqp' or 'kafka'" nil }}
+{{- end -}}
+{{- end }}
+
+
 {{/*
 Configuration for the AMQP messaging network clients.
 The scope passed in is expected to be a dict with keys
@@ -194,6 +213,25 @@ messaging:
 {{- end }}
 
 
+{{/*
+Add configuration properties for Kafka based messaging to YAML file.
+
+The scope passed in is expected to be a dict with keys
+- (mandatory) "dot": the root scope (".") and
+- (mandatory) "component": the name of the component
+*/}}
+{{- define "hono.kafkaMessagingConfig" -}}
+{{- if .dot.Values.kafkaMessagingClusterExample.enabled -}}
+kafka:
+  defaultClientIdPrefix: {{ .component }}
+  commonClientConfig:
+    bootstrap.servers: {{ .dot.Release.Name }}-kafka:9092
+{{- else -}}
+  {{- required ".Values.adapters.kafkaMessagingSpec MUST be set if example Kafka cluster is disabled" .dot.Values.adapters.kafkaMessagingSpec | toYaml | nindent 2 }}
+{{- end -}}
+{{- end -}}
+
+
 {{/*
 Configuration for the clients accessing the example Device Registry.
 The scope passed in is expected to be a dict with keys
@@ -218,7 +256,8 @@ The scope passed in is expected to be a dict with keys
 */}}
 {{- define "hono.serviceClientConfig" -}}
 {{- $adapter := default "adapter" .component -}}
-{{- include "hono.amqpMessagingNetworkClientConfig" ( dict "dot" .dot "component" $adapter ) }}
+{{- include "hono.messagingNetworkClientConfig" ( dict "dot" .dot "component" $adapter ) }}
+{{- if eq .dot.Values.messagingNetworkType "amqp" }}
 command:
 {{- if .dot.Values.amqpMessagingNetworkExample.enabled }}
   name: Hono {{ $adapter }}
@@ -232,6 +271,8 @@ command:
 {{- else }}
   {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AMQP Messaging Network is disabled" .dot.Values.adapters.commandAndControlSpec | toYaml | nindent 2 }}
 {{- end }}
+{{/* commands with Kafka use the config from hono.messagingNetworkClientConfig */}}
+{{- end }}
 tenant:
 {{- if .dot.Values.adapters.tenantSpec }}
   {{- .dot.Values.adapters.tenantSpec | toYaml | nindent 2 }}

diff --git a/charts/hono/templates/hono-service-command-router/hono-service-command-router-secret.yaml b/charts/hono/templates/hono-service-command-router/hono-service-command-router-secret.yaml
@@ -71,6 +71,7 @@ stringData:
       tenant:
         {{- required ".Values.adapters.tenantSpec MUST be set if example Device Registry is disabled" .Values.adapters.tenantSpec | toYaml | nindent 8 }}
       {{- end }}
+      {{- if eq .Values.messagingNetworkType "amqp" }}
       command:
       {{- if .Values.amqpMessagingNetworkExample.enabled }}
         name: {{ printf "Hono %s" $args.component | quote }}
@@ -82,7 +83,12 @@ stringData:
         trustStorePath: {{ .Values.adapters.commandAndControlSpec.trustStorePath }}
         hostnameVerificationRequired: {{ .Values.adapters.commandAndControlSpec.hostnameVerificationRequired }}
       {{- else }}
-        {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AQMP Messaging Network is disabled" .Values.adapters.commandAndControlSpec | toYaml | nindent 8 }}
+        {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AMQP Messaging Network is disabled" .Values.adapters.commandAndControlSpec | toYaml | nindent 8 }}
+      {{- end }}
+      {{- else if eq .Values.messagingNetworkType "kafka" }}
+        {{- include "hono.kafkaMessagingConfig" $args | nindent 6 }}
+      {{- else }}
+        {{- required "Values.messagingNetworkType MUST be either 'amqp' or 'kafka'" nil }}
       {{- end }}
       {{- include "hono.healthServerConfig" .Values.commandRouterService.hono.healthCheck | nindent 6 }}
     {{- include "hono.quarkusConfig" $args | indent 4 }}

diff --git a/...hono/templates/hono-service-device-registry-file/hono-service-device-registry-secret.yaml b/...hono/templates/hono-service-device-registry-file/hono-service-device-registry-secret.yaml
@@ -67,7 +67,7 @@ stringData:
         svc:
           filename: "/var/lib/hono/device-registry/tenants.json"
           saveToFile: true
-      {{- include "hono.amqpMessagingNetworkClientConfig" $args | nindent 6 }}
+      {{- include "hono.messagingNetworkClientConfig" $args | nindent 6 }}
       {{- include "hono.healthServerConfig" .Values.deviceRegistryExample.hono.healthCheck | nindent 6 }}
 data:
   key.pem: {{ .Files.Get "example/certs/device-registry-key.pem" | b64enc }}

diff --git a/...hono/templates/hono-service-device-registry-jdbc/hono-service-device-registry-secret.yaml b/...hono/templates/hono-service-device-registry-jdbc/hono-service-device-registry-secret.yaml
@@ -66,7 +66,7 @@ stringData:
           {{- if .Values.deviceRegistryExample.jdbcBasedDeviceRegistry.tenant.jdbc }}
           {{- .Values.deviceRegistryExample.jdbcBasedDeviceRegistry.tenant.jdbc | toYaml | nindent 10 }}
           {{- end }}
-      {{- include "hono.amqpMessagingNetworkClientConfig" $args | nindent 6 }}
+      {{- include "hono.messagingNetworkClientConfig" $args | nindent 6 }}
       {{- include "hono.healthServerConfig" .Values.deviceRegistryExample.hono.healthCheck | nindent 6 }}
 data:
   key.pem: {{ .Files.Get "example/certs/device-registry-key.pem" | b64enc }}

diff --git a/...o/templates/hono-service-device-registry-mongodb/hono-service-device-registry-secret.yaml b/...o/templates/hono-service-device-registry-mongodb/hono-service-device-registry-secret.yaml
@@ -67,7 +67,7 @@ stringData:
         username: {{ .Values.mongodb.mongodbUsername | quote }}
         password: {{ .Values.mongodb.mongodbPassword | quote }}
         {{- end }}
-      {{- include "hono.amqpMessagingNetworkClientConfig" $args | nindent 6 }}
+      {{- include "hono.messagingNetworkClientConfig" $args | nindent 6 }}
       {{- include "hono.healthServerConfig" .Values.deviceRegistryExample.hono.healthCheck | nindent 6 }}
 data:
   key.pem: {{ .Files.Get "example/certs/device-registry-key.pem" | b64enc }}

diff --git a/charts/hono/values.yaml b/charts/hono/values.yaml
@@ -39,6 +39,8 @@ livenessProbeInitialDelaySeconds: 300
 # component level.
 readinessProbeInitialDelaySeconds: 20
 
+# amqpMessagingNetworkExample contains properties for configuring an example AMQP network
+# to be used for messaging if "messagingNetworkType" is set to "amqp"
 amqpMessagingNetworkExample:
   # enabled indicates whether the example AMQP Messaging Network
   # consisting of a single Dispatch Router and Broker should be
@@ -314,6 +316,14 @@ useLoadBalancer: true
 # Device Connection service towards the new Command Router service.
 useCommandRouter: true
 
+# messagingNetworkType indicates which type of messaging should be used.
+# The following types are defined:
+# - amqp: AMQP 1.0 based messaging. Also refer to the
+#     sections "amqpMessagingNetworkExample" and "amqpMessagingNetworkSpec".
+# - kafka: Apache Kafka based messaging. Also refer to the
+#     sections "kafkaMessagingClusterExample" and "amqpMessagingNetworkSpec"
+messagingNetworkType: amqp
+
 # Configuration properties for protocol adapters.
 adapters:
 
@@ -344,7 +354,8 @@ adapters:
 
   # amqpMessagingNetworkSpec contains Hono client properties used by all protocol
   # adapters for connecting to the AMQP Messaging Network to forward downstream messages to.
-  # This property MUST be set if "amqpMessagingNetworkExample.enabled" is set to false.
+  # This property MUST be set if "messagingNetworkType" is "amqp" and
+  # "amqpMessagingNetworkExample.enabled" is set to false.
   # Please refer to https://www.eclipse.org/hono/docs/admin-guide/hono-client-configuration/
   # for a description of supported properties.
   # However, if "amqpMessagingNetworkExample.enabled" is set to true, only
@@ -361,7 +372,8 @@ adapters:
   # commandAndControlSpec contains Hono client properties used by all protocol
   # adapters for connecting to the AMQP Messaging Network which is used by applications
   # to send commands to devices.
-  # This property MUST be set if "amqpMessagingNetworkExample.enabled" is set to false.
+  # This property MUST be set if "messagingNetworkType" is "amqp" and
+  # "amqpMessagingNetworkExample.enabled" is set to false.
   # Please refer to https://www.eclipse.org/hono/docs/admin-guide/hono-client-configuration/
   # for a description of supported properties.
   # However, if "amqpMessagingNetworkExample.enabled" is set to true, only
@@ -372,6 +384,14 @@ adapters:
     trustStorePath: /etc/hono/trusted-certs.pem
     hostnameVerificationRequired: false
 
+  # kafkaMessagingSpec contains the configuration used by all protocol
+  # adapters for connecting to the Kafka cluster to be uses for messaging.
+  # This property MUST be set if "messagingNetworkType" is "kafka" and
+  # "kafkaMessagingClusterExample.enabled" is set to false.
+  # Please refer to https://www.eclipse.org/hono/docs/admin-guide/hono-kafka-client-configuration/
+  # for a description of supported properties.
+  kafkaMessagingSpec:
+
   # tenantSpec contains Hono client properties used by all protocol adapters for
   # connecting to the Tenant service.
   # This property MUST be set if "deviceRegistryExample.enabled" is set to false.
@@ -1680,3 +1700,40 @@ grafana:
 
   rbac:
     namespaced: true
+
+# kafkaMessagingClusterExample contains properties for configuring an example Kafka cluster
+# to be used for messaging if "messagingNetworkType" is set to "kafka"
+kafkaMessagingClusterExample:
+  # enabled indicates whether the example Kafka cluster consisting of a single broker
+  # and one Zookeeper instance should be deployed. This minimal deployment is not suitable
+  # for production purposes. To use an already existing Kafka cluster instead,
+  # set this property to false and configure "adapters.kafkaMessagingSpec".
+  enabled: false
+
+# configuration of the example Kafka cluster to be deployed if "kafkaMessagingClusterExample.enabled" is "true"
+kafka:
+  replicaCount: 1
+  # The data is stored in Persistent Volume Claims. For more information regarding persistence and
+  # potential problems with permissions refer to: https://github.com/bitnami/charts/tree/master/bitnami/kafka#persistence
+  persistence:
+    size: 1Gi
+  zookeeper:
+    persistence:
+      size: 1Gi
+  # Expose the Kafka service to be accessed from outside the cluster (LoadBalancer service).
+  # Alternatively use NodePort configuration, for more information refer to
+  # https://github.com/bitnami/charts/tree/master/bitnami/kafka#accessing-kafka-brokers-from-outside-the-cluster
+  externalAccess:
+    enabled: true
+    service:
+      type: LoadBalancer
+      port: 9094
+      loadBalancerIPs: []
+    autoDiscovery:
+      enabled: true
+  serviceAccount:
+    create: true
+  rbac:
+    # Note that this could require creating RBAC rules, for more information refer to
+    # https://github.com/bitnami/charts/tree/master/bitnami/kafka#accessing-kafka-brokers-from-outside-the-cluster
+    create: true