[#222] Support Kafka based messaging in Hono.

charts/hono/Chart.yaml

@@ -15,7 +15,7 @@ name: hono
 description: |
   Eclipse Hono™ provides remote service interfaces for connecting large numbers of IoT devices to a back end and
   interacting with them in a uniform way regardless of the device communication protocol.
-version: 1.8.3
+version: 1.8.4
 # Version of Hono being deployed by the chart
 appVersion: 1.8.0
 keywords:

charts/hono/README.md

@@ -529,3 +529,65 @@ helm install --dependency-update -n hono --set honoImagesType=quarkus-native ecl
 ```
 
 of Quarkus based services images.
+
+
+## Using Kafka based Messaging
+
+The chart can be configured to use Kafka as the messaging network instead of an AMQP 1.0 messaging network.
+The property `messagingNetworkType` is used to select the type of the messaging network.
+
+The following command provides a quickstart for Kafka based messaging (ensure `minikube tunnel` is running when using Minikube):
+
+```bash
+helm install --dependency-update -n hono --set messagingNetworkType=kafka --set kafkaMessagingClusterExample.enabled=true --set amqpMessagingNetworkExample.enabled=false  eclipse-hono eclipse-iot/hono
+```
+
+It enables the deployment of an example Kafka cluster, disables the deployment of the AMQP 1.0 messaging network 
+and configures adapters and services to use Kafka based messaging.
+
+### Using a production grade Kafka cluster
+
+If Kafka based messaging is enabled by setting `messagingNetworkType` to `kafka`, the Kafka clients need to
+be configured with connection information for a Kafka cluster. The Helm chart can deploy an example Kafka cluster. 
+This is enabled by setting `kafkaMessagingClusterExample.enabled` to `true`. With this setting the chart
+deploys a Kafka cluster consisting of a single broker and a single Zookeeper instance and configures the 
+protocol adapters to connect to the example cluster.
+
+In a production environment, though, usage of the example Kafka cluster is strongly discouraged as it does not provide 
+any redundancy.
+
+The Helm chart can be configured to use an existing Kafka cluster instead of the example deployment.
+In order to do so, the protocol adapters need to be configured with information about the bootstrap server addresses
+and configuration properties.
+
+The easiest way to set these properties is by means of putting them into a YAML file with content like this:
+
+```yaml
+# configure protocol adapters for Kafka messaging
+messagingNetworkType: kafka
+
+# do not deploy example AMQP Messaging Network
+amqpMessagingNetworkExample:
+  enabled: false
+
+# do not deploy example Kafka cluster
+kafkaMessagingClusterExample:
+  enabled: false
+
+adapters:
+  # provide connection params
+  kafkaMessagingSpec:
+    commonClientConfig:
+      bootstrap.servers: broker0.my-custom-kafka.org:9092,broker1.my-custom-kafka.org:9092
+```
+
+*adapters.kafkaMessagingSpec* needs to contain configuration properties as described in Hono's
+[Kafka client admin guide](https://www.eclipse.org/hono/docs/admin-guide/hono-kafka-client-configuration/).
+Make sure to adapt/add properties as required by the Kafka cluster.
+
+Assuming that the file is named `customKafkaCluster.yaml`, the values can then be passed in to the Helm `install`
+command as follows:
+
+```bash
+helm install --dependency-update -n hono -f /path/to/customKafkaCluster.yaml eclipse-hono eclipse-iot/hono
+```

charts/hono/example/ca_opts

@@ -146,3 +146,10 @@ subjectKeyIdentifier = hash
 keyUsage             = keyAgreement,keyEncipherment,digitalSignature
 extendedKeyUsage     = serverAuth, clientAuth
 subjectAltName       = DNS.1:localhost
+
+[ req_ext_kafka ]
+
+subjectKeyIdentifier = hash
+keyUsage             = keyAgreement,keyEncipherment,digitalSignature
+extendedKeyUsage     = serverAuth, clientAuth
+subjectAltName       = DNS.1:*.hono-kafka-headless,DNS.2:*.hono-kafka-headless.hono,DNS.3:localhost

charts/hono/example/certs/kafka-cert.pem

@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIICPTCCAeOgAwIBAgIUB2vxWI9wj32OHLaABVV+iuVkdRwwCgYIKoZIzj0EAwIw
+UDELMAkGA1UEBhMCQ0ExDzANBgNVBAcMBk90dGF3YTEUMBIGA1UECgwLRWNsaXBz
+ZSBJb1QxDTALBgNVBAsMBEhvbm8xCzAJBgNVBAMMAmNhMB4XDTIxMDYwMjE1MjUw
+N1oXDTIyMDYwMjE1MjUwN1owUzELMAkGA1UEBhMCQ0ExDzANBgNVBAcMBk90dGF3
+YTEUMBIGA1UECgwLRWNsaXBzZSBJb1QxDTALBgNVBAsMBEhvbm8xDjAMBgNVBAMM
+BWthZmthMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEHfvUCACcO9wS9c/57EfA
+i34dNdNTUPwAib143fEUiaC9wPCp6EPzIjFHx78n8DgY7iXc+rZE1BXqAbqVO/n0
+3KOBlzCBlDAdBgNVHQ4EFgQUErFQDWfU3iYKEYv8ws7Ka6N7AvAwCwYDVR0PBAQD
+AgOoMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjBHBgNVHREEQDA+ghUq
+Lmhvbm8ta2Fma2EtaGVhZGxlc3OCGiouaG9uby1rYWZrYS1oZWFkbGVzcy5ob25v
+gglsb2NhbGhvc3QwCgYIKoZIzj0EAwIDSAAwRQIhANeuZW+OCsrM23R2p2g5iH7/
+SyoSVU8d6DkcVpawSxgtAiAPWibmpN0qWTrf3s4N1zoaYC6EB7LY6D1cstaQ+/Lf
+rA==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB4zCCAYmgAwIBAgIUDvfsevHpF7ObReAAmGXXHHsAXDswCgYIKoZIzj0EAwIw
+UjELMAkGA1UEBhMCQ0ExDzANBgNVBAcMBk90dGF3YTEUMBIGA1UECgwLRWNsaXBz
+ZSBJb1QxDTALBgNVBAsMBEhvbm8xDTALBgNVBAMMBHJvb3QwHhcNMjEwMTI2MTMx
+MzI1WhcNMjIwMTI2MTMxMzI1WjBQMQswCQYDVQQGEwJDQTEPMA0GA1UEBwwGT3R0
+YXdhMRQwEgYDVQQKDAtFY2xpcHNlIElvVDENMAsGA1UECwwESG9ubzELMAkGA1UE
+AwwCY2EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQrWtTxDTpqzkLfkZWT+RMp
+w3y6/Mbmrj3S4DTfEv9bsuwUvZwcF7yy5X5YWFq+WOESLBh3nykxxg0MBRHdN0fx
+oz8wPTAdBgNVHQ4EFgQUBxIgSnCFs43mB6a9umhpKCA2I30wDwYDVR0TAQH/BAUw
+AwEB/zALBgNVHQ8EBAMCAQYwCgYIKoZIzj0EAwIDSAAwRQIgRau0yW4JCG+2e3w5
+KFWzCYV20/DNJ2Lj5ospGvNhl9sCIQCYde5228wNvKT3Qw6vk70HiS5r/mhFNJaZ
+aPyf7W2E4g==
+-----END CERTIFICATE-----

charts/hono/example/certs/kafka-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg0xs9OqU6CWzt1swR
+qsf9pHWPducg3NGNAYG23hxHNkehRANCAAQd+9QIAJw73BL1z/nsR8CLfh0101NQ
+/ACJvXjd8RSJoL3A8KnoQ/MiMUfHvyfwOBjuJdz6tkTUFeoBupU7+fTc
+-----END PRIVATE KEY-----

charts/hono/example/create_certs.sh

@@ -41,6 +41,9 @@ AMQP_ADAPTER_KEY_STORE=amqpKeyStore.p12
 AMQP_ADAPTER_KEY_STORE_PWD=amqpkeys
 EXAMPLE_GATEWAY_KEY_STORE=exampleGatewayKeyStore.p12
 EXAMPLE_GATEWAY_KEY_STORE_PWD=examplegatewaykeys
+KAFKA_KEY_STORE=kafkaKeyStore.jks
+# the bitnami Kafka chart expects truststore and keystore to have the same password
+KAFKA_KEY_STORE_PWD=honotrust
 # set to either EC or RSA
 KEY_ALG=EC
 
@@ -141,5 +144,6 @@ create_cert artemis $ARTEMIS_KEY_STORE $ARTEMIS_KEY_STORE_PWD
 create_cert coap-adapter $COAP_ADAPTER_KEY_STORE $COAP_ADAPTER_KEY_STORE_PWD
 create_cert amqp-adapter $AMQP_ADAPTER_KEY_STORE $AMQP_ADAPTER_KEY_STORE_PWD
 create_cert example-gateway $EXAMPLE_GATEWAY_KEY_STORE $EXAMPLE_GATEWAY_KEY_STORE_PWD
+create_cert kafka $KAFKA_KEY_STORE $KAFKA_KEY_STORE_PWD
 
 create_client_cert 4711

charts/hono/profileKafkaMessaging.yaml

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License 2.0 which is available at
+# http://www.eclipse.org/legal/epl-2.0
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+messagingNetworkType: kafka
+
+kafkaMessagingClusterExample:
+  enabled: true
+
+amqpMessagingNetworkExample:
+  enabled: false

charts/hono/requirements.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -23,3 +23,7 @@ dependencies:
     repository: "https://charts.bitnami.com/bitnami"
     version: ~7.14.7
     condition: mongodb.createInstance
+  - name: kafka
+    repository: "https://charts.bitnami.com/bitnami"
+    version: ~12.17.6
+    condition: kafkaMessagingClusterExample.enabled

charts/hono/templates/_helpers.tpl

@@ -171,6 +171,25 @@ healthCheck:
 {{- end }}
 
 
+{{/*
+Configuration for the messaging network clients.
+It configures for either AMQP based or Kafka based messaging.
+The scope passed in is expected to be a dict with keys
+- (mandatory) "dot": the root scope (".") and
+- (mandatory) "component": the name of the component
+*/}}
+{{- define "hono.messagingNetworkClientConfig" -}}
+{{- $args := dict "dot" .dot "component" .component -}}
+{{- if eq .dot.Values.messagingNetworkType "amqp" -}}
+  {{- include "hono.amqpMessagingNetworkClientConfig" $args }}
+{{- else if eq .dot.Values.messagingNetworkType "kafka" -}}
+  {{- include "hono.kafkaMessagingConfig" $args }}
+{{- else }}
+  {{- required "Property messagingNetworkType MUST be either 'amqp' or 'kafka'" nil }}
+{{- end -}}
+{{- end }}
+
+
 {{/*
 Configuration for the AMQP messaging network clients.
 The scope passed in is expected to be a dict with keys
@@ -194,6 +213,56 @@ messaging:
 {{- end }}
 
 
+{{/*
+Add configuration properties for Kafka based messaging to YAML file.
+
+The scope passed in is expected to be a dict with keys
+- (mandatory) "dot": the root scope (".") and
+- (mandatory) "component": the name of the component
+*/}}
+{{- define "hono.kafkaMessagingConfig" -}}
+{{- include "hono.kafkaConfigCheck" (dict "dot" .dot) }}
+kafka:
+  defaultClientIdPrefix: {{ .component }}
+{{- if .dot.Values.kafkaMessagingClusterExample.enabled }}
+  commonClientConfig:
+    bootstrap.servers: {{ .dot.Release.Name }}-{{ .dot.Values.kafka.nameOverride }}-0.{{ .dot.Release.Name }}-{{ .dot.Values.kafka.nameOverride }}-headless.{{ .dot.Release.Namespace }}:{{ .dot.Values.kafka.service.port }}
+    security.protocol: SASL_SSL
+    sasl.mechanism: SCRAM-SHA-512
+    sasl.jaas.config: "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"{{ first .dot.Values.kafka.auth.sasl.jaas.clientUsers }}\" password=\"{{ first .dot.Values.kafka.auth.sasl.jaas.clientPasswords }}\";"
+    ssl.truststore.location: /etc/hono/truststore.jks
+    ssl.truststore.password: {{ .dot.Values.kafka.auth.tls.password }}
+    ssl.endpoint.identification.algorithm: "" # Disables hostname verification. Don't do this in productive setups!
+{{- else if not .dot.Values.adapters.kafkaMessagingSpec }}
+  {{- required ".Values.adapters.kafkaMessagingSpec MUST be provided if example Kafka cluster is disabled" nil }}
+{{- else if not (index .dot.Values.adapters.kafkaMessagingSpec.commonClientConfig "bootstrap.servers") }}
+  {{- required ".Values.adapters.kafkaMessagingSpec.commonClientConfig MUST contain 'bootstrap.servers' if example Kafka cluster is disabled" nil }}
+{{- end }}
+{{- if .dot.Values.adapters.kafkaMessagingSpec }}
+  {{- .dot.Values.adapters.kafkaMessagingSpec | toYaml | nindent 2 }}
+{{- end }}
+{{- end }}
+
+
+{{/*
+Check configuration for consistency in case of Kafka based messaging.
+
+The scope passed in is expected to be a dict with keys
+- (mandatory) "dot": the root scope (".")
+*/}}
+{{- define "hono.kafkaConfigCheck" -}}
+  {{- if and (eq .dot.Values.messagingNetworkType "kafka") .dot.Values.kafkaMessagingClusterExample.enabled }}
+    {{- if .dot.Values.useLoadBalancer }}
+      {{- if not (eq .dot.Values.kafka.externalAccess.service.type "LoadBalancer") }}
+        {{- required ".Values.kafka.externalAccess.service.type MUST be 'LoadBalancer' if .Values.useLoadBalancer is true" nil }}
+      {{- end }}
+    {{- else if not (eq .dot.Values.kafka.externalAccess.service.type "NodePort") }}
+      {{- required ".Values.kafka.externalAccess.service.type MUST be 'NodePort' if .Values.useLoadBalancer is false" nil }}
+    {{- end }}
+  {{- end }}
+{{- end }}
+
+
 {{/*
 Configuration for the clients accessing the example Device Registry.
 The scope passed in is expected to be a dict with keys
@@ -218,7 +287,8 @@ The scope passed in is expected to be a dict with keys
 */}}
 {{- define "hono.serviceClientConfig" -}}
 {{- $adapter := default "adapter" .component -}}
-{{- include "hono.amqpMessagingNetworkClientConfig" ( dict "dot" .dot "component" $adapter ) }}
+{{- include "hono.messagingNetworkClientConfig" ( dict "dot" .dot "component" $adapter ) }}
+{{- if eq .dot.Values.messagingNetworkType "amqp" }}
 command:
 {{- if .dot.Values.amqpMessagingNetworkExample.enabled }}
   name: Hono {{ $adapter }}
@@ -232,6 +302,8 @@ command:
 {{- else }}
   {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AMQP Messaging Network is disabled" .dot.Values.adapters.commandAndControlSpec | toYaml | nindent 2 }}
 {{- end }}
+{{/* commands with Kafka use the config from hono.messagingNetworkClientConfig */}}
+{{- end }}
 tenant:
 {{- if .dot.Values.adapters.tenantSpec }}
   {{- .dot.Values.adapters.tenantSpec | toYaml | nindent 2 }}

charts/hono/templates/hono-adapter-amqp/hono-adapter-amqp-vertx-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.amqp.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -41,4 +41,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/amqp-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/amqp-adapter.credentials" | b64enc }}
-{{- end }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
+{{- end }}

charts/hono/templates/hono-adapter-coap/hono-adapter-coap-vertx-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.coap.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -40,4 +40,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/coap-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/coap-adapter.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}

charts/hono/templates/hono-adapter-http/hono-adapter-http-vertx-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.http.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -41,4 +41,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/http-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/http-adapter.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}

charts/hono/templates/hono-adapter-kura/hono-adapter-kura-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.kura.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -40,4 +40,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/kura-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/kura-adapter.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}

charts/hono/templates/hono-adapter-lora/hono-adapter-lora-vertx-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.lora.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -40,4 +40,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/lora-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/lora-adapter.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}

charts/hono/templates/hono-adapter-mqtt/hono-adapter-mqtt-vertx-secret.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.adapters.mqtt.enabled }}
 #
-# Copyright (c) 2019, 2020 Contributors to the Eclipse Foundation
+# Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation
 #
 # See the NOTICE file(s) distributed with this work for additional
 # information regarding copyright ownership.
@@ -45,4 +45,5 @@ data:
   cert.pem: {{ .Files.Get "example/certs/mqtt-adapter-cert.pem" | b64enc }}
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/mqtt-adapter.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}

charts/hono/templates/hono-service-command-router/hono-service-command-router-secret.yaml

@@ -71,6 +71,7 @@ stringData:
       tenant:
         {{- required ".Values.adapters.tenantSpec MUST be set if example Device Registry is disabled" .Values.adapters.tenantSpec | toYaml | nindent 8 }}
       {{- end }}
+      {{- if eq .Values.messagingNetworkType "amqp" }}
       command:
       {{- if .Values.amqpMessagingNetworkExample.enabled }}
         name: {{ printf "Hono %s" $args.component | quote }}
@@ -82,7 +83,12 @@ stringData:
         trustStorePath: {{ .Values.adapters.commandAndControlSpec.trustStorePath }}
         hostnameVerificationRequired: {{ .Values.adapters.commandAndControlSpec.hostnameVerificationRequired }}
       {{- else }}
-        {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AQMP Messaging Network is disabled" .Values.adapters.commandAndControlSpec | toYaml | nindent 8 }}
+        {{- required ".Values.adapters.commandAndControlSpec MUST be set if example AMQP Messaging Network is disabled" .Values.adapters.commandAndControlSpec | toYaml | nindent 8 }}
+      {{- end }}
+      {{- else if eq .Values.messagingNetworkType "kafka" }}
+        {{- include "hono.kafkaMessagingConfig" $args | nindent 6 }}
+      {{- else }}
+        {{- required "Values.messagingNetworkType MUST be either 'amqp' or 'kafka'" nil }}
       {{- end }}
       {{- include "hono.healthServerConfig" .Values.commandRouterService.hono.healthCheck | nindent 6 }}
     {{- include "hono.quarkusConfig" $args | indent 4 }}
@@ -96,4 +102,5 @@ data:
   trusted-certs.pem: {{ .Files.Get "example/certs/trusted-certs.pem" | b64enc }}
   auth-server-cert.pem: {{ .Files.Get "example/certs/auth-server-cert.pem" | b64enc }}
   adapter.credentials: {{ .Files.Get "example/command-router.credentials" | b64enc }}
+  truststore.jks: {{ .Files.Get "example/certs/trustStore.jks" | b64enc }}
 {{- end }}