From 2241ad3ef96defe8f548e92d118aa35cbd286c0f Mon Sep 17 00:00:00 2001 From: Gianluca Date: Tue, 5 Mar 2024 19:41:49 +0100 Subject: [PATCH] Checkpoint --- examples/quickstart-confluent-cloud/README.md | 66 +++++++++++++++++++ .../quickstart-confluent-cloud/adapters.xml | 23 +++---- .../docker-compose.yml | 11 +++- examples/quickstart-schema-registry/README.md | 14 ++-- .../quickstart-schema-registry/adapters.xml | 11 ++-- examples/quickstart-ssl/README.md | 21 +++--- examples/quickstart-ssl/adapters.xml | 11 ++-- 7 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 examples/quickstart-confluent-cloud/README.md diff --git a/examples/quickstart-confluent-cloud/README.md b/examples/quickstart-confluent-cloud/README.md new file mode 100644 index 00000000..8f21abe9 --- /dev/null +++ b/examples/quickstart-confluent-cloud/README.md @@ -0,0 +1,66 @@ +# Quick Start with Confluent Cloud + +This folder contains a variant of the [_Quick Start_](../quickstart-ssl/README.md#quick-start-ssl) app configured to use _Confluent Cloud_ as the target Kafka cluster. + +The [docker-compose.yml](docker-compose.yml) file has been revised to realize the integration with Confluent Cloud: + +- Removal of the _broker_ service, because replaced by the remote Kafka cluster, as follows: + +- _kafka-connector_ + - Definition of new environment variables to configure the remote endpoint and credentials in the `adapters.xml` trough the _variable-expansion_ feature of Lightstreamer. + ```yaml + ... + environment: + - bootstrap_server=${bootstrap_server} + - api_key=${api_key} + - secret=${secret} + ... + ``` + - Adaption of [`adapters.xml`](./adapters.xml) to include: + - New Kafka cluster address retrieved from the environment variable `bootstrap_server`: + ```xml + $env.bootstrap_server + ``` + + - Encryption settings: + ```xml + true + TLSv1.2 + true + ``` + + - Authentication settings, with credentials retrieved from environment variables `api_key` and `secret`: + ```xml + true + PLAIN + $env.api_key + $env.secret + ``` + +- _producer_ + + Provisioning of the `producer.properties` configuration file to enable `SASL/PLAN` over TLS, with username and password retrieved from the environment variables `api_key` and `secret`: + + ```yaml + # Configure SASL/PLAIN mechanism + sasl.mechanism=PLAIN + # Enable SSL encryption + security.protocol=SASL_SSL + # JAAS configuration + sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="${api_key}" password="${secret}"; + ``` + +## Run + +From this directory, run follow the command: + +```sh +api_key= secret= bootstrap_server= ./start.sh +``` + +where +- `API.key` and `secret` are the credentials generated on the _Confluent CLI_ or from the _Confluent Cloud Console_. +- `bootstrap_server` is the Kafla cluster address. + +Then, point your browser to [http://localhost:8080/QuickStart](http://localhost:8080/QuickStart). + diff --git a/examples/quickstart-confluent-cloud/adapters.xml b/examples/quickstart-confluent-cloud/adapters.xml index 1978c153..f509ef25 100644 --- a/examples/quickstart-confluent-cloud/adapters.xml +++ b/examples/quickstart-confluent-cloud/adapters.xml @@ -1,14 +1,5 @@ - - - com.lightstreamer.kafka_connector.adapters.pub.KafkaConnectorMetadataAdapter @@ -18,32 +9,36 @@ com.lightstreamer.kafka_connector.adapters.KafkaConnectorDataAdapter - pkc-z9doz.eu-west-1.aws.confluent.cloud:9092 + + $env.bootstrap_server + quick-start-group + true TLSv1.2 true + true PLAIN $env.api_key $env.secret + stock-#{index=KEY} item-template.stock + + earliest INTEGER JSON - earliest - + #{TIMESTAMP} #{TOPIC} #{OFFSET} #{PARTITION} - - #{VALUE.timestamp} #{VALUE.time} #{VALUE.name} diff --git a/examples/quickstart-confluent-cloud/docker-compose.yml b/examples/quickstart-confluent-cloud/docker-compose.yml index d7745186..71a8b05a 100644 --- a/examples/quickstart-confluent-cloud/docker-compose.yml +++ b/examples/quickstart-confluent-cloud/docker-compose.yml @@ -5,10 +5,12 @@ services: container_name: kafka-connector image: lightstreamer-kafka-connector-${version} depends_on: + - broker - producer ports: - 8080:8080 environment: + - bootstrap_server=${bootstrap_server} - api_key=${api_key} - secret=${secret} volumes: @@ -17,6 +19,8 @@ services: producer: container_name: producer + depends_on: + - broker build: context: ../compose-templates dockerfile: Dockerfile.producer @@ -25,11 +29,14 @@ services: configs: - source: producer.properties target: /usr/app/producer.properties - command: ["--bootstrap-servers", "pkc-z9doz.eu-west-1.aws.confluent.cloud:9092", "--topic", "topic_0", "--config-file", "/usr/app/producer.properties"] + command: ["--bootstrap-servers", "${bootstrap_server}", "--topic", "topic_0", "--config-file", "/usr/app/producer.properties"] configs: producer.properties: content: | + # Configure SASL/PLAIN mechanism + sasl.mechanism=PLAIN + # Enable SSL encryption security.protocol=SASL_SSL + # JAAS configuration sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="${api_key}" password="${secret}"; - sasl.mechanism=PLAIN diff --git a/examples/quickstart-schema-registry/README.md b/examples/quickstart-schema-registry/README.md index a54a8bec..93a393dd 100644 --- a/examples/quickstart-schema-registry/README.md +++ b/examples/quickstart-schema-registry/README.md @@ -2,15 +2,13 @@ This folder contains a variant of the [_Quick Start SSL_](../quickstart-ssl/README.md#quick-start-ssl) app configured to use the _Confluent Schema Registry_. -The [docker-compose.yml](docker-compose.yml) has been revised to configure the integration with [_Confluent Docker Image for Schema Registry_](https://hub.docker.com/r/confluentinc/cp-schema-registry): +The [docker-compose.yml](docker-compose.yml) file has been revised to configure the integration with [_Confluent Docker Image for Schema Registry_](https://hub.docker.com/r/confluentinc/cp-schema-registry) as follows: -- _schema-registry_ - - The added service, pointing to the mentioned Docker image, with full configuration of the security settings. +- New service _schema-registry_, pulled from mentioned Docker image and configured with security settings. - _kafka-connector_ - - The new version of the [`adapters.xml`](./adapters.xml) includes: + + Adaption of [`adapters.xml`](./adapters.xml) to include: - Enabling of the Schema Registry: ```xml true @@ -37,10 +35,10 @@ The [docker-compose.yml](docker-compose.yml) has been revised to configure the i - _producer_ - The `producer.properties` configuration file adds the settings required to communicate with the Schema Registry: + Extension of the `producer.properties` configuration file with the settings required to communicate with the Schema Registry: ```yaml - .. + ... # JSON deserializer with support for the Schema Registry value.serializer=io.confluent.kafka.serializers.json.KafkaJsonSchemaSerializer # Schema Registry URL diff --git a/examples/quickstart-schema-registry/adapters.xml b/examples/quickstart-schema-registry/adapters.xml index d0ac148b..7fbf0231 100644 --- a/examples/quickstart-schema-registry/adapters.xml +++ b/examples/quickstart-schema-registry/adapters.xml @@ -4,7 +4,6 @@ com.lightstreamer.kafka_connector.adapters.pub.KafkaConnectorMetadataAdapter log4j.properties - @@ -13,7 +12,7 @@ broker:29094 quick-start-group - + true TLSv1.3 false @@ -24,14 +23,17 @@ kafka-connector-password kafka-connector-password + stock-#{index=KEY} item-template.stock + earliest INTEGER JSON true + https://schema-registry:8084 secrets/kafka-connector.truststore.jks kafka-connector-truststore-password @@ -40,14 +42,11 @@ kafka-connector-password kafka-connector-password - - + #{TIMESTAMP} #{TOPIC} #{OFFSET} #{PARTITION} - - #{VALUE.timestamp} #{VALUE.time} #{VALUE.name} diff --git a/examples/quickstart-ssl/README.md b/examples/quickstart-ssl/README.md index 8a7430b9..29da77c9 100644 --- a/examples/quickstart-ssl/README.md +++ b/examples/quickstart-ssl/README.md @@ -2,11 +2,11 @@ This folder contains all the resources needed to launch the [_Quick Start_](../../README.md#quick-start) app configured to establish a secure connection with the Kafka broker. -The [docker-compose.yml](docker-compose.yml) has been revised to enable support for SSL: +The [docker-compose.yml](docker-compose.yml) file has been revised to enable support for SSL, as follows: - _broker_ - - SSL enabled on port 29094. - - New environment variables to configure keystore, truststore, client authentication, and secrets: + - Enabling of SSL enabled on port 29094. + - Definition of new environment variables to configure keystore, truststore, client authentication, and secrets: - `KAFKA_SSL_TRUSTSTORE_FILENAME` - `KAFKA_SSL_TRUSTSTORE_CREDENTIALS` - `KAFKA_SSL_KEYSTORE_FILENAME` @@ -16,26 +16,26 @@ The [docker-compose.yml](docker-compose.yml) has been revised to enable support - _kafka-connector_ - The new version of the [`adapters.xml`](./adapters.xml) includes: - - Parameter `boostrap.servers` pointing to the SSL endpoint (`broker:29094`). + Adaption of [`adapters.xml`](./adapters.xml) to include: + - New SSL endpoint (`broker:29094`): ```xml broker:29094 ``` - - Encryption enabled. + - Encryption settings: ```xml true TLSv1.2 false ``` - - Configuration of the truststore to authenticate the broker. + - Configuration of the truststore to authenticate the broker: ```xml secrets/kafka.connector.truststore.jks kafka-connector-truststore-password ``` - - Configuration of the keystore for client authentication with the broker. + - Configuration of the keystore for client authentication with the broker: ```xml true secrets/kafka-connector.keystore.jks @@ -44,8 +44,8 @@ The [docker-compose.yml](docker-compose.yml) has been revised to enable support ``` - _producer_ - - The new `producer.properties` configuration file enables SSL support: - + - New SSL endpoint (`broker:29094`). + - Provisioning of the `producer.properties` configuration file to enable SSL support: ```yaml # Enable SSL security.protocol=SSL @@ -59,7 +59,6 @@ The [docker-compose.yml](docker-compose.yml) has been revised to enable support # Disable host name verification ssl.endpoint.identification.algorithm= ``` - - Changed target broker to new SSL endpoint (`broker:29094`). In addition, all services reference the local [`secrets`](../compose-templates/secrets/) folder to retrieve their secrets: diff --git a/examples/quickstart-ssl/adapters.xml b/examples/quickstart-ssl/adapters.xml index c19744d3..3d2fc209 100644 --- a/examples/quickstart-ssl/adapters.xml +++ b/examples/quickstart-ssl/adapters.xml @@ -4,7 +4,6 @@ com.lightstreamer.kafka_connector.adapters.pub.KafkaConnectorMetadataAdapter log4j.properties - @@ -13,7 +12,7 @@ broker:29094 quick-start-group - + true TLSv1.3 false @@ -24,20 +23,20 @@ kafka-connector-password kafka-connector-password + stock-#{index=KEY} item-template.stock + + earliest INTEGER JSON - earliest - + #{TIMESTAMP} #{TOPIC} #{OFFSET} #{PARTITION} - - #{VALUE.timestamp} #{VALUE.time} #{VALUE.name}