diff --git a/README.md b/README.md
index 2b3a1908..cb269509 100644
--- a/README.md
+++ b/README.md
@@ -79,9 +79,9 @@ As you can see from the diagram above, in this variant the stream of simulated m
To provide a complete stack, the app is based on _Docker Compose_. The [Docker Compose file](examples/quickstart/docker-compose.yaml) comprises the following services:
1. A Kafka broker, based on the [Confluent Local Docker Image](confluentinc/confluent-local:latest).
-2. Lighstreamer Server with Kafka Connector, based on the [Lightstreamer Kafka Connector Docker image example](examples/docker-image/)
+2. Lighstreamer Server with Kafka Connector, based on the [Lightstreamer Kafka Connector Docker image example](examples/docker-image/).
3. The web client, mounted on the Lightreamer service.
-4. A native Kafka Producer, based on the provided [Dockerfile.producer](examples/quickstart/Dockerfile.producer) file and [kafka-connector-samples](kafka-connector-samples/) submodule of the project root.
+4. A native Kafka Producer, based on the provided [Dockerfile.producer](examples/quickstart/Dockerfile.producer) file and [kafka-connector-samples](kafka-connector-samples/) submodule of this repository.
### Run
@@ -91,7 +91,15 @@ To run the app:
2. From the [`examples/quickstart`](examples/quickstart/) folder, run the command:
```sh
- ./launch_quickstart.sh
+ ./start.sh
+ ...
+ ⠏ Network quickstart_default Created
+ ✔ Container broker Started
+ ✔ Container producer Started
+ ✔ Container kafka-connector Started
+ ...
+ Service started. Now you can point your browser to http://localhost:8080/QuickStart to see real-time data.
+ ...
```
3. Once all containers are ready, point your browser to [http://localhost:8080/QuickStart](http://localhost:8080/QuickStart).
@@ -101,7 +109,7 @@ To run the app:
4. To shutdown Docker Compose and clean up all temporary resources:
```sh
- ./shutdown_quickstart.sh
+ ./stop.sh
```
## Installation
diff --git a/buildSrc/src/main/groovy/lightstreamer-kafka-connector.gradle b/buildSrc/src/main/groovy/lightstreamer-kafka-connector.gradle
index 6a5958a6..7386a65d 100644
--- a/buildSrc/src/main/groovy/lightstreamer-kafka-connector.gradle
+++ b/buildSrc/src/main/groovy/lightstreamer-kafka-connector.gradle
@@ -71,7 +71,6 @@ tasks.named('test') {
useJUnitPlatform()
}
-
-task cleanQuickStart(type: Delete) {
- delete "$rootDir/$quickstartDeployDirName"
-}
\ No newline at end of file
+clean {
+ delete "$rootDir/$deployDirName"
+}
diff --git a/examples/docker-image/Dockerfile b/examples/docker-image/Dockerfile
index 68790c5b..bbff8a38 100644
--- a/examples/docker-image/Dockerfile
+++ b/examples/docker-image/Dockerfile
@@ -1,14 +1,15 @@
# Start form the Official Lightstramer Server image
FROM lightstreamer
-# The Kafka Connector version
+# The Kafka Connector project version
ARG VERSION
-ENV KAFKA_CONNECTOR=lightstreamer-kafka-connector-${VERSION}
-ENV DEPLOY_DIR=/lightstreamer/adapters/${KAFKA_CONNECTOR}
+ENV KAFKA_CONNECTOR_NAME=lightstreamer-kafka-connector-${VERSION}
+ENV KAFKA_CONNECTOR_ZIP=${KAFKA_CONNECTOR_NAME}.zip
+ENV DEPLOY_DIR=/lightstreamer/adapters/${KAFKA_CONNECTOR_NAME}
# Copy the distribution package
-COPY tmp/${KAFKA_CONNECTOR}.zip /tmp/${KAFKA_CONNECTOR}.zip
+COPY tmp/${KAFKA_CONNECTOR_ZIP} /tmp/${KAFKA_CONNECTOR_ZIP}
# Copy custom resources
COPY resources /tmp/resources
@@ -17,7 +18,7 @@ USER root
RUN apt-get -y update; \
apt-get install -y unzip; \
# Unzip the distribution package into to Lightstreamer's adapters fodler.
- unzip /tmp/${KAFKA_CONNECTOR}.zip -d /lightstreamer/adapters; \
+ unzip /tmp/${KAFKA_CONNECTOR_ZIP} -d /lightstreamer/adapters; \
# Copy the custom resources
cp /tmp/resources/* ${DEPLOY_DIR}; \
# Fix ownership
diff --git a/examples/docker-image/build.sh b/examples/docker-image/build.sh
index f78e19f7..0c0902cf 100755
--- a/examples/docker-image/build.sh
+++ b/examples/docker-image/build.sh
@@ -1,23 +1,16 @@
#!/bin/bash
-
-# Set the root project directory
-projectDir=../../
-
-# Alias to the local gradlew command
-_gradle="${projectDir}/gradlew --project-dir ${projectDir}"
-
+source ../utils/helpers.sh
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+TMP_DIR=${SCRIPT_DIR}/tmp
# Generate the distribution
-echo "Making the distribution"
+echo "Making the distribution package"
$_gradle distribuite
-# Get the current version
-version=$($_gradle properties -q --console=plain | grep '^version:' | awk '{ printf $2}')
-
-mkdir -p tmp
-cp ${projectDir}/deploy/lightstreamer-kafka-connector-${version}.zip tmp/
+rm -fr ${TMP_DIR}; mkdir ${TMP_DIR}
+cp ${projectDir}/deploy/lightstreamer-kafka-connector-${version}.zip ${TMP_DIR}
echo "Build the Lightstramer Kafka Connector Docker image"
-docker build -t lightstreamer-kafka-connector-${version} . --build-arg VERSION=${version}
+docker build -t lightstreamer-kafka-connector-${version} $SCRIPT_DIR --build-arg VERSION=${version}
if [ $? == 0 ]; then
echo "Launch the image with:"
diff --git a/examples/quickstart/Dockerfile.producer b/examples/quickstart/Dockerfile.producer
index 382d1e58..41bdcaf6 100644
--- a/examples/quickstart/Dockerfile.producer
+++ b/examples/quickstart/Dockerfile.producer
@@ -1,6 +1,9 @@
FROM eclipse-temurin:17
-RUN mkdir /opt/producer
-COPY tmp/lightstreamer-kafka-connector-samples-all-*.jar /opt/producer/lightstreamer-kafka-connector-samples-all.jar
+# The Kafka Connector project version
+ARG VERSION
-ENTRYPOINT ["java", "-jar", "/opt/producer/lightstreamer-kafka-connector-samples-all.jar"]
+# Copy the Produer jar
+COPY tmp/lightstreamer-kafka-connector-samples-all-${VERSION}.jar /usr/app/producer.jar
+
+ENTRYPOINT ["java", "-jar", "/usr/app/producer.jar"]
diff --git a/examples/quickstart/docker-compose.yaml b/examples/quickstart/docker-compose.yaml
index 2001430b..14dd9966 100644
--- a/examples/quickstart/docker-compose.yaml
+++ b/examples/quickstart/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '2'
services:
kafka-connector:
container_name: kafka-connector
- image: lightstreamer-kafka-connector-0.1.0
+ image: lightstreamer-kafka-connector-${version}
depends_on:
- producer
ports:
@@ -17,6 +17,8 @@ services:
- broker
build:
dockerfile: Dockerfile.producer
+ args:
+ VERSION: ${version}
command: ["--bootstrap-servers", "broker:29092", "--topic", "stocks"]
broker:
@@ -33,8 +35,6 @@ services:
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
- KAFKA_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/kafka.server.keystore.jks
- KAFKA_SSL_KEYSTORE_PASSWORD: password
KAFKA_PROCESS_ROLES: 'broker,controller'
KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
@@ -44,4 +44,3 @@ services:
KAFKA_REST_HOST_NAME: rest-proxy
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
- # CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
\ No newline at end of file
diff --git a/examples/quickstart/launch_quickstart.sh b/examples/quickstart/launch_quickstart.sh
deleted file mode 100755
index e6078b7b..00000000
--- a/examples/quickstart/launch_quickstart.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-cwd=$(pwd)
-
-# Build the Lightstreamer Kafka Connector Docker image
-cd ../docker-image
-./build.sh
-
-if [ $? == 0 ]; then
- cd ../..
- # Deploy the Kafka Producer to the tmp folder
- ./gradlew QuickStart
- cd $cwd
- docker compose up --build -d
-fi
\ No newline at end of file
diff --git a/examples/quickstart/shutdown_quickstart.sh b/examples/quickstart/shutdown_quickstart.sh
deleted file mode 100755
index 1f58adfe..00000000
--- a/examples/quickstart/shutdown_quickstart.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-docker compose down
-rm -fr tmp
\ No newline at end of file
diff --git a/examples/quickstart/start.sh b/examples/quickstart/start.sh
new file mode 100755
index 00000000..4d1c2520
--- /dev/null
+++ b/examples/quickstart/start.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+source ../utils/helpers.sh
+rm -fr tmp; mkdir -p tmp
+
+# Build the Lightstreamer Kafka Connector Docker image
+../docker-image/build.sh
+
+if [ $? == 0 ]; then
+ cp ../../deploy/lightstreamer-kafka-connector-samples-all-${version}.jar tmp/
+ # Export the version env variable to be used by Compose
+ export version
+ docker compose up --build -d
+ sleep 20
+ echo "Service started. Now you can point your browers to http://localhost:8080/QuickStart to see real-time data."
+ fi
\ No newline at end of file
diff --git a/examples/quickstart/stop.sh b/examples/quickstart/stop.sh
new file mode 100755
index 00000000..51064efa
--- /dev/null
+++ b/examples/quickstart/stop.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+source ../utils/helpers.sh
+
+# Export the version env variable to be used by Compose
+export version
+docker compose down
+rm -fr tmp
+$_gradle clean
\ No newline at end of file
diff --git a/examples/utils/helpers.sh b/examples/utils/helpers.sh
new file mode 100755
index 00000000..953106d1
--- /dev/null
+++ b/examples/utils/helpers.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+HELPER_DIR=$(dirname ${BASH_SOURCE[0]}) ## realpath
+projectDir="$HELPER_DIR/../.."
+
+# Alias to the the local gradlew command
+_gradle="${projectDir}/gradlew --project-dir ${projectDir}"
+
+# Get the current version
+version=$(grep '^version' ${projectDir}/gradle.properties | awk -F= '{ printf $2}')
diff --git a/examples/utils/remote.sh b/examples/utils/remote.sh
new file mode 100644
index 00000000..fbee0a8d
--- /dev/null
+++ b/examples/utils/remote.sh
@@ -0,0 +1,26 @@
+# Add Docker's official GPG key:
+sudo apt-get update
+sudo apt-get install ca-certificates curl
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo \
+ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+ $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+sudo apt-get update
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+sudo docker run hello-world
+sudo /usr/sbin/usermod -a -G docker ubuntu
+
+sudo apt install net-tools
+sudo apt-get install zip unzip
+curl -s "https://get.sdkman.io" | bash
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+sdk version
+sdk install java
+mkdir confluent
+git clone https://github.com/confluentinc/kafka-images.git confluent/kafka-images
diff --git a/gradle.properties b/gradle.properties
index ec5634cc..b035701e 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,3 +1,3 @@
-version = 0.1.0
-deployDirName = deploy
-quickstartDeployDirName = examples/quickstart/tmp
+version=0.1.1
+deployDirName=deploy
+quickstartDeployDirName=examples/quickstart/tmp
diff --git a/kafka-connector/src/connector/dist/adapters.xml b/kafka-connector/src/connector/dist/adapters.xml
index 2d6a363d..8a467530 100644
--- a/kafka-connector/src/connector/dist/adapters.xml
+++ b/kafka-connector/src/connector/dist/adapters.xml
@@ -134,6 +134,7 @@
INTEGER
JSON
+ earliest
#{KEY}
diff --git a/kafka-connector/src/main/java/com/lightstreamer/kafka_connector/adapters/consumers/ConsumerLoop.java b/kafka-connector/src/main/java/com/lightstreamer/kafka_connector/adapters/consumers/ConsumerLoop.java
index 3aa462ad..e43f654d 100644
--- a/kafka-connector/src/main/java/com/lightstreamer/kafka_connector/adapters/consumers/ConsumerLoop.java
+++ b/kafka-connector/src/main/java/com/lightstreamer/kafka_connector/adapters/consumers/ConsumerLoop.java
@@ -225,7 +225,7 @@ private boolean subscribed() {
// Check the actual available topics on Kafka.
try (Admin admin = AdminClient.create(config.consumerProperties())) {
ListTopicsOptions options = new ListTopicsOptions();
- options.timeoutMs(5000);
+ options.timeoutMs(30000);
ListTopicsResult listTopics = admin.listTopics(options);
boolean notAllPresent = false;
diff --git a/kafka-connector/src/test/java/com/lightstreamer/kafka_connector/adapters/config/ConnectorConfigTest.java b/kafka-connector/src/test/java/com/lightstreamer/kafka_connector/adapters/config/ConnectorConfigTest.java
index d18adb38..a590601b 100644
--- a/kafka-connector/src/test/java/com/lightstreamer/kafka_connector/adapters/config/ConnectorConfigTest.java
+++ b/kafka-connector/src/test/java/com/lightstreamer/kafka_connector/adapters/config/ConnectorConfigTest.java
@@ -744,6 +744,20 @@ public void shouldGetEnabled() {
assertThat(config.isEnabled()).isFalse();
}
+ @Test
+ public void shouldOverrideAutoOffset() {
+ ConnectorConfig config =
+ ConnectorConfig.newConfig(adapterDir.toFile(), standardParameters());
+
+ Map updatedConfig = new HashMap<>(standardParameters());
+ updatedConfig.put(ConnectorConfig.CONSUMER_AUTO_OFFSET_RESET_CONFIG, "earliest");
+ config = ConnectorConfig.newConfig(adapterDir.toFile(), updatedConfig);
+ assertThat(config.getText(ConnectorConfig.CONSUMER_AUTO_OFFSET_RESET_CONFIG))
+ .isEqualTo("earliest");
+ assertThat(config.baseConsumerProps())
+ .containsAtLeast(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ }
+
@Test
public void shouldGetEncryptionEnabled() {
ConnectorConfig config =