Add experimental support for KVP format and subscription to regex

Lightstreamer · Dec 16, 2024 · 5c14cf7 · 5c14cf7
1 parent 9b60c71
commit 5c14cf7
Show file tree

Hide file tree

Showing 62 changed files with 2,149 additions and 54 deletions.
diff --git a/docs/javadoc/allclasses-index.html b/docs/javadoc/allclasses-index.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>All Classes and Interfaces (kafka-connector 1.1.0 API)</title>
+<title>All Classes and Interfaces (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="class index">

diff --git a/docs/javadoc/allpackages-index.html b/docs/javadoc/allpackages-index.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>All Packages (kafka-connector 1.1.0 API)</title>
+<title>All Packages (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="package index">

diff --git a/docs/javadoc/com/lightstreamer/kafka/adapters/pub/KafkaConnectorMetadataAdapter.html b/docs/javadoc/com/lightstreamer/kafka/adapters/pub/KafkaConnectorMetadataAdapter.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>KafkaConnectorMetadataAdapter (kafka-connector 1.1.0 API)</title>
+<title>KafkaConnectorMetadataAdapter (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="declaration: package: com.lightstreamer.kafka.adapters.pub, class: KafkaConnectorMetadataAdapter">

diff --git a/docs/javadoc/com/lightstreamer/kafka/adapters/pub/package-summary.html b/docs/javadoc/com/lightstreamer/kafka/adapters/pub/package-summary.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>com.lightstreamer.kafka.adapters.pub (kafka-connector 1.1.0 API)</title>
+<title>com.lightstreamer.kafka.adapters.pub (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="declaration: package: com.lightstreamer.kafka.adapters.pub">

diff --git a/docs/javadoc/com/lightstreamer/kafka/adapters/pub/package-tree.html b/docs/javadoc/com/lightstreamer/kafka/adapters/pub/package-tree.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>com.lightstreamer.kafka.adapters.pub Class Hierarchy (kafka-connector 1.1.0 API)</title>
+<title>com.lightstreamer.kafka.adapters.pub Class Hierarchy (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="tree: package: com.lightstreamer.kafka.adapters.pub">

diff --git a/docs/javadoc/help-doc.html b/docs/javadoc/help-doc.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>API Help (kafka-connector 1.1.0 API)</title>
+<title>API Help (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="help">

diff --git a/docs/javadoc/index-all.html b/docs/javadoc/index-all.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>Index (kafka-connector 1.1.0 API)</title>
+<title>Index (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="index">

diff --git a/docs/javadoc/index.html b/docs/javadoc/index.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>Overview (kafka-connector 1.1.0 API)</title>
+<title>Overview (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="package index">

diff --git a/docs/javadoc/overview-summary.html b/docs/javadoc/overview-summary.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>kafka-connector 1.1.0 API</title>
+<title>kafka-connector 1.1.1-kvp API</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="index redirect">

diff --git a/docs/javadoc/overview-tree.html b/docs/javadoc/overview-tree.html
@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
 <!-- Generated by javadoc (17) -->
-<title>Class Hierarchy (kafka-connector 1.1.0 API)</title>
+<title>Class Hierarchy (kafka-connector 1.1.1-kvp API)</title>
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
 <meta name="description" content="class tree">

diff --git a/examples/kvp-demo/README.md b/examples/kvp-demo/README.md
@@ -0,0 +1,3 @@
+# Quick Start
+
+This folder contains all the resources needed to launch the _Quick Start_ app. See the [Quick Start](../../README.md#quick-start-set-up-in-5-minutes) section for more details.
diff --git a/examples/kvp-demo/adapters.xml b/examples/kvp-demo/adapters.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+
+<!-- Mandatory. Define the Kafka Connector Adapter Set and its unique ID. -->
+<adapters_conf id="KafkaConnector">
+    <metadata_provider>
+        <adapter_class>com.lightstreamer.kafka.adapters.pub.KafkaConnectorMetadataAdapter</adapter_class>
+        <param name="logging.configuration.path">log4j.properties</param>
+    </metadata_provider>
+
+    <data_provider name="prices">
+        <adapter_class>com.lightstreamer.kafka.adapters.KafkaConnectorDataAdapter</adapter_class>
+
+        <param name="bootstrap.servers">broker:29092</param>
+
+        <param name="group.id">quick-start-group</param>
+
+        <param name="record.key.evaluator.type">STRING</param>
+        <param name="record.value.evaluator.type">KVP</param>
+
+        <param name="item-template.quotes">quote-#{topic=TOPIC}</param>
+
+        <param name="map.MPDF.SIT.DATA.QUOTE.SAMA.(\d+).to">item-template.quotes</param>
+        <!--
+        <param name="map.MPDF.SIT.DATA.QUOTE.SAMA.(\d+)_15m.to">quotes_15m</param>
+        <param name="map.MPDF.SIT.DATA.QUOTE.SAMA.(\d+)_SABE.to">quotes_sabe</param>
+        <param name="map.MPDF.SIT.DATA.QUOTE.SAMA.(\d+)_SABE_15m.to">quotes_sabe_15m</param>
+        -->
+
+        <param name="field.SYMBOL">#{VALUE.NAME}</param>
+        <param name="field.QCHARTTOT">#{VALUE.QCHARTTOT}</param>
+        <param name="field.TRow">#{VALUE.TRow}</param>
+        <param name="field.TMSTMP">#{VALUE.TMSTMP}</param>
+        <param name="field.QCHART">#{VALUE.QCHART}</param>
+        <param name="field.VTOT">#{VALUE.VTOT}</param>
+        <param name="field.QTOT">#{VALUE.QTOT}</param>
+        <param name="field.QA">#{VALUE.QA}</param>
+        <param name="field.Q">#{VALUE.Q}</param>
+        <param name="field.PA">#{VALUE.PA}</param>
+        <param name="field.PCHART">#{VALUE.PCHART}</param>
+        <param name="field.NTRAD">#{VALUE.NTRAD}</param>
+        <param name="field.PLIM_MIN">#{VALUE.PLIM_MIN}</param>
+        <param name="field.PLIM_MAX">#{VALUE.PLIM_MAX}</param>
+
+    </data_provider>
+
+</adapters_conf>
diff --git a/examples/kvp-demo/docker b/examples/kvp-demo/docker
@@ -0,0 +1 @@
+../compose-templates/docker
diff --git a/examples/kvp-demo/docker-compose.yml b/examples/kvp-demo/docker-compose.yml
@@ -0,0 +1,50 @@
+---
+name: quickstart-kafka-connector
+services:
+  kafka-connector:
+    container_name: kafka-connector
+    image: lightstreamer-kafka-connector-${version}
+    depends_on:
+      - broker
+      - producer
+    ports:
+      - 8080:8080
+    volumes:
+      - ./web:/lightstreamer/pages/QuickStart
+      - ./adapters.xml:/lightstreamer/adapters/lightstreamer-kafka-connector-${version}/adapters.xml      
+      - ./log4j.properties:/lightstreamer/adapters/lightstreamer-kafka-connector-${version}/log4j.properties
+
+  producer:
+    container_name: producer
+    depends_on:
+      - broker
+    build:
+      context: ./producer
+      args:
+        VERSION: ${version}
+    command: ["--bootstrap-servers", "broker:29092"]
+
+  broker:
+    image: apache/kafka
+    hostname: broker
+    container_name: broker
+    ports:
+      - "9092:9092"
+      - "8082:8082"
+    environment:
+      KAFKA_NODE_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@broker:29093'
+      KAFKA_LISTENERS: 'PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
+      KAFKA_REST_HOST_NAME: rest-proxy
+      KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
+      KAFKA_REST_LISTENERS: 'http://0.0.0.0:8082'
diff --git a/examples/kvp-demo/helpers.sh b/examples/kvp-demo/helpers.sh
@@ -0,0 +1 @@
+../compose-templates/helpers.sh
diff --git a/examples/kvp-demo/log4j.properties b/examples/kvp-demo/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.logger.com.lightstreamer.kafka.adapters.pub.KafkaConnectorMetadataAdapter
+log4j.logger.org.apache.kafka=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] [%-20t] [%-10c{1}] %-5p %m%n
+log4j.appender.stdout.Target=System.out
+
+# prices logger
+log4j.logger.prices=INFO, stdout
diff --git a/examples/kvp-demo/producer/Dockerfile b/examples/kvp-demo/producer/Dockerfile
@@ -0,0 +1,23 @@
+# Use the offical Gradle Docker image as builder
+FROM gradle:jdk17 AS builder
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the project resources
+COPY src ./src
+COPY build.gradle .
+COPY gradlew .
+COPY settings.gradle .
+COPY gradle.properties .
+
+# Generate the jar file
+RUN gradle build
+
+# Use the official Eclipse Temuring image
+FROM eclipse-temurin:21-jdk
+
+# Copy the Producer jar from the previuos build stage
+COPY --from=builder /app/build/libs/quickstart-producer-all.jar /usr/app/producer.jar
+
+ENTRYPOINT ["java", "-jar", "/usr/app/producer.jar"]
diff --git a/examples/kvp-demo/producer/README.md b/examples/kvp-demo/producer/README.md
@@ -0,0 +1,17 @@
+# Producer for the Quick Start App
+
+This folder contains the Gradle project of the Kafka native producer used for the _Quick Start_ App. See the [Quick Start](../../README.md#quick-start-set-up-in-5-minutes) section for more details.
+
+To build the producer:
+
+```sh
+$ ./gradlew build
+```
+
+which generates the `quickstart-producer-all.jar` file under the `build/libs` folder.
+
+To run it:
+
+```sh
+$ java -jar build/libs/quickstart-producer-all.jar --bootstrap-servers <kafka.connection.string> --topic stocks
+```
diff --git a/...ucer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$1.class b/...ucer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$1.class
diff --git a/...ightstreamer/kafka/examples/quick_start/producer/FeedSimulator$ExternalFeedListener.class b/...ightstreamer/kafka/examples/quick_start/producer/FeedSimulator$ExternalFeedListener.class
diff --git a/.../bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$FIELD.class b/.../bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$FIELD.class
diff --git a/...n/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$QuoteProducer.class b/...n/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator$QuoteProducer.class
diff --git a/...oducer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator.class b/...oducer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/FeedSimulator.class
diff --git a/.../producer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/Producer$1.class b/.../producer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/Producer$1.class
diff --git a/...mo/producer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/Producer.class b/...mo/producer/bin/main/com/lightstreamer/kafka/examples/quick_start/producer/Producer.class
diff --git a/examples/kvp-demo/producer/bin/main/log4j.properties b/examples/kvp-demo/producer/bin/main/log4j.properties
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=WARN, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] [%-10c{1}] %-5p %m%n
+log4j.appender.stdout.Target=System.out
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=[%d] [%t-%c] %p %m%n
+log4j.appender.stderr.Target=System.err
diff --git a/examples/kvp-demo/producer/build.gradle b/examples/kvp-demo/producer/build.gradle
@@ -0,0 +1,73 @@
+plugins {
+    id 'java'
+    id 'maven-publish'
+}
+
+repositories {
+    mavenCentral()
+    maven {
+        url "https://packages.confluent.io/maven"
+    }
+}
+
+dependencies {
+    implementation group: 'info.picocli', name:'picocli', version:'4.7.5'
+
+    implementation group: 'org.slf4j', name: 'slf4j-reload4j', 'version': '2.0.10'
+    implementation group: 'org.apache.kafka', name:'kafka-clients', version:'7.7.0-ccs'
+    implementation group: 'io.confluent', name: 'kafka-avro-serializer', version:'7.7.0'
+    implementation group: 'io.confluent', name: 'kafka-json-serializer', version:'7.7.0'
+    implementation group: 'io.confluent', name: 'kafka-json-schema-serializer', version:'7.7.0'
+}
+
+ext {
+    appendix = 'all'
+}
+
+group = 'com.lightstreamer.kafka'
+version = '1.0.0'
+
+java {
+    toolchain {
+        languageVersion = JavaLanguageVersion.of(17)
+    }
+}
+
+tasks.withType(JavaCompile) {
+    options.encoding = 'UTF-8'
+}
+
+tasks.withType(Javadoc) {
+    options.encoding = 'UTF-8'
+}
+
+jar {
+    manifest {
+        attributes 'Main-Class': "com.lightstreamer.kafka.examples.quick_start.producer.Producer"
+    }
+    archiveAppendix = appendix
+    archiveVersion = ''
+
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+    from configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
+}
+
+publishing {
+    repositories {
+        maven {
+            name = "GitHubPackages"
+            url = uri("https://maven.pkg.github.com/lightstreamer/Lightstreamer-kafka-connector")
+            credentials {
+                username = project.findProperty("gpr.user") ?: System.getenv("USERNAME")
+                password = project.findProperty("gpr.key") ?: System.getenv("TOKEN")
+            }
+        }
+    }
+    publications {
+        gpr(MavenPublication) {
+            artifactId project.name + '-' + appendix
+            version = version
+            artifact jar
+        }
+    }
+}
diff --git a/examples/kvp-demo/producer/gradle.properties b/examples/kvp-demo/producer/gradle.properties
@@ -0,0 +1 @@
+deployDirName=deploy
diff --git a/examples/kvp-demo/producer/gradle/wrapper/gradle-wrapper.jar b/examples/kvp-demo/producer/gradle/wrapper/gradle-wrapper.jar
diff --git a/examples/kvp-demo/producer/gradle/wrapper/gradle-wrapper.properties b/examples/kvp-demo/producer/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Quick Start

		This folder contains all the resources needed to launch the _Quick Start_ app. See the [Quick Start](../../README.md#quick-start-set-up-in-5-minutes) section for more details.