adapters.xml

<?xml version="1.0"?>

<!--
    This is the configuration file of the Lightstreamer Kafka Connector pluggable into Lightstreamer Server.

    A very simple variable-expansion feature is available; see
    <enable_expansion_for_adapters_config> in the Server's main configuration file.
-->

<!-- Mandatory. Define the Kafka Connector Adapter Set and its unique ID. -->
<adapters_conf id="KafkaConnector">
    <metadata_provider>
        <!-- Mandatory. Java class name of the Kafka Connector Metadata Adapter. It is possible to provide a
             custom implementation by extending this class. -->
        <adapter_class>com.lightstreamer.kafka.adapters.pub.KafkaConnectorMetadataAdapter</adapter_class>

        <!-- Mandatory. The path of the reload4j configuration file, relative to the deployment folder
             (LS_HOME/adapters/lightstreamer-kafka-connector). -->
        <param name="logging.configuration.path">log4j.properties</param>

    </metadata_provider>

    <!-- Mandatory. The Kafka Connector allows the configuration of different independent connections to different Kafka
         broker/clusters.

         Every single connection is configured via the definition of its own Lightstreamer Data Adapter. At least one connection
         configuration must be provided.

         Since the Kafka Connector manages the physical connection to Kafka by wrapping an internal Kafka Consumer, several
         configuration settings in the Data Adapter are identical to those required by the usual Kafka Consumer
         configuration.

         The Kafka Connector leverages the "name" attribute of the <data_provider> tag as the connection name, which will
         be used by the Clients to request real-time data from this specific Kafka connection through a Subscription object.

         The connection name is also used to group all logging messages belonging to the same connection.

         Its default value is "DEFAULT", but only one "DEFAULT" configuration is permitted. -->
    <data_provider name="QuickStart">
        <!-- ##### GENERAL PARAMETERS ##### -->

        <!-- Java class name of the Kafka Connector Data Adapter. DO NOT EDIT IT. -->
        <adapter_class>com.lightstreamer.kafka.adapters.KafkaConnectorDataAdapter</adapter_class>

        <!-- Optional. Enable this connection configuration. Can be one of the following:
             - true
             - false

             If disabled, Lightstreamer Server will automatically deny every subscription made to this connection.

             Default value: true. -->
        <!--
        <param name="enable">false</param>
        -->

        <!-- Mandatory. The Kafka Cluster bootstrap server endpoint expressed as the list of host/port pairs used to
             establish the initial connection.

             The parameter sets the value of the "bootstrap.servers" key to configure the internal Kafka Consumer.
             See https://kafka.apache.org/documentation/#consumerconfigs_bootstrap.servers for more details.
         -->
        <param name="bootstrap.servers">broker:29092</param>

        <!-- Optional. The name of the consumer group this connection belongs to.

             The parameter sets the value for the "group.id" key used to configure the internal
             Kafka Consumer. See https://kafka.apache.org/documentation/#consumerconfigs_group.id for more details.

             Default value: Adapter Set id + the Data Adapter name + randomly generated suffix. -->
        <!--
        param name="group.id">kafka-connector-group</param>
        -->
        <param name="group.id">quick-start-group</param>

        <!-- ##### ENCRYPTION SETTINGS ##### -->

        <!-- A TCP secure connection to Kafka is configured through parameters with
             the `encryption` prefix. -->

        <!-- Enable encryption -->
        <!-- Optional. Enable encryption of this connection. Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="encryption.enable">true</param>
        -->

        <!-- Optional. The SSL protocol to be used. Can be one of the following:
             - TLSv1.2
             - TLSv1.3

             Default value: TLSv1.3 when running on Java 11 or newer, TLSv1.2 otherwise. -->
        <!--
        <param name="encryption.protocol">TLSv1.2</param>
        -->

        <!-- Optional. The list of enabled secure communication protocols.

             Default value: TLSv1.2,TLSv1.3 when running on Java 11 or newer, `TLSv1.2` otherwise. -->
        <!--
        <param name="encryption.enabled.protocols">TLSv1.3</param>
        -->

        <!--Optional. The list of enabled secure cipher suites.

            Default value: all the available cipher suites in the running JVM. -->
        <!--
        <param name="encryption.cipher.suites">TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA</param>
        -->

        <!-- Optional. Enable hostname verification. Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="encryption.hostname.verification.enable">true</param>
        -->

        <!-- Optional. The path of the trust store file, relative to the deployment folder
             (LS_HOME/adapters/lightstreamer-kafka-connector-<version>). -->
        <!--
        <param name="encryption.truststore.path">secrets/kafka-connector.truststore.jks</param>
        -->

        <!-- Optional. The password of the trust store.

             If not set, checking the integrity of the trust store file configured will not
             be possible. -->
        <!--
        <param name="encryption.truststore.password">kafka-connector-truststore-password</param>
        -->

        <!-- Optional. Enable a key store. Can be one of the following:
             - true
             - false

            A key store is required if the mutual TLS is enabled on Kafka.

            If enabled, the following parameters configure the key store settings:
            - encryption.keystore.path
            - encryption.keystore.password
            - encryption.keystore.key.password

            Defalt value: false. -->
        <!--
        <param name="encryption.keystore.enable">true</param>
        -->

        <!-- Mandatory if key store is enabled. The path of the key store file, relative to
             the deployment folder (LS_HOME/adapters/lightstreamer-kafka-connector-<version>). -->
        <!--
        <param name="encryption.keystore.path">secrets/kafka-connector.keystore.jks</param>
        -->

        <!-- Optional. The password of the key store.

             If not set, checking the integrity of the key store file configured
             will not be possible. -->
        <!--
        <param name="encryption.keystore.password">kafka-connector-password</param>
        -->

        <!-- Optional. The password of the private key in the key store file. -->
        <!--
        <param name="encryption.keystore.key.password">kafka-connector-private-key-password</param>
        -->

        <!-- ##### AUTHENTICATION SETTINGS ##### -->

        <!-- Broker authentication is configured through parameters with the
             `authentication` prefix. -->

        <!-- Optional. Enable the authentication of this connection against the Kafka Cluster.
             Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="authentication.enable">true</param>
        -->

        <!-- Mandatory if authentication is enabled. The SASL mechanism type.
             The Kafka Connector accepts the following authentication mechanisms:

             - PLAIN
             - SCRAM-SHA-256
             - SCRAM-SHA-512
             - GSSAPI

             Default value: PLAIN.-->
        <!--
        <param name="authentication.mechanism">PLAIN</param>
        -->

        <!-- Mandatory if authentication.mechanism is one of PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. The credentials. -->
        <!--
        <param name="authentication.username">authorized-kafka-user</param>
        <param name="authentication.password">authorized-kafka-user-password</param>
        -->

        <!-- ##### GSSAPI AUTHENTICATION SETTINGS ##### -->

        <!-- In the case of GSSAPI authentication mechanism, the following parameters will be part of
             the authentication configuration. -->

        <!-- Optional. Enable the use of a keytab. Can be one of the following:
            - true
            - false

             Default value: false. -->
        <!--
        <param name="authentication.gssapi.key.tab.enable">true</param>
        -->

        <!-- Mandatory if keytab is enabled.  The path to the kaytab file, relative to
             the deployment folder (LS_HOME/adapters/lightstreamer-kafka-connector-<version>). -->
        <!--
        <param name="authentication.gssapi.key.tab.path">gssapi/kafka-connector.keytab</param>
        -->

        <!--  Optional. Enable storage of the principal key. Can be one of the following:
            - true
            - false

            Default value: false- -->
        <!--
        <param name="authentication.gssapi.store.key.enable">true</param>
        -->

        <!-- Mandatory. The name of the Kerberos service. -->
        <!--
        <param name="authentication.gssapi.kerberos.service.name">kafka</param>
        -->

        <!-- Mandatory. if ticket cache is disabled. The name of the principal to be used. -->
        <!--
        <param name="authentication.gssapi.principal">kafka-connector-1@LIGHTSTREAMER.COM</param>
        -->

        <!-- Optional. Enable the use of a ticket cache. Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="authentication.gssapi.ticket.cache.enable">true</param>
        -->

        <!-- ##### RECORD EVALUATION SETTINGS ##### -->

        <!-- Optional. Specifies where to start consuming events from:
             - LATEST: start consuming events from the end of the topic partition
             - EARLIEST: start consuming events from the beginning of the topic partition

             Default value: LATEST. -->
        <param name="record.consume.from">EARLIEST</param>

        <!-- Optional. The number of threads to be used for concurrent processing of the
             incoming deserialized records. If set to `-1`, the number of threads will be automatically
             determined based on the number of available CPU cores.

             Default value: 1. -->
        <!--
        <param name="record.consume.with.num.threads">4</param>
        -->

        <!-- Optional but only effective if "record.consume.with.num.threads" is set to a value greater than 1 (which includes hte default value).
             The order strategy to be used for concurrent processing of the incoming
             deserialized records. Can be one of the following:

             - ORDER_BY_PARTITION: maintain the order of records within each partition
             - ORDER_BY_KEY: maintain the order among the records sharing the same key
             - UNORDERED: provide no ordering guarantees

             Default value: ORDER_BY_PARTITION. -->
        <!--
        <param name="record.consume.with.order.strategy">ORDER_BY_KEY</param>
        -->

        <!-- Optional. The format to be used to deserialize respectively the key and value of a Kafka record.
             Can be one of the following:
             - AVRO
             - JSON
             - STRING
             - INTEGER
             - BOOLEAN
             - BYTE_ARRAY
             - BYTE_BUFFER
             - BYTES
             - DOUBLE
             - FLOAT
             - LONG
             - SHORT
             - UUID

             Default: STRING -->
        <param name="record.key.evaluator.type">INTEGER</param>
        <param name="record.value.evaluator.type">JSON</param>

        <!-- Mandatory if evaluator type is AVRO and the Confluent Schema Registry is disabled. The path of the local schema
             file relative to the deployment folder (LS_HOME/adapters/lightstreamer-kafka-connector-<version>) for
             message validation respectively of the key and the value. -->
        <!--
        <param name="record.key.evaluator.schema.path">schema/record_key.avsc</param>
        <param name="record.value.evaluator.schema.path">schemas/record_value.avsc</param>
        -->

        <!-- Mandatory if evaluator type is AVRO and no local schema paths are specified. Enable the use of the Confluent Schema Registry for validation respectively of the key and
             value. Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="record.key.evaluator.schema.registry.enable">true</param>
        <param name="record.value.evaluator.schema.registry.enable">true</param>
        -->

        <!-- Optional. The error handling strategy to be used if an error occurs while extracting data from incoming
             deserialized records.
             Can be one of the following:
             - IGNORE_AND_CONTINUE: ignore the error and continue to process the next record
             - FORCE_UNSUBSCRIPTION: stop processing records and force unsubscription of the items
                                     requested by all the Lightstreamer clients subscribed to this connection

             Default: "IGNORE_AND_CONTINUE". -->
        <!--
        <param name="record.extraction.error.strategy">FORCE_UNSUBSCRIPTION</param>
        -->

        <!-- ##### RECORD ROUTING SETTINGS ##### -->

        <!-- Multiple and Optional. Define an item template expression, which is made of:
             - ITEM_PREFIX: the prefix of the item name
             - BINDABLE_EXPRESSIONS: a sequence of bindable extraction expressions. See documentation at:
             https://github.com/lightstreamer/Lightstreamer-kafka-connector?tab=readme-ov-file#filtered-record-routing-item-templatetemplate_name
        -->
        <!--
        <param name="item-template.TEMPLATE_NAME">ITEM_PREFIX-BINDABLE_EXPRESSIONS</param>
        -->
        <param name="item-template.stock">stock-#{index=KEY}</param>

        <!-- Multiple and mandatory. Map the Kafka topic TOPIC_NAME to:
             - one or more simple items
             - one or more item templates
             - any combination of the above

             At least one mapping must be provided. -->
        <!-- Example 1:
        <param name="map.aTopicName.to">item1,item2,itemN,...</param>
        -->
        <!-- Example 2:
        <param name="map.aTopicName.to">item-itemplate.template-name1,item-template.template-name2...</param>
        -->
        <!-- Example 3:
        <param name="map.aTopicName.to">item-itemplate.template-name1,item1,item-template.template-name2,item2,...</param>
        -->
        <param name="map.stocks.to">item-template.stock</param>

        <!-- ##### RECORD MAPPING SETTINGS ##### -->

        <!-- Multiple and Mandatory. Map the value extracted through "extraction_expression" to
             field FIELD_NAME. The expression is written in the Data Extraction Language. See documentation at:
             https://github.com/lightstreamer/Lightstreamer-kafka-connector?tab=readme-ov-file#record-mapping-fieldfield_name

             At least one mapping must be provided. -->
        <!--
        <param name="field.FIELD_NAME">extraction_expression</param>
        -->
        <param name="field.timestamp">#{VALUE.timestamp}</param>
        <param name="field.time">#{VALUE.time}</param>
        <param name="field.stock_name">#{VALUE.name}</param>
        <param name="field.last_price">#{VALUE.last_price}</param>
        <param name="field.ask">#{VALUE.ask}</param>
        <param name="field.ask_quantity">#{VALUE.ask_quantity}</param>
        <param name="field.bid">#{VALUE.bid}</param>
        <param name="field.bid_quantity">#{VALUE.bid_quantity}</param>
        <param name="field.pct_change">#{VALUE.pct_change}</param>
        <param name="field.min">#{VALUE.min}</param>
        <param name="field.max">#{VALUE.max}</param>
        <param name="field.ref_price">#{VALUE.ref_price}</param>
        <param name="field.open_price">#{VALUE.open_price}</param>
        <param name="field.item_status">#{VALUE.item_status}</param>
        <param name="field.ts">#{TIMESTAMP}</param>
        <param name="field.topic">#{TOPIC}</param>
        <param name="field.offset">#{OFFSET}</param>
        <param name="field.partition">#{PARTITION}</param>

        <!-- ##### SCHEMA REGISTRY SETTINGS ##### -->

        <!-- Mandatory if the Confluent Schema Registry is enabled. The URL of the Confluent Schema Registry.
             An encrypted connection is enabled by specifying the "https" protocol. -->
        <!--
        <param name="schema.registry.url">https://schema-registry:8084</param>
        -->

        <!-- Optional. Enable Basic HTTP authentication of this connection against the Schema Registry. Can be one of the following:
             - true
             - false

             Default value: false. -->
        <!--
        <param name="schema.registry.basic.authentication.enable">true</param>
        -->

        <!-- Mandatory if Basic HTTP authentication is enabled. The credentials. -->
        <!--
        <param name="schema.registry.basic.authentication.username">authorized-schema-registry-user</param>
        <param name="schema.registry.basic.authentication.password">authorized-schema-registry-user-password</param>
        -->

        <!-- The following parameters have the same meaning as the homologous ones defined in
             the ENCRYPTION SETTINGS section. -->

        <!-- Set general encryption settings -->
        <!--
        <param name="schema.registry.encryption.enabled.protocols">TLSv1.3</param>
        <param name="schema.registry.encryption.cipher.suites">TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA</param>
        <param name="schema.registry.encryption.hostname.verification.enable">true</param>
        -->

        <!-- If required, configure the trust store to trust the Confluent Schema Registry certificates -->
        <!--
        <param name="schema.registry.encryption.truststore.path">secrets/kafka-connector.truststore.jks</param>
        <param name="schema.registry.encryption.truststore.password">kafka-connector-truststore-password</param>
        -->

        <!-- If mutual TLS is enabled on the Confluent Schema Registry, enable and configure the key store -->
        <!--
        <param name="schema.registry.encryption.keystore.enable">true</param>
        <param name="schema.registry.encryption.keystore.path">secrets/kafka-connector.keystore.jks</param>
        <param name="schema.registry.encryption.keystore.password">kafka-connector-password</param>
        <param name="schema.registry.encryption.keystore.key.password">kafka-connector-private-key-password</param>
        -->
    </data_provider>

    <data_provider name="QuickStartConfluentCloud">
        <!-- ##### GENERAL PARAMETERS ##### -->

        <adapter_class>com.lightstreamer.kafka.adapters.KafkaConnectorDataAdapter</adapter_class>

        <param name="bootstrap.servers">broker:29092</param>
        <param name="group.id">quick-start-group-confluent</param>

        <!-- ##### ENCRYPTION SETTINGS ##### -->
        <param name="encryption.enable">true</param>
        <param name="encryption.protocol">TLSv1.2</param>
        <param name="encryption.hostname.verification.enable">true</param>

        <!-- ##### AUTHENTICATION SETTINGS ##### -->
        <param name="authentication.enable">true</param>
        <param name="authentication.mechanism">PLAIN</param>
        <param name="authentication.username">API.key</param>
        <param name="authentication.password">API.secret</param>

        <!-- ##### RECORD EVALUATION SETTINGS ##### -->
        <param name="record.consume.from">EARLIEST</param>
        <param name="record.key.evaluator.type">INTEGER</param>
        <param name="record.value.evaluator.type">JSON</param>

        <!-- ##### RECORD ROUTING SETTINGS ##### -->
        <param name="item-template.stock">stock-#{index=KEY}</param>
        <param name="map.stocks.to">item-template.stock</param>

        <!-- ##### RECORD MAPPING SETTINGS ##### -->
        <param name="field.timestamp">#{VALUE.timestamp}</param>
        <param name="field.time">#{VALUE.time}</param>
        <param name="field.stock_name">#{VALUE.name}</param>
        <param name="field.last_price">#{VALUE.last_price}</param>
        <param name="field.ask">#{VALUE.ask}</param>
        <param name="field.ask_quantity">#{VALUE.ask_quantity}</param>
        <param name="field.bid">#{VALUE.bid}</param>
        <param name="field.bid_quantity">#{VALUE.bid_quantity}</param>
        <param name="field.pct_change">#{VALUE.pct_change}</param>
        <param name="field.min">#{VALUE.min}</param>
        <param name="field.max">#{VALUE.max}</param>
        <param name="field.ref_price">#{VALUE.ref_price}</param>
        <param name="field.open_price">#{VALUE.open_price}</param>
        <param name="field.item_status">#{VALUE.item_status}</param>
        <param name="field.ts">#{TIMESTAMP}</param>
        <param name="field.topic">#{TOPIC}</param>
        <param name="field.offset">#{OFFSET}</param>
        <param name="field.partition">#{PARTITION}</param>

    </data_provider>

</adapters_conf>