ci: run python hstream-kafka examples

.github/workflows/ci.yaml

@@ -121,7 +121,27 @@ jobs:
           ./tests/wait_hserver.sh
           sleep 5
 
-      - name: Run examples
+      - name: Run hstream examples
         run: |
           python3 -m pip install hstreamdb
           python3 examples/py/snippets/guides.py
+
+  test-python-kafka-examples:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Start cluster
+        run: |
+          docker compose -f ./assets/quick-start-kafka-lite.yaml up -d
+          sleep 5
+          ./tests/wait_hstream_kafka.sh
+          sleep 5
+
+      - name: Run hstream kafka examples
+        run: |
+          python3 -m pip install kafka-python
+          python3 kafka-examples/python/snippets/kafka_python.py
+
+          python3 -m pip install confluent-kafka
+          python3 kafka-examples/python/snippets/confluent_kafka_python.py

assets/quick-start-kafka-lite.yaml

@@ -0,0 +1,74 @@
+version: "3.5"
+
+services:
+  hserver:
+    image: hstreamdb/hstream:latest
+    depends_on:
+      - zookeeper
+      - hstore
+    ports:
+      - "127.0.0.1:9092:9092"
+    expose:
+      - 9092
+    networks:
+      - hstream-quickstart
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp:/tmp
+      - data_store:/data/store
+    command:
+      - bash
+      - "-c"
+      - |
+        set -e
+        /usr/local/script/wait-for-storage.sh hstore 6440 zookeeper 2181 600 \
+        /usr/local/bin/hstream-server kafka \
+          --bind-address 0.0.0.0 --port 9092 \
+          --gossip-port 6571 \
+          --server-id 100 \
+          --seed-nodes "$$(hostname -I | awk '{print $$1}'):6571" \
+          --advertised-address $$(hostname -I | awk '{print $$1}') \
+          --metastore-uri zk://zookeeper:2181 \
+          --store-config /data/store/logdevice.conf \
+          --store-log-level warning
+
+  hstore:
+    image: hstreamdb/hstream:latest
+    networks:
+      - hstream-quickstart
+    volumes:
+      - data_store:/data/store
+    command:
+      - bash
+      - "-c"
+      - |
+        set -ex
+        # N.B. "enable-dscp-reflection=false" is required for linux kernel which
+        # doesn't support dscp reflection, e.g. centos7.
+        /usr/local/bin/ld-dev-cluster --root /data/store \
+        --use-tcp --tcp-host $$(hostname -I | awk '{print $$1}') \
+        --user-admin-port 6440 \
+        --param enable-dscp-reflection=false \
+        --no-interactive
+
+  zookeeper:
+    image: zookeeper:3.7
+    expose:
+      - 2181
+    networks:
+      - hstream-quickstart
+    volumes:
+      - data_zk_data:/data
+      - data_zk_datalog:/datalog
+
+networks:
+  hstream-quickstart:
+    name: hstream-quickstart
+
+volumes:
+  data_store:
+    name: quickstart_data_store
+  data_zk_data:
+    name: quickstart_data_zk_data
+  data_zk_datalog:
+    name: quickstart_data_zk_datalog

tests/wait_hstream_kafka.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+timeout=120
+container_name=wait_hserver
+
+docker run -td --rm --name $container_name --network host hstreamdb/hstream bash
+until (docker exec -t $container_name hstream-kafka-cli --port 9092 node status); do
+    >&2 echo "Waiting for 127.0.0.1:9092 ..."
+    sleep 1
+    timeout=$((timeout - 1))
+    [ $timeout -le 0 ] && echo "Timeout!" && docker kill $container_name && exit 1;
+done
+docker kill $container_name