ci deb: extract confluent test case

In the previous versions, testing with confluent is executed in serverspec:linux. As confluent test case should be out of scope about basic serverspec to distinct failure, so extract them as isolated test case. * github ci: added "Confluent Test" * Upgrade confluent from 6.0 to 7.6 * Replace deprecated --zookeeper option to --bootstrap-server to create kafka topics Signed-off-by: Kentaro Hayashi <[email protected]>
fluent · Apr 11, 2024 · a1a334a · a1a334a
1 parent c5133a4
commit a1a334a
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 78 deletions.
diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml
@@ -88,6 +88,15 @@ jobs:
           --volume ${PWD}:/fluentd:ro \
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/serverspec-test.sh
+      - name: Confluent Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/confluent-test.sh
       - name: Binstubs Test
         run: |
           mkdir -p .bundle

diff --git a/fluent-package/apt/confluent-test.sh b/fluent-package/apt/confluent-test.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -exu
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt update
+apt install -V -y lsb-release
+
+. $(dirname $0)/commonvar.sh
+
+apt install -V -y \
+  ${repositories_dir}/${distribution}/pool/${code_name}/${channel}/*/*/*_${architecture}.deb
+
+fluentd --version
+
+case ${code_name} in
+    xenial)
+	apt install -V -y gnupg2 wget apt-transport-https
+	;;
+    *)
+	apt install -V -y gnupg2 wget
+	;;
+esac
+
+/usr/sbin/fluent-gem install --no-document serverspec
+wget https://packages.confluent.io/deb/7.6/archive.key
+gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
+chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
+echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/7.6 stable main" > /etc/apt/sources.list.d/confluent.list
+apt update && apt install -y confluent-community-2.13 ${java_jdk} netcat-openbsd
+
+export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
+/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
+N_POLLING=30
+n=1
+while true ; do
+    sleep 1
+    status=$(echo ruok | nc localhost 2181)
+    if [ "$status" = "imok" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from zookeeper-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-server-start /etc/kafka/server.properties &
+n=1
+while true ; do
+    sleep 1
+    status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
+    if [ "$status" = "[0]" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from kafka-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
+export PATH=/opt/fluent/bin:$PATH
+cd /fluentd && rake serverspec:kafka
+/usr/sbin/fluentd -c /fluentd/serverspec/test.conf &
diff --git a/fluent-package/apt/serverspec-test.sh b/fluent-package/apt/serverspec-test.sh
@@ -14,54 +14,7 @@ apt install -V -y \
 
 fluentd --version
 
-case ${code_name} in
-    xenial)
-	apt install -V -y gnupg2 wget apt-transport-https
-	;;
-    *)
-	apt install -V -y gnupg2 wget
-	;;
-esac
-
-/usr/sbin/fluent-gem install --no-document serverspec
-wget https://packages.confluent.io/deb/6.0/archive.key
-gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
-chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
-echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list
-apt update && apt install -y confluent-community-2.13 ${java_jdk} netcat-openbsd
-
-export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
-/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
-N_POLLING=30
-n=1
-while true ; do
-    sleep 1
-    status=$(echo ruok | nc localhost 2181)
-    if [ "$status" = "imok" ]; then
-	break
-    fi
-    n=$((n + 1))
-    if [ $n -ge $N_POLLING ]; then
-	echo "failed to get response from zookeeper-server"
-	exit 1
-    fi
-done
-/usr/bin/kafka-server-start /etc/kafka/server.properties &
-n=1
-while true ; do
-    sleep 1
-    status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
-    if [ "$status" = "[0]" ]; then
-	break
-    fi
-    n=$((n + 1))
-    if [ $n -ge $N_POLLING ]; then
-	echo "failed to get response from kafka-server"
-	exit 1
-    fi
-done
-/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
-/usr/sbin/fluentd -c /fluentd/serverspec/test.conf &
 export PATH=/opt/fluent/bin:$PATH
 export INSTALLATION_TEST=true
+/usr/sbin/fluent-gem install --no-document serverspec
 cd /fluentd && rake serverspec:linux
diff --git a/serverspec/kafka/confluent.rb b/serverspec/kafka/confluent.rb
@@ -0,0 +1,31 @@
+require_relative "../spec_helper"
+require "rdkafka"
+
+describe "rdkafka", :if => !centos8?(os) do
+  it "can receive message via Rdkafka client" do
+    config = {
+      "bootstrap.servers": "localhost:9092",
+      "group.id": "test"
+    }
+    consumer = Rdkafka::Config.new(config).consumer
+    consumer.subscribe("test")
+
+    wait_for_consumer_assignment(consumer)
+
+    `echo "Hello, rdkafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test`
+
+    message = consumer.each { |message| break message }
+    expect(message.payload).to eq "Hello, rdkafka"
+  end
+end
+
+describe "fluent-plugin-kafka", :if => !centos8?(os) do
+  it "can receive message via fluent-plugin-kafka" do
+    `echo "Hello, fluent-plugin-kafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test`
+    Dir.glob("/tmp/log/td-agent/*.log") do |path|
+      File.open(path) do |file|
+        expect(JSON.parse(file.readlines.last)["message"]).to eq "Hello, fluent-plugin-kafka"
+      end
+    end
+  end
+end
diff --git a/serverspec/linux/td-agent.rb b/serverspec/linux/td-agent.rb
@@ -1,5 +1,4 @@
 require_relative "../spec_helper"
-require "rdkafka"
 require "json"
 require "bundler"
 
@@ -40,32 +39,3 @@
     end
   end
 end
-
-describe "rdkafka", :if => !centos8?(os) do
-  it "can receive message via Rdkafka client" do
-    config = {
-      "bootstrap.servers": "localhost:9092",
-      "group.id": "test"
-    }
-    consumer = Rdkafka::Config.new(config).consumer
-    consumer.subscribe("test")
-
-    wait_for_consumer_assignment(consumer)
-
-    `echo "Hello, rdkafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test`
-
-    message = consumer.each { |message| break message }
-    expect(message.payload).to eq "Hello, rdkafka"
-  end
-end
-
-describe "fluent-plugin-kafka", :if => !centos8?(os) do
-  it "can receive message via fluent-plugin-kafka" do
-    `echo "Hello, fluent-plugin-kafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test`
-    Dir.glob("/tmp/log/td-agent/*.log") do |path|
-      File.open(path) do |file|
-        expect(JSON.parse(file.readlines.last)["message"]).to eq "Hello, fluent-plugin-kafka"
-      end
-    end
-  end
-end