diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml index be57d9324..fb118d72b 100644 --- a/.github/workflows/apt.yml +++ b/.github/workflows/apt.yml @@ -88,6 +88,15 @@ jobs: --volume ${PWD}:/fluentd:ro \ ${{ matrix.test-docker-image }} \ /fluentd/fluent-package/apt/serverspec-test.sh + - name: Confluent Test + run: | + mkdir -p .bundle + docker run \ + --rm \ + --tty \ + --volume ${PWD}:/fluentd:ro \ + ${{ matrix.test-docker-image }} \ + /fluentd/fluent-package/apt/confluent-test.sh - name: Binstubs Test run: | mkdir -p .bundle diff --git a/fluent-package/apt/confluent-test.sh b/fluent-package/apt/confluent-test.sh new file mode 100755 index 000000000..f1f3584eb --- /dev/null +++ b/fluent-package/apt/confluent-test.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -exu + +export DEBIAN_FRONTEND=noninteractive + +apt update +apt install -V -y lsb-release + +. $(dirname $0)/commonvar.sh + +apt install -V -y \ + ${repositories_dir}/${distribution}/pool/${code_name}/${channel}/*/*/*_${architecture}.deb + +fluentd --version + +case ${code_name} in + xenial) + apt install -V -y gnupg2 wget apt-transport-https + ;; + *) + apt install -V -y gnupg2 wget + ;; +esac + +/usr/sbin/fluent-gem install --no-document serverspec +wget https://packages.confluent.io/deb/7.6/archive.key +gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key +chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg +echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/7.6 stable main" > /etc/apt/sources.list.d/confluent.list +apt update && apt install -y confluent-community-2.13 ${java_jdk} netcat-openbsd + +export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok +/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties & +N_POLLING=30 +n=1 +while true ; do + sleep 1 + status=$(echo ruok | nc localhost 2181) + if [ "$status" = "imok" ]; then + break + fi + n=$((n + 1)) + if [ $n -ge $N_POLLING ]; then + echo "failed to get response from zookeeper-server" + exit 1 + fi +done +/usr/bin/kafka-server-start /etc/kafka/server.properties & +n=1 +while true ; do + sleep 1 + status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p) + if [ "$status" = "[0]" ]; then + break + fi + n=$((n + 1)) + if [ $n -ge $N_POLLING ]; then + echo "failed to get response from kafka-server" + exit 1 + fi +done +/usr/bin/kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test +export PATH=/opt/fluent/bin:$PATH +cd /fluentd && rake serverspec:kafka +/usr/sbin/fluentd -c /fluentd/serverspec/test.conf & diff --git a/fluent-package/apt/serverspec-test.sh b/fluent-package/apt/serverspec-test.sh index 8bfe9d6c7..9258d6df6 100755 --- a/fluent-package/apt/serverspec-test.sh +++ b/fluent-package/apt/serverspec-test.sh @@ -14,54 +14,7 @@ apt install -V -y \ fluentd --version -case ${code_name} in - xenial) - apt install -V -y gnupg2 wget apt-transport-https - ;; - *) - apt install -V -y gnupg2 wget - ;; -esac - -/usr/sbin/fluent-gem install --no-document serverspec -wget https://packages.confluent.io/deb/6.0/archive.key -gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key -chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg -echo "deb [arch=${architecture} signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list -apt update && apt install -y confluent-community-2.13 ${java_jdk} netcat-openbsd - -export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok -/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties & -N_POLLING=30 -n=1 -while true ; do - sleep 1 - status=$(echo ruok | nc localhost 2181) - if [ "$status" = "imok" ]; then - break - fi - n=$((n + 1)) - if [ $n -ge $N_POLLING ]; then - echo "failed to get response from zookeeper-server" - exit 1 - fi -done -/usr/bin/kafka-server-start /etc/kafka/server.properties & -n=1 -while true ; do - sleep 1 - status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p) - if [ "$status" = "[0]" ]; then - break - fi - n=$((n + 1)) - if [ $n -ge $N_POLLING ]; then - echo "failed to get response from kafka-server" - exit 1 - fi -done -/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test -/usr/sbin/fluentd -c /fluentd/serverspec/test.conf & export PATH=/opt/fluent/bin:$PATH export INSTALLATION_TEST=true +/usr/sbin/fluent-gem install --no-document serverspec cd /fluentd && rake serverspec:linux diff --git a/serverspec/kafka/confluent.rb b/serverspec/kafka/confluent.rb new file mode 100644 index 000000000..4fdcc5d0d --- /dev/null +++ b/serverspec/kafka/confluent.rb @@ -0,0 +1,31 @@ +require_relative "../spec_helper" +require "rdkafka" + +describe "rdkafka", :if => !centos8?(os) do + it "can receive message via Rdkafka client" do + config = { + "bootstrap.servers": "localhost:9092", + "group.id": "test" + } + consumer = Rdkafka::Config.new(config).consumer + consumer.subscribe("test") + + wait_for_consumer_assignment(consumer) + + `echo "Hello, rdkafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test` + + message = consumer.each { |message| break message } + expect(message.payload).to eq "Hello, rdkafka" + end +end + +describe "fluent-plugin-kafka", :if => !centos8?(os) do + it "can receive message via fluent-plugin-kafka" do + `echo "Hello, fluent-plugin-kafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test` + Dir.glob("/tmp/log/td-agent/*.log") do |path| + File.open(path) do |file| + expect(JSON.parse(file.readlines.last)["message"]).to eq "Hello, fluent-plugin-kafka" + end + end + end +end diff --git a/serverspec/linux/td-agent.rb b/serverspec/linux/td-agent.rb index a50dd3772..321e7bb1c 100644 --- a/serverspec/linux/td-agent.rb +++ b/serverspec/linux/td-agent.rb @@ -1,5 +1,4 @@ require_relative "../spec_helper" -require "rdkafka" require "json" require "bundler" @@ -40,32 +39,3 @@ end end end - -describe "rdkafka", :if => !centos8?(os) do - it "can receive message via Rdkafka client" do - config = { - "bootstrap.servers": "localhost:9092", - "group.id": "test" - } - consumer = Rdkafka::Config.new(config).consumer - consumer.subscribe("test") - - wait_for_consumer_assignment(consumer) - - `echo "Hello, rdkafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test` - - message = consumer.each { |message| break message } - expect(message.payload).to eq "Hello, rdkafka" - end -end - -describe "fluent-plugin-kafka", :if => !centos8?(os) do - it "can receive message via fluent-plugin-kafka" do - `echo "Hello, fluent-plugin-kafka" | /usr/bin/kafka-console-producer --broker-list localhost:9092 --topic test` - Dir.glob("/tmp/log/td-agent/*.log") do |path| - File.open(path) do |file| - expect(JSON.parse(file.readlines.last)["message"]).to eq "Hello, fluent-plugin-kafka" - end - end - end -end