From f860f532343588c4d88978827ab9e801366fe775 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Thu, 15 Feb 2024 14:20:51 +0900 Subject: [PATCH] Guard launching duplicated fluentd instance with same configuration Concept; Inject blocking code to /usr/sbin/fluentd. Before: If you launch multiple Fluentd instance with same configuration file, it causes a disaster with inconsistent processed buffer or pos file. After: Detect fluentd service's main process and fetch FLUENT_CONF. if configuration is same as spawned process, abort it. It can block the following conditions are met: * fluentd is launched as a systemd service. configuration file is specified via FLUENT_CONF in fluentd.service. * manually try to launch fluentd with same configuration file like this: sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf sudo /usr/sbin/fluentd sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf sudo /opt/fluent/bin/fluentd Note that following case doen't resolved. * /usr/sbin/fluentd -c /etc/fluent/fluentd.conf * /usr/sbin/fluentd * /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf * /opt/fluent/bin/fluentd This is because there is no appropriate privilege to retrieve command line parameters. NOTE: Windows is out of scope in this PR. Signed-off-by: Kentaro Hayashi --- fluent-package/Rakefile | 15 ++- .../apt/systemd-test/update-from-v4.sh | 4 + ...ext-version-with-backward-compat-for-v4.sh | 4 + .../systemd-test/update-to-next-version.sh | 4 + fluent-package/templates/conflict_detector.rb | 103 ++++++++++++++++++ fluent-package/templates/usr/sbin/fluentd.erb | 1 + .../yum/systemd-test/install-newly.sh | 1 + .../yum/systemd-test/update-from-v4.sh | 5 + ...ext-version-with-backward-compat-for-v4.sh | 5 + .../systemd-test/update-to-next-version.sh | 5 + 10 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 fluent-package/templates/conflict_detector.rb diff --git a/fluent-package/Rakefile b/fluent-package/Rakefile index 5b1b37912..bcb9dc026 100755 --- a/fluent-package/Rakefile +++ b/fluent-package/Rakefile @@ -83,6 +83,10 @@ def macos? /darwin/ =~ RUBY_PLATFORM end +def linux? + /linux/ =~ RUBY_PLATFORM +end + def ensure_directory(dirname) mkdir_p(dirname) unless File.exist?(dirname) if block_given? @@ -421,6 +425,13 @@ class BuildTask # for fat gems which depend on nonexistent libraries # mainly for nokogiri 1.11 or later on CentOS 6 rebuild_gems + + # Patch against generated file + if linux? + sh('sed', '--in-place', + '--expression', '/^if Gem.respond_to?/i load "/opt/fluent/share/conflict_detector.rb"', + File.join(staging_bindir, "fluentd")) + end end desc "Install fluentd" @@ -1108,11 +1119,13 @@ class BuildTask configs.concat([ "etc/logrotate.d/#{SERVICE_NAME}", fluentd_conf_default, - ]) unless windows? || macos? + "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"]) unless windows? || macos? configs.each do |config| src = if config == fluentd_conf_default template_path(fluentd_conf) + elsif config == "opt/#{PACKAGE_DIR}/share/conflict_detector.rb" + template_path("conflict_detector.rb") else template_path(config) end diff --git a/fluent-package/apt/systemd-test/update-from-v4.sh b/fluent-package/apt/systemd-test/update-from-v4.sh index e0973d221..b690afd60 100755 --- a/fluent-package/apt/systemd-test/update-from-v4.sh +++ b/fluent-package/apt/systemd-test/update-from-v4.sh @@ -79,6 +79,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh index 762e70bda..b4ece24c4 100755 --- a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh +++ b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh @@ -75,6 +75,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/apt/systemd-test/update-to-next-version.sh b/fluent-package/apt/systemd-test/update-to-next-version.sh index 6e8ee1ebb..3dae8ee3a 100755 --- a/fluent-package/apt/systemd-test/update-to-next-version.sh +++ b/fluent-package/apt/systemd-test/update-to-next-version.sh @@ -44,6 +44,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/templates/conflict_detector.rb b/fluent-package/templates/conflict_detector.rb new file mode 100644 index 000000000..e23fe594b --- /dev/null +++ b/fluent-package/templates/conflict_detector.rb @@ -0,0 +1,103 @@ +require 'optparse' +module Fluent + class ConflictDetector + def self.linux? + /linux/ === RUBY_PLATFORM + end + + def self.under_systemd?(pid) + pid == 1 or @pidmap[@pidmap[pid]] == 1 + end + + def self.running_fluentd_conf + unless linux? + return nil + end + + @pidmap = {} + IO.popen(["/usr/bin/ps", "-e", "-o", "uid,pid,ppid,cmd"]) do |_io| + _io.readlines.each do |line| + uid, pid, ppid, cmd = line.split(' ', 4) + # skip current running process + next if Process.pid == pid.to_i + @pidmap[pid.to_i] = ppid.to_i + if cmd and cmd.chomp.include?("fluentd") and under_systemd?(pid.to_i) + # check only under systemd control + File.open("/proc/#{pid.to_i}/environ") do |file| + conf = file.read.split("\u0000").select { |entry| entry.include?("FLUENT_CONF") } + return conf.first.split('=').last unless conf.empty? + end + end + end + end + return nil + end + end + + class FakeOptionParser < OptionParser + attr_reader :config_path + def initialize + @config_path = nil + @opt = OptionParser.new + @opt.on('-c', '--config VAL') { |v| + @config_path = v + } + @opt.on('-s', '--setup DIR') + @opt.on('--dry-run') + @opt.on('--show-plugin-config=PLUGIN') + @opt.on('-p', '--plugin DIR') + @opt.on('-I PATH') + @opt.on('-r NAME') + @opt.on('-d', '--daemon PIDFILE') + @opt.on('--under-supervisor') + @opt.on('--no-supervisor') + @opt.on('--workers NUM') + @opt.on('--user USER') + @opt.on('--group GROUP') + @opt.on('--umask UMASK') + @opt.on('-o', '--log PATH') + @opt.on('--log-rotate-age AGE') + @opt.on('--log-rotate-size BYTES') + @opt.on('--log-event-verbose') + @opt.on('-i', '--inline-config CONFIG_STRING') + @opt.on('-h', '--help') + end + + def parse(argv) + @opt.parse(argv) + end + + end +end + +begin + running_fluentd_conf = Fluent::ConflictDetector.running_fluentd_conf + unless Fluent::ConflictDetector.under_systemd?(Process.pid) + if ENV["FLUENT_CONF"] and ENV["FLUENT_CONF"] == running_fluentd_conf + # /usr/sbin/fluentd sets FLUENT_CONF=/etc/fluent/fluentd.conf by default + # If it matches with running other instance, abort it + puts "Error: can't start duplicate Fluentd instance with same #{ENV['FLUENT_CONF']}" + exit 2 + else + # /opt/fluent/bin/fluentd does not set FLUENT_CONF=/etc/fluent/fluentd.conf, + # if -c option is given from command line, check and abort it. + unless ARGV.empty? + # preflight with dummy parser + opt = Fluent::FakeOptionParser.new + begin + opt.parse(ARGV) + if opt.config_path and opt.config_path == running_fluentd_conf + puts "Error: can't start duplicate Fluentd instance with same #{running_fluentd_conf}" + exit 2 + end + rescue + end + end + end + end +rescue Errno::EACCES + # e.g. unprivileged access error, can't detect duplicated instance from command line parameter. +rescue Errno::ENOENT + # e.g. can't detect duplicated instance from ps. +end + diff --git a/fluent-package/templates/usr/sbin/fluentd.erb b/fluent-package/templates/usr/sbin/fluentd.erb index 8e3833a42..c5e811554 100755 --- a/fluent-package/templates/usr/sbin/fluentd.erb +++ b/fluent-package/templates/usr/sbin/fluentd.erb @@ -12,4 +12,5 @@ if ARGV.include?("--version") puts "fluent-package #{PACKAGE_VERSION} fluentd #{Fluent::VERSION} (#{FLUENTD_REVISION})" exit 0 end + load "<%= install_path %>/bin/fluentd" diff --git a/fluent-package/yum/systemd-test/install-newly.sh b/fluent-package/yum/systemd-test/install-newly.sh index a698f08c4..ad315c88a 100755 --- a/fluent-package/yum/systemd-test/install-newly.sh +++ b/fluent-package/yum/systemd-test/install-newly.sh @@ -41,6 +41,7 @@ systemctl status --no-pager fluentd sleep 3 test -e /var/log/fluent/fluentd.log +cat /var/log/fluent/fluentd.log (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) sudo $DNF remove -y fluent-package diff --git a/fluent-package/yum/systemd-test/update-from-v4.sh b/fluent-package/yum/systemd-test/update-from-v4.sh index e59508f8a..21764b3c4 100755 --- a/fluent-package/yum/systemd-test/update-from-v4.sh +++ b/fluent-package/yum/systemd-test/update-from-v4.sh @@ -89,8 +89,13 @@ test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock" # (v4 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs) sleep 3 test -e /var/log/fluent/fluentd.log +cat /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package sudo systemctl daemon-reload diff --git a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh index 123df58af..f874679ca 100755 --- a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh +++ b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh @@ -108,8 +108,13 @@ test -h /usr/sbin/td-agent-gem # (v4 default config outputs 'warn' log, so we should check only 'error' and 'fatal' logs) sleep 3 test -e /var/log/fluent/fluentd.log +cat /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/yum/systemd-test/update-to-next-version.sh b/fluent-package/yum/systemd-test/update-to-next-version.sh index c661384e8..1bfa1045e 100755 --- a/fluent-package/yum/systemd-test/update-to-next-version.sh +++ b/fluent-package/yum/systemd-test/update-to-next-version.sh @@ -66,8 +66,13 @@ test $(eval $env_vars && echo $FLUENT_SOCKET) = "/var/run/fluent/fluentd.sock" # Test: logs sleep 3 test -e /var/log/fluent/fluentd.log +cat /var/log/fluent/fluentd.log (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package sudo systemctl daemon-reload