From ab8f2bd4f754c2e1f07b48c7cc5fa06f0f0705f1 Mon Sep 17 00:00:00 2001 From: Kentaro Hayashi Date: Thu, 15 Feb 2024 14:20:51 +0900 Subject: [PATCH] Guard launching duplicated fluentd instance with same configuration Before: If you launch multiple Fluentd instance with same configuration file, it causes a disaster with inconsistent processed buffer or pos file. After: Detect fluentd service's main process and fetch FLUENT_CONF. if configuration is same as spawned process, abort it. It can block the following conditions are met: * fluentd is launched as a systemd service. configuration file is specified via FLUENT_CONF in fluentd.service. * manually try to launch fluentd with same configuration file like this: sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf Note that following case doen't resolved. sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf Thus running fluentd service and manually try to launch normal user case can't be detected. NOTE: Windows is out of scope in this PR. Signed-off-by: Kentaro Hayashi --- fluent-package/Rakefile | 15 ++- .../apt/systemd-test/update-from-v4.sh | 4 + ...ext-version-with-backward-compat-for-v4.sh | 4 + .../systemd-test/update-to-next-version.sh | 4 + fluent-package/templates/conflict_detector.rb | 96 +++++++++++++++++++ fluent-package/templates/usr/sbin/fluentd.erb | 2 + .../yum/systemd-test/update-from-v4.sh | 4 + ...ext-version-with-backward-compat-for-v4.sh | 4 + .../systemd-test/update-to-next-version.sh | 4 + 9 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 fluent-package/templates/conflict_detector.rb diff --git a/fluent-package/Rakefile b/fluent-package/Rakefile index 759830c8e..ebab7324e 100755 --- a/fluent-package/Rakefile +++ b/fluent-package/Rakefile @@ -83,6 +83,10 @@ def macos? /darwin/ =~ RUBY_PLATFORM end +def linux? + /linux/ =~ RUBY_PLATFORM +end + def ensure_directory(dirname) mkdir_p(dirname) unless File.exist?(dirname) if block_given? @@ -421,6 +425,13 @@ class BuildTask # for fat gems which depend on nonexistent libraries # mainly for nokogiri 1.11 or later on CentOS 6 rebuild_gems + + # Patch against generated file + if linux? + sh('sed', '--in-place', + '--expression', '/^if Gem.respond_to?/i load "/opt/fluent/share/conflict_detector.rb"', + File.join(staging_bindir, "fluentd")) + end end desc "Install fluentd" @@ -1099,11 +1110,13 @@ class BuildTask configs.concat([ "etc/logrotate.d/#{SERVICE_NAME}", fluentd_conf_default, - ]) unless windows? || macos? + "opt/#{PACKAGE_DIR}/share/conflict_detector.rb"]) unless windows? || macos? configs.each do |config| src = if config == fluentd_conf_default template_path(fluentd_conf) + elsif config == "opt/#{PACKAGE_DIR}/share/conflict_detector.rb" + template_path("conflict_detector.rb") else template_path(config) end diff --git a/fluent-package/apt/systemd-test/update-from-v4.sh b/fluent-package/apt/systemd-test/update-from-v4.sh index 14657a05e..045670b7c 100755 --- a/fluent-package/apt/systemd-test/update-from-v4.sh +++ b/fluent-package/apt/systemd-test/update-from-v4.sh @@ -66,6 +66,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh index b8436f773..53c4f753c 100755 --- a/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh +++ b/fluent-package/apt/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh @@ -63,6 +63,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/apt/systemd-test/update-to-next-version.sh b/fluent-package/apt/systemd-test/update-to-next-version.sh index 6e8ee1ebb..3dae8ee3a 100755 --- a/fluent-package/apt/systemd-test/update-to-next-version.sh +++ b/fluent-package/apt/systemd-test/update-to-next-version.sh @@ -44,6 +44,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo apt remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/templates/conflict_detector.rb b/fluent-package/templates/conflict_detector.rb new file mode 100644 index 000000000..7882c2558 --- /dev/null +++ b/fluent-package/templates/conflict_detector.rb @@ -0,0 +1,96 @@ +require 'optparse' +module Fluent + class ConflictDetector + def self.linux? + /linux/ === RUBY_PLATFORM + end + + def self.running_fluentd_conf + # Detect if same configuration is used + unless linux? + return nil + end + + IO.popen(["/usr/bin/ps", "-e", "-o", "uid,pid,ppid,cmd"]) do |_io| + _io.readlines.each do |line| + uid, pid, ppid, cmd = line.split(' ', 4) + # skip self and supervisor process + next if Process.pid == pid.to_i or Process.pid == ppid.to_i + if cmd and cmd.chomp.include?("fluentd") and ppid.to_i == 1 + # under systemd control + File.open("/proc/#{pid.to_i}/environ") do |file| + conf = file.read.split("\u0000").select { |entry| entry.include?("FLUENT_CONF") } + return conf.first.split('=').last unless conf.empty? + end + end + end + end + return nil + end + end + + class FakeOptionParser < OptionParser + attr_reader :config_path + def initialize + @config_path = nil + @opt = OptionParser.new + @opt.on('-c', '--config VAL') { |v| + @config_path = v + } + @opt.on('-s', '--setup DIR') + @opt.on('--dry-run') + @opt.on('--show-plugin-config=PLUGIN') + @opt.on('-p', '--plugin DIR') + @opt.on('-I PATH') + @opt.on('-r NAME') + @opt.on('-d', '--daemon PIDFILE') + @opt.on('--under-supervisor') + @opt.on('--no-supervisor') + @opt.on('--workers NUM') + @opt.on('--user USER') + @opt.on('--group GROUP') + @opt.on('--umask UMASK') + @opt.on('-o', '--log PATH') + @opt.on('--log-rotate-age AGE') + @opt.on('--log-rotate-size BYTES') + @opt.on('--log-event-verbose') + @opt.on('-i', '--inline-config CONFIG_STRING') + @opt.on('-h', '--help') + end + + def parse(argv) + @opt.parse(argv) + end + + end +end + +begin + running_fluentd_conf = Fluent::ConflictDetector.running_fluentd_conf + if ENV["FLUENT_CONF"] and ENV["FLUENT_CONF"] == running_fluentd_conf + # /usr/sbin/fluentd sets FLUENT_CONF=/etc/fluent/fluentd.conf by default + # If it matches with running other instance, abort it + puts "Error: can't start duplicate Fluentd instance with same #{ENV['FLUENT_CONF']}" + exit 2 + else + # /opt/fluent/bin/fluentd does not set FLUENT_CONF=/etc/fluent/fluentd.conf, + # if -c option is given from command line, check and abort it. + unless ARGV.empty? + # preflight with dummy parser + opt = Fluent::FakeOptionParser.new + begin + opt.parse(ARGV) + if opt.config_path and opt.config_path == running_fluentd_conf + puts "Error: can't start duplicate Fluentd instance with same #{running_fluentd_conf}" + exit 2 + end + rescue + end + end + end +rescue Errno::EACCES + # e.g. unprivileged access error, can't detect duplicated instance from command line parameter. +rescue Errno::ENOENT + # e.g. can't detect duplicated instance from ps. +end + diff --git a/fluent-package/templates/usr/sbin/fluentd.erb b/fluent-package/templates/usr/sbin/fluentd.erb index 8e3833a42..011ddb986 100755 --- a/fluent-package/templates/usr/sbin/fluentd.erb +++ b/fluent-package/templates/usr/sbin/fluentd.erb @@ -12,4 +12,6 @@ if ARGV.include?("--version") puts "fluent-package #{PACKAGE_VERSION} fluentd #{Fluent::VERSION} (#{FLUENTD_REVISION})" exit 0 end + +load "<%= install_path %>/share/conflict_detector.rb" load "<%= install_path %>/bin/fluentd" diff --git a/fluent-package/yum/systemd-test/update-from-v4.sh b/fluent-package/yum/systemd-test/update-from-v4.sh index acdc7d027..fa56f43b7 100755 --- a/fluent-package/yum/systemd-test/update-from-v4.sh +++ b/fluent-package/yum/systemd-test/update-from-v4.sh @@ -77,6 +77,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package sudo systemctl daemon-reload diff --git a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh index a077a9e0b..d18bf960b 100755 --- a/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh +++ b/fluent-package/yum/systemd-test/update-to-next-version-with-backward-compat-for-v4.sh @@ -97,6 +97,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package (! systemctl status --no-pager td-agent) diff --git a/fluent-package/yum/systemd-test/update-to-next-version.sh b/fluent-package/yum/systemd-test/update-to-next-version.sh index c661384e8..4e0da9986 100755 --- a/fluent-package/yum/systemd-test/update-to-next-version.sh +++ b/fluent-package/yum/systemd-test/update-to-next-version.sh @@ -68,6 +68,10 @@ sleep 3 test -e /var/log/fluent/fluentd.log (! grep -q -e '\[warn\]' -e '\[error\]' -e '\[fatal\]' /var/log/fluent/fluentd.log) +# Test: Guard duplicated instance +(! sudo /usr/sbin/fluentd -c /etc/fluent/fluentd.conf) +(! sudo /opt/fluent/bin/fluentd -c /etc/fluent/fluentd.conf) + # Uninstall sudo $DNF remove -y fluent-package sudo systemctl daemon-reload