From a7c850744c9bc1ac2664b73d310d5b24a4565ed6 Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Thu, 19 Nov 2015 19:29:25 +0100 Subject: [PATCH 1/3] add metadata to the events appendend to the pending queue in case of a non match --- lib/logstash/filters/multiline.rb | 2 +- spec/filters/multiline_spec.rb | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/logstash/filters/multiline.rb b/lib/logstash/filters/multiline.rb index 8691401..24c4610 100644 --- a/lib/logstash/filters/multiline.rb +++ b/lib/logstash/filters/multiline.rb @@ -228,7 +228,7 @@ def previous_filter!(event, match) # this line is not part of the previous event if we have a pending event, it's done, send it. # put the current event into pending unless pending.empty? - tmp = event.to_hash + tmp = event.to_hash_with_metadata event.overwrite(merge(pending)) pending.clear # avoid array creation pending << LogStash::Event.new(tmp) diff --git a/spec/filters/multiline_spec.rb b/spec/filters/multiline_spec.rb index a7f311b..84253c7 100644 --- a/spec/filters/multiline_spec.rb +++ b/spec/filters/multiline_spec.rb @@ -245,4 +245,23 @@ end end + + describe "keeps metadata fields after two consecutive non multline lines" do + config <<-CONFIG + filter { + mutate { add_field => { "[@metadata][index]" => "logstash-2015.11.19" } } + multiline { + pattern => "^%{NUMBER}" + what => "previous" + } + } + CONFIG + + sample ["line1", "line2"] do + expect(subject).to be_a(Array) + expect(subject[0]["@metadata"]).to include("index"=>"logstash-2015.11.19") + expect(subject[1]["@metadata"]).to include("index"=>"logstash-2015.11.19") + end + end + end From 3e887f001ba5818e65863220f95f07e35f4bb6db Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Tue, 24 Nov 2015 17:08:08 +0100 Subject: [PATCH 2/3] fix metadata handling when using event.ovewrite as its not coping the necessary metadata fields --- lib/logstash/filters/multiline.rb | 10 +++++++--- spec/filters/multiline_spec.rb | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/logstash/filters/multiline.rb b/lib/logstash/filters/multiline.rb index 24c4610..1da4c50 100644 --- a/lib/logstash/filters/multiline.rb +++ b/lib/logstash/filters/multiline.rb @@ -228,8 +228,10 @@ def previous_filter!(event, match) # this line is not part of the previous event if we have a pending event, it's done, send it. # put the current event into pending unless pending.empty? - tmp = event.to_hash_with_metadata - event.overwrite(merge(pending)) + tmp = event.to_hash_with_metadata + merged_events = merge(pending) + event.overwrite(merged_events) + event["@metadata"] = merged_events["@metadata"] # Override does not copy the metadata pending.clear # avoid array creation pending << LogStash::Event.new(tmp) else @@ -253,7 +255,9 @@ def next_filter!(event, match) # if we have something in pending, join it with this message and send it. # otherwise, this is a new message and not part of multiline, send it. unless pending.empty? - event.overwrite(merge(pending << event)) + merged_events = merge(pending << event) + event.overwrite(merged_events) + event["@metadata"] = merged_events["@metadata"] # Override does not copy the metadata pending.clear end end # if match diff --git a/spec/filters/multiline_spec.rb b/spec/filters/multiline_spec.rb index 84253c7..8ac829e 100644 --- a/spec/filters/multiline_spec.rb +++ b/spec/filters/multiline_spec.rb @@ -254,6 +254,7 @@ pattern => "^%{NUMBER}" what => "previous" } + mutate { add_field => { "[@metadata][type]" => "foo" } } } CONFIG @@ -261,6 +262,29 @@ expect(subject).to be_a(Array) expect(subject[0]["@metadata"]).to include("index"=>"logstash-2015.11.19") expect(subject[1]["@metadata"]).to include("index"=>"logstash-2015.11.19") + expect(subject[0]["@metadata"]).to include("type"=>"foo") + expect(subject[1]["@metadata"]).to include("type"=>"foo") + end + end + + describe "keeps metadata fields after two consecutive non multline lines" do + config <<-CONFIG + filter { + mutate { add_field => { "[@metadata][index]" => "logstash-2015.11.19" } } + multiline { + pattern => "^%{NUMBER}" + what => "next" + } + mutate { add_field => { "[@metadata][type]" => "foo" } } + } + CONFIG + + sample ["line1", "line2"] do + expect(subject).to be_a(Array) + expect(subject[0]["@metadata"]).to include("index"=>"logstash-2015.11.19") + expect(subject[1]["@metadata"]).to include("index"=>"logstash-2015.11.19") + expect(subject[0]["@metadata"]).to include("type"=>"foo") + expect(subject[1]["@metadata"]).to include("type"=>"foo") end end From 9e5798a414cc65740f233154d8bdf256ab306f3a Mon Sep 17 00:00:00 2001 From: Pere Urbon-Bayes Date: Thu, 3 Dec 2015 17:48:47 +0100 Subject: [PATCH 3/3] version 1.1.0 bump updated the logstash core requirement to < 2.0.0.alpha0 --- CHANGELOG.md | 2 ++ Gemfile | 2 +- logstash-filter-multiline.gemspec | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29..acd51fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +## 1.1.0 + - Fix metadata handling, fixes #19 and #22 diff --git a/Gemfile b/Gemfile index d926697..851fabc 100644 --- a/Gemfile +++ b/Gemfile @@ -1,2 +1,2 @@ source 'https://rubygems.org' -gemspec \ No newline at end of file +gemspec diff --git a/logstash-filter-multiline.gemspec b/logstash-filter-multiline.gemspec index d33c7ed..c37989a 100644 --- a/logstash-filter-multiline.gemspec +++ b/logstash-filter-multiline.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-multiline' - s.version = '1.0.0' + s.version = '1.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "This filter will collapse multiline messages from a single source into one Logstash event." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -20,7 +20,7 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } # Gem dependencies - s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0' + s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0.alpha0' s.add_runtime_dependency 'logstash-patterns-core' s.add_runtime_dependency 'logstash-filter-mutate' s.add_runtime_dependency 'jls-grok', '~> 0.11.0'