first commit

.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--format doc

Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

README.md

@@ -0,0 +1,32 @@
+# fluent-plugin-fw1_loggrabber_parser
+
+Parsing a LEA format file from FW1-LogGrabber.
+https://github.com/certego/fw1-loggrabber
+
+A separator of a LEA format file should be '|'(0x7c).
+
+
+## Installation
+
+```bash
+# for fluentd
+gem install fluent-plugin-fw1_loggrabber_parser
+
+# for td-agent2
+td-agent-gem install fluent-plugin-fw1_loggrabber_parser
+```
+
+## Usage
+
+```xml
+<source>
+  @type tail
+  path /var/log//fw1.log
+  pos_file /var/log/td-agent/fw1.log.pos
+  tag fw1.log
+  format fw1_loggrabber
+</source>
+```
+
+## parameters
+- n/a

Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

fluent-plugin-fw1_loggrabber_parser.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/parser_fw1_loggrabber/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-fw1_loggrabber_parser"
+  spec.version       = Fluent::Plugin::Fw1LoggrabberParser::VERSION
+  spec.authors       = ["Tomoyuki Sugimura"]
+  spec.email         = ["[email protected]"]
+
+  spec.summary       = %q{parse checkpoint firewall-1 LEA formatted log}
+  spec.description   = %q{parse checkpoint firewall-1 LEA formatted log from file}
+  spec.homepage      = "https://localhost.localdomain"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  #if spec.respond_to?(:metadata)
+  #  spac.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  #else
+  #  raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  #end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_development_dependency "fluentd", "~> 0.10", ">= 0.10.43"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "test-unit"
+end

lib/fluent/plugin/parser_fw1_loggrabber.rb

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+module Fluent
+  class TextParser
+
+    class Fw1LoggrabberParser < Parser
+
+      # Register this parser as a parser plugin
+      Plugin.register_parser('fw1_loggrabber', self)
+
+      # This method is called after config_params have read configuration parameter
+      def initialize
+        super
+        @pattern_key_value = /(?<=^|[^\\]\|)([^=\s]+)=((?:[^|]|(?:(?<=\\)\|))+)/
+      end
+
+      def configure(conf={})
+        super
+      end
+
+      def parse(text)
+        record = logparse(text)
+        yield Engine.now, record
+      end
+
+
+      def logparse(text)
+
+        return {} if (nil == text)
+
+        record = Hash.new
+
+        begin
+          for pair in text.scan(@pattern_key_value) do
+            record[pair[0]] = pair[1]
+          end
+        rescue => e
+          log.error e.message
+          return {}
+        end
+
+        return record
+      end
+
+    end
+
+  end
+
+end

lib/fluent/plugin/parser_fw1_loggrabber/version.rb

@@ -0,0 +1,7 @@
+module Fluent
+  module Plugin
+    module Fw1LoggrabberParser
+      VERSION = "1.0.0"
+    end
+  end
+end

spec/helper.rb

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems."
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval { |obj|
+    def method_missing(method, *args)
+      #pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/test'
+require 'fluent/plugin/parser_fw1_loggrabber'

spec/parser_fw1_loggrabber_spec.rb

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+require 'helper'
+
+class TestFw1LoggrabberParser
+  RSpec.describe Fluent::TextParser::Fw1LoggrabberParser do
+
+    CONFIG = %[]
+
+    def create_driver(conf=CONFIG, tag='test')
+      Fluent::Test::ParserTestDriver.new(Fluent::TextParser::Fw1LoggrabberParser, tag).configure(conf)
+    end
+
+    def create_parser
+      @parser = Fluent::TextParser::Fw1LoggrabberParser.new
+    end
+
+    before :all do
+      Fluent::Test.setup
+    end
+
+    before :each do
+      @log = 'time=2016-04-18 17:11:20|action=drop|orig=192.168.0.155|i/f_dir=inbound|i/f_name=eth1|has_accounting=0|uuid=<00000000,00000000,00000000,00000000>|product=VPN-1 & FireWall-1|rule=4|rule_uid={6F779801F-9827-4084-A513-8C6094CCDF29}|src=192.168.0.90|s_port=54438|dst=224.0.0.252|service=5355|proto=udp|__policy_id_tag=product=VPN-1 & FireWall-1[db_tag={EFBF0C12-72BB-5542-909E-6976080402E5};mgmt=mgmt_4400;date=1460945227;policy_name=Standard]|origin_sic_name=cn=cp_mgmt,o=4400..wewbq8'
+      @log_with_escaped_separator = 'time=2016-04-18 17:11:20|action=drop|orig=192.168.0.155|i/f_dir=inbound|i/f_name=eth1|has_accounting=0|uuid=<00000000,00000000,00000000,00000000>|product=VPN-1 & FireWall-1|rule=4|rule_uid={6F779801F-9827-4084-A513-8C6094CCDF29}|src=192.168.0.90|s_port=54438|dst=224.0.0.252|service=5355|proto=udp|__policy_id_tag=product=VPN-1 & FireWall-1[db_tag={EFBF0C12-72BB-5542-909E-6976080402E5};mgmt=mgmt_4400;date=1460945227;policy_name=Standard\|Irregular]|origin_sic_name=cn=cp_mgmt,o=4400..wewbq8'
+
+      @driver = create_driver
+      @parser = create_parser()
+
+    end
+
+
+
+
+
+    context "#parse" do
+      it "returns record because of good text" do
+        text = @log
+        expect_result = {
+          '__policy_id_tag' => 'product=VPN-1 & FireWall-1[db_tag={EFBF0C12-72BB-5542-909E-6976080402E5};mgmt=mgmt_4400;date=1460945227;policy_name=Standard]',
+          'action'          => 'drop',
+          'dst'             => '224.0.0.252',
+          'has_accounting'  => '0',
+          'i/f_dir'         => 'inbound',
+          'i/f_name'        => 'eth1',
+          'orig'            => '192.168.0.155',
+          'origin_sic_name' => 'cn=cp_mgmt,o=4400..wewbq8',
+          'product'         => 'VPN-1 & FireWall-1',
+          'proto'           => 'udp',
+          'rule'            => '4',
+          'rule_uid'        => '{6F779801F-9827-4084-A513-8C6094CCDF29}',
+          's_port'          => '54438',
+          'service'         => '5355',
+          'src'             => '192.168.0.90',
+          'time'            => '2016-04-18 17:11:20',
+          'uuid'            => '<00000000,00000000,00000000,00000000>',
+        }
+        result = @driver.parse(text) { |time, record|
+          record
+        }
+        expect(result).to match(expect_result)
+      end
+
+      it "returns record with escaped separator because of good text" do
+        text = @log_with_escaped_separator
+        expect_result = {
+          '__policy_id_tag' => 'product=VPN-1 & FireWall-1[db_tag={EFBF0C12-72BB-5542-909E-6976080402E5};mgmt=mgmt_4400;date=1460945227;policy_name=Standard\|Irregular]',
+          'action'          => 'drop',
+          'dst'             => '224.0.0.252',
+          'has_accounting'  => '0',
+          'i/f_dir'         => 'inbound',
+          'i/f_name'        => 'eth1',
+          'orig'            => '192.168.0.155',
+          'origin_sic_name' => 'cn=cp_mgmt,o=4400..wewbq8',
+          'product'         => 'VPN-1 & FireWall-1',
+          'proto'           => 'udp',
+          'rule'            => '4',
+          'rule_uid'        => '{6F779801F-9827-4084-A513-8C6094CCDF29}',
+          's_port'          => '54438',
+          'service'         => '5355',
+          'src'             => '192.168.0.90',
+          'time'            => '2016-04-18 17:11:20',
+          'uuid'            => '<00000000,00000000,00000000,00000000>',
+        }
+        result = @driver.parse(text) { |time, record|
+          record
+        }
+        expect(result).to match(expect_result)
+      end
+
+      it "returns empty record because of empty text" do
+        text = ""
+        expect_result = {}
+        result = @driver.parse(text) { |time, record|
+          record
+        }
+        expect(result).to match(expect_result)
+      end
+
+      it "returns empty record because of text is nil" do
+        text = nil
+        expect_result = {}
+        result = @driver.parse(text) { |time, record|
+          record
+        }
+        expect(result).to match(expect_result)
+      end
+
+    end
+
+  end
+
+end

spec/spec_helper.rb

@@ -0,0 +1,96 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end