diff --git a/config/processors/syslog_security_skyhigh.swg.conf b/config/processors/syslog_security_skyhigh.swg.conf index 724c86ee..41f619f6 100644 --- a/config/processors/syslog_security_skyhigh.swg.conf +++ b/config/processors/syslog_security_skyhigh.swg.conf @@ -8,100 +8,92 @@ input { } } filter { + # mutate { + remove_field => [ "host","event" ] add_field => { "[event][module]" => "skyhigh" } add_field => { "[event][dataset]" => "skyhigh.swg" } - copy => { "message" => "[event][original]" } + copy => { "message" => "[event][original]" } + gsub => [ + "message", "connection type=(.*?), ssl", "connection type=\1, ssl" + ] } grok { - tag_on_failure => "_parsefailure_header" - match => { "message" => "(^(.*?)(<(?\d+)>)(\s)?(?.*$))|(^(?.*)$)" } + match => { "message" => "^(.*?)(<(?\d+)>)(\s)?.*?mwg:( )?(?.*?)$" } timeout_millis => 500 - } syslog_pri { syslog_pri_field_name => "pri" remove_field => [ "pri" ] + ecs_compatibility => v8 } - ### If regular MWG traffic log - if [message] =~ "mprob=" { - dissect { - tag_on_failure => "_dissectfailure_2" - mapping => { - "actual_msg" => '%{?data} ts=%{[[event][created]]}, sip=%{[[source][ip]]}, usr=%{[[user][name]]}, sprt=%{[[source][port]]}, stat=%{[[http][response][status_code]]}, cat=%{[[rule][category]]}, sev=%{[[event][severity_name]]}, media=%{[[http][request][body][content]]}, rbytes=%{[[http][response][bytes]]}, sbytes=%{[[http][request][bytes]]}, agent=%{[[user_agent][original]]}, virus=%{[[rule][name]]}, mprob=%{[[event][risk_score]]}, blockid=%{[[rule][id]]}, block=%{[[rule][ruleset]]}, app=%{[[network][application]]}, dip=%{[[destination][ip]]}, dprt=%{[[destination][port]]}, sslcertserialclient=%{[[tls][client][certificate]]}, sslcipherclient=%{[[tls][client][supported_ciphers]]}, sslversionclient=%{[[tls][client][x509][version_number]]}, sslcnsrvr=%{[[tls][server][issuer]]}, sslsha1digestsrvr=%{[[tls][server][hash][sha1]]}, sslsha2digestsrvr=%{[[tls][server][hash][sha256]]}, sslsigmethodserver=%{[[tls][server][x509][signature_algorithm]]}, sslciphersrvrt=%{[tls][cipher]}, sslversionsrvr=%{[[tls][version]]}, rule=%{[rule][uuid]}, method=%{tmp}' - } + if [tmp_csv] !~ "\w,\w" { + kv { + source => "tmp_csv" + target => "tmp" + field_split_pattern => ",( | $)" + value_split => "=" + recursive => "true" + trim_key => " " + trim_value => " " } - } else if [message] =~ "method=" { - dissect { - tag_on_failure => "_dissectfailure_3" - mapping => { - "actual_msg" => '%{?data} ts=%{[[event][created]]}, sip=%{[[source][ip]]}, usr=%{[[user][name]]}, sprt=%{[[source][port]]}, stat=%{[[http][response][status_code]]}, cat=%{[[rule][category]]}, sev=%{[[event][severity_name]]}, media=%{[[http][request][body][content]]}, rbytes=%{[[http][response][bytes]]}, sbytes=%{[[http][request][bytes]]}, agent=%{[[user_agent][original]]}, virus=%{[[rule][name]]}, blockid=%{[[rule][id]]}, block=%{[[rule][ruleset]]}, app=%{[[network][application]]}, dip=%{[[destination][ip]]}, dprt=%{[[destination][port]]}, sslcertserialclient=%{[[tls][client][certificate]]}, sslcipherclient=%{[[tls][client][supported_ciphers]]}, sslversionclient=%{[[tls][client][x509][version_number]]}, sslcnsrvr=%{[[tls][server][issuer]]}, sslsha1digestsrvr=%{[[tls][server][hash][sha1]]}, sslsha2digestsrvr=%{[[tls][server][hash][sha256]]}, sslsigmethodserver=%{[[tls][server][x509][signature_algorithm]]}, sslciphersrvrt=%{[tls][cipher]}, sslversionsrvr=%{[[tls][version]]}, rule=%{[rule][uuid]}, method=%{tmp}' - } + } else { + kv { + source => "tmp_csv" + target => "tmp" + field_split => "," + value_split => "=" + recursive => "true" + trim_key => " " + trim_value => " " } - ### MWG error logs - } else if [message] =~ "Severity: " { - mutate { - gsub => ["message",'[\"]',","] - } - dissect { - tag_on_failure => "_dissectfailure_4" - mapping => { - "actual_msg" => "%{?data} %{?data} %{?data} %{ob[server][address]]} %{rest_msg}" - } - } - if [rest_msg] =~ "user" { - dissect { - tag_on_failure => "_dissectfailure_5" - mapping => { - rest_msg => "%{?data},%{?data},%{?data},%{[[error][message]]},%{?data},%{?data} ,%{[[user][name]]}, (%{[[source][ip]]}),%{?data},Severity: %{[[log][level]]}" - } - } - } else { - dissect { - tag_on_failure => "_dissectfailure_6" - mapping => { - rest_msg => "%{?data},%{?data},%{?data},%{[[event][reason]]},%{?data},%{[[error][message]]},%{?data},Severity: %{[[log][level]]}" - } - } - } - } - mutate { - gsub => [ - "[event][created]", "\[", "", - "[event][created]", "\]", "" - ] } - if [tmp] !~ "ref=.*?$" { + if [tmp_csv] !~ "ref=.*?$" { mutate { # identify long uri i.e. possible DNS exfiltration add_tag => "long uri" } } - grok { - match => { "tmp" => "^(?<[http][request][method]>.*?) (?<[url][full]>.*?)( |$)((?<[tls][next_protocol]>.*?))?(,|)( ref=(?<[http][request][referrer]>.*?))?( |)$"} - tag_on_failure => "_grokparsefailure_url" - timeout_millis => 500 - } - - if [event][created] { - date { - # "26/aug/2020:19:35:09.533 +0000" - # ts=[12/oct/2020:17:24:01 +0000] - match => ["[event][created]","MMM dd HH:mm:ss","ISO8601","dd/MMM/yyyy:HH:mm:ss ZZ" ] - timezone => "GMT" - locale => "en" - target => "[event][created]" - tag_on_failure => "_dateparsefailure_ec" - } - if "_dateparsefailure" in [tags] { - mutate { - remove_field => ["[event][created]"] - } + mutate { + rename => { + "[tmp][usr]" => "[user][name]" + "[tmp][app]" => "[process][name]" + "[tmp][block]" => "[rule][ruleset]" + "[tmp][rbytes]" => "[http][request][bytes]" + "[tmp][sprt]" => "[source][port]" + "[tmp][rule]" => "[rule][uuid]" + "[tmp][stat]" => "[http][response][status_code]" + "[tmp][agent]" => "[user_agent][original]" + "[tmp][sbytes]" => "[http][response][bytes]" + "[tmp][blockid]" => "[rule][id]" + "[tmp][sip]" => "[source][ip]" + "[tmp][sev]" => "[event][severity_name]" + "[tmp][ref]" => "[http][request][referrer]" + "[tmp][dip]" => "[destination][ip]" + "[tmp][cat]" => "[rule][category]" + "[tmp][ts]" => "[event][created]" + "[tmp][dprt]" => "[destination][port]" + "[tmp][media]" => "[http][response][mime_type]" + "[tmp][sslsigmethodserver]" => "[tls][server][x509][signature_algorithm]" + "[tmp][sslciphersrvrt]" => "[tls][cipher]" + "[tmp][sslversionsrvr]" => "[tls][version]" + "[tmp][sslsha2digestsrvr]" => "[tls][server][hash][sha256]" + "[tmp][sslsha1digestsrvr]" => "[tls][server][hash][sha1]" + "[tmp][sslcnsrvr]" => "[tls][server][issuer]" + "[tmp][sslcipherclient]" => "[tls][client][supported_ciphers]" + "[tmp][sslversionclient]" => "[tls][client][x509][version_number]" + "[tmp][sslcertserialclient]" => "[tls][client][x509][serial_number]" + "[tmp][mprob]" => "[event][risk_score]" + "[tmp][virus]" => "[rule][name]" + "[tmp][ver]" => "[tls][next_protocol]" + "[tmp][url]" => "[observer][ip]" } } - - mutate { - remove_field => [ "actual_msg", "tmp", "rest_msg"] + # URI + grok { + match => { "[tmp][method]" => "^(?<[http][request][method]>.*?) (?<[url][full]>.*?)( |$)((?<[tls][next_protocol]>.*?).*$)?" } + tag_on_failure => "_grokparsefailure_uri" + timeout_millis => 500 } translate { @@ -161,6 +153,26 @@ filter { add_field => { "[event][action]" => "denied" } } } + + if [event][created] { + date { + # "26/aug/2020:19:35:09.533 +0000" + # ts=[12/oct/2020:17:24:01 +0000] + match => ["[event][created]", "ISO8601","MMM dd HH:mm:ss","dd/MMM/yyyy:HH:mm:ss ZZ" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + } + if [http][request][referrer] == "," { + mutate { + remove_field => ["[http][request][referrer]"] + } + } + mutate { + remove_field => ["tmp", "tmp_csv"] + } } output { pipeline { send_to => [enrichments] }