From dcce88c54096846f13d4590c4f0f4e2f955d8402 Mon Sep 17 00:00:00 2001 From: Brian Grabau Date: Mon, 13 May 2024 11:07:27 -0500 Subject: [PATCH 1/2] added parsing for pipeline health test message --- .../processors/api_security_skyhigh.scp.conf | 187 +++++++++--------- 1 file changed, 95 insertions(+), 92 deletions(-) diff --git a/config/processors/api_security_skyhigh.scp.conf b/config/processors/api_security_skyhigh.scp.conf index 0ff069ce..6068b0ec 100644 --- a/config/processors/api_security_skyhigh.scp.conf +++ b/config/processors/api_security_skyhigh.scp.conf @@ -16,99 +16,102 @@ filter { if ![message] or [message] == "" { drop {} } - csv { - source => "message" - columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ] - convert => { - "[destination][bytes]" => "integer" - "[source][bytes]" => "integer" - } - skip_empty_columns => true - skip_empty_rows => true - } - # row empty - if "_csvskippedemptyfield" in [tags] { - drop {} - } + if ![message] =~ "^test message for.*?$" { + csv { + source => "message" + columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ] + convert => { + "[destination][bytes]" => "integer" + "[source][bytes]" => "integer" + } + skip_empty_columns => true + skip_empty_rows => true + } + # row empty + if "_csvskippedemptyfield" in [tags] { + drop {} + } - mutate { - split => { "[rule][category]" => ", " } - } - # Message is csv, this creates [event][original] to include fields names - mutate { - add_field => { "[event][original]" => "request_timestamp_epoch: %{[event][time]}, num: %{num}, usr: %{usr}, source.nat.ip: %{[source][ip]}, http.request.method: %{[http][request][method]}, destination.bytes: %{[destination][bytes]}, source.bytes: %{[source][bytes]}, url.domain: %{[url][domain]}, url.path: %{[url][path]}, event.action: %{[event][action]}, rule.name: %{[rule][name]}, request_timestamp_epoch: %{request_timestamp_epoch}, url.scheme: %{[url][scheme]}, rule.category: %{[rule][category]}, http.request.body.content: %{[http][request][body][content]}, service.name: %{[network][application]}, event.severity_name: %{[event][severity_name]}, last_rule: %{[rule][uuid]}, http_status_code: %{[http][response][status_code]}, client_ip: %{[source][nat][ip]}, location: %{[rule][description]}, block_reason: %{[rule][ruleset]}, user_agent_comment: %{[user_agent][version]}, user_agent_product: %{[user_agent][name]}, user_agent_version: %{[user_agent][original]}, process_name: %{[process][name]}, destination_ip: %{[destination][ip]}, destination_port: %{[destination][port]}, event.risk_score: %{[event][risk_score]}, discarded_host: %{discarded_host}, tls.client.x509.version_number: %{[tls][client][x509][version_number]}, tls.version: %{[tls][version]}" } - } - mutate { - gsub => [ "[event][original]", "%\{.*?}(,)? ", "" ] - gsub => [ "[event][original]", "%\{.*?}", "" ] - } - - if [usr] and [usr] =~ ".*?\\.*?" { - grok { - match => { "usr" => "(?<[user][domain]>.*?)\\(?<[user][name]>.*?)$" } - timeout_millis => 500 - } - } else { - mutate { - rename => { "[usr]" => "[user][name]" } - } - } - if [source][ip] and [source][ip] =~ "source_ip" { - mutate { - remove_field => ["[source][ip]"] - } - } - # Create [url][full] - mutate { - add_field => { "[url][full]" => "%{[[url][scheme]]}://%{[[url][domain]]}%{[[url][path]]}" } - } - date { - match => [ "request_timestamp_epoch", "UNIX" ] - timezone => "GMT" - locale => "en" - target => "[event][created]" - } - mutate { - add_field => { "[cloud][provider]" => "mcafee" } - #add_field => { "[event][module]" => "mcafee" } - #add_field => { "[event][dataset]" => "mcafee.mcp" } - add_field => { "[log][source][hostname]" => "api_mcp"} - } - mutate { - lowercase => [ "[event][action]" ] - } - if [event][action] =~ "observed" { - mutate { - replace => { "[event][action]" => "allowed" } - } - } - if [event][action] =~ "denied" { - mutate { - replace => { "[event][action]" => "denied" } - } - } - mutate { - add_field => { "[service][state]" => "ssl_scanned: %{ssl_scanned}" } - } - if [av_scanned_up] { - mutate { - add_field => { "[service][state]" => "av_scanned_up: %{av_scanned_up}" } - } - } - if [av_scanned_down] { - mutate { - add_field => { "[service][state]" => "av_scanned_down: %{av_scanned_down}" } - } - } - if [rbi] { - mutate { - add_field => { "[service][state]" => "rbi: %{rbi}" } - } - } - if [dlp] { - mutate { - add_field => { "[service][state]" => "dlp: %{dlp}" } - } + mutate { + split => { "[rule][category]" => ", " } + } + # Message is csv, this creates [event][original] to include fields names + mutate { + add_field => { "[event][original]" => "request_timestamp_epoch: %{[event][time]}, num: %{num}, usr: %{usr}, source.nat.ip: %{[source][ip]}, http.request.method: %{[http][request][method]}, destination.bytes: %{[destination][bytes]}, source.bytes: %{[source][bytes]}, url.domain: %{[url][domain]}, url.path: %{[url][path]}, event.action: %{[event][action]}, rule.name: %{[rule][name]}, request_timestamp_epoch: %{request_timestamp_epoch}, url.scheme: %{[url][scheme]}, rule.category: %{[rule][category]}, http.request.body.content: %{[http][request][body][content]}, service.name: %{[network][application]}, event.severity_name: %{[event][severity_name]}, last_rule: %{[rule][uuid]}, http_status_code: %{[http][response][status_code]}, client_ip: %{[source][nat][ip]}, location: %{[rule][description]}, block_reason: %{[rule][ruleset]}, user_agent_comment: %{[user_agent][version]}, user_agent_product: %{[user_agent][name]}, user_agent_version: %{[user_agent][original]}, process_name: %{[process][name]}, destination_ip: %{[destination][ip]}, destination_port: %{[destination][port]}, event.risk_score: %{[event][risk_score]}, discarded_host: %{discarded_host}, tls.client.x509.version_number: %{[tls][client][x509][version_number]}, tls.version: %{[tls][version]}" } + } + mutate { + gsub => [ "[event][original]", "%\{.*?}(,)? ", "" ] + gsub => [ "[event][original]", "%\{.*?}", "" ] + } + + if [usr] and [usr] =~ ".*?\\.*?" { + grok { + match => { "usr" => "(?<[user][domain]>.*?)\\(?<[user][name]>.*?)$" } + timeout_millis => 500 + } + } else { + mutate { + rename => { "[usr]" => "[user][name]" } + } + } + if [source][ip] and [source][ip] =~ "source_ip" { + mutate { + remove_field => ["[source][ip]"] + } + } + # Create [url][full] + mutate { + add_field => { "[url][full]" => "%{[[url][scheme]]}://%{[[url][domain]]}%{[[url][path]]}" } + } + date { + match => [ "request_timestamp_epoch", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + } + mutate { + add_field => { "[cloud][provider]" => "mcafee" } + #add_field => { "[event][module]" => "mcafee" } + #add_field => { "[event][dataset]" => "mcafee.mcp" } + add_field => { "[log][source][hostname]" => "api_mcp"} + } + mutate { + lowercase => [ "[event][action]" ] + } + if [event][action] =~ "observed" { + mutate { + replace => { "[event][action]" => "allowed" } + } + } + if [event][action] =~ "denied" { + mutate { + replace => { "[event][action]" => "denied" } + } + } + mutate { + add_field => { "[service][state]" => "ssl_scanned: %{ssl_scanned}" } + } + if [av_scanned_up] { + mutate { + add_field => { "[service][state]" => "av_scanned_up: %{av_scanned_up}" } + } + } + if [av_scanned_down] { + mutate { + add_field => { "[service][state]" => "av_scanned_down: %{av_scanned_down}" } + } + } + if [rbi] { + mutate { + add_field => { "[service][state]" => "rbi: %{rbi}" } + } + } + if [dlp] { + mutate { + add_field => { "[service][state]" => "dlp: %{dlp}" } + } + } + } mutate { # host field is added by file input plugin From be55c68599e8b9b1f3e071c44dccd192e9ff3b50 Mon Sep 17 00:00:00 2001 From: Brian Grabau Date: Mon, 13 May 2024 11:21:08 -0500 Subject: [PATCH 2/2] fixed conditional --- config/processors/api_security_skyhigh.scp.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/processors/api_security_skyhigh.scp.conf b/config/processors/api_security_skyhigh.scp.conf index 6068b0ec..7a6b760d 100644 --- a/config/processors/api_security_skyhigh.scp.conf +++ b/config/processors/api_security_skyhigh.scp.conf @@ -16,7 +16,7 @@ filter { if ![message] or [message] == "" { drop {} } - if ![message] =~ "^test message for.*?$" { + if [message] !~ "^test message for.*?$" { csv { source => "message" columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ]