Skip to content

Commit

Permalink
Merge pull request #496 from Cargill/scp_pipeline_test
Browse files Browse the repository at this point in the history 
added parsing for pipeline health test message
  • Loading branch information
@lyradc
lyradc authored May 13, 2024
2 parents b942985 + be55c68 commit cd795ab
Showing 1 changed file with 95 additions and 92 deletions.
187 changes: 95 additions & 92 deletions config/processors/api_security_skyhigh.scp.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,99 +16,102 @@ filter {
if ![message] or [message] == "" {
drop {}
}
csv {
source => "message"
columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ]
convert => {
"[destination][bytes]" => "integer"
"[source][bytes]" => "integer"
}
skip_empty_columns => true
skip_empty_rows => true
}
# row empty
if "_csvskippedemptyfield" in [tags] {
drop {}
}
if [message] !~ "^test message for.*?$" {
csv {
source => "message"
columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ]
convert => {
"[destination][bytes]" => "integer"
"[source][bytes]" => "integer"
}
skip_empty_columns => true
skip_empty_rows => true
}
# row empty
if "_csvskippedemptyfield" in [tags] {
drop {}
}

mutate {
split => { "[rule][category]" => ", " }
}
# Message is csv, this creates [event][original] to include fields names
mutate {
add_field => { "[event][original]" => "request_timestamp_epoch: %{[event][time]}, num: %{num}, usr: %{usr}, source.nat.ip: %{[source][ip]}, http.request.method: %{[http][request][method]}, destination.bytes: %{[destination][bytes]}, source.bytes: %{[source][bytes]}, url.domain: %{[url][domain]}, url.path: %{[url][path]}, event.action: %{[event][action]}, rule.name: %{[rule][name]}, request_timestamp_epoch: %{request_timestamp_epoch}, url.scheme: %{[url][scheme]}, rule.category: %{[rule][category]}, http.request.body.content: %{[http][request][body][content]}, service.name: %{[network][application]}, event.severity_name: %{[event][severity_name]}, last_rule: %{[rule][uuid]}, http_status_code: %{[http][response][status_code]}, client_ip: %{[source][nat][ip]}, location: %{[rule][description]}, block_reason: %{[rule][ruleset]}, user_agent_comment: %{[user_agent][version]}, user_agent_product: %{[user_agent][name]}, user_agent_version: %{[user_agent][original]}, process_name: %{[process][name]}, destination_ip: %{[destination][ip]}, destination_port: %{[destination][port]}, event.risk_score: %{[event][risk_score]}, discarded_host: %{discarded_host}, tls.client.x509.version_number: %{[tls][client][x509][version_number]}, tls.version: %{[tls][version]}" }
}
mutate {
gsub => [ "[event][original]", "%\{.*?}(,)? ", "" ]
gsub => [ "[event][original]", "%\{.*?}", "" ]
}

if [usr] and [usr] =~ ".*?\\.*?" {
grok {
match => { "usr" => "(?<[user][domain]>.*?)\\(?<[user][name]>.*?)$" }
timeout_millis => 500
}
} else {
mutate {
rename => { "[usr]" => "[user][name]" }
}
}
if [source][ip] and [source][ip] =~ "source_ip" {
mutate {
remove_field => ["[source][ip]"]
}
}
# Create [url][full]
mutate {
add_field => { "[url][full]" => "%{[[url][scheme]]}://%{[[url][domain]]}%{[[url][path]]}" }
}
date {
match => [ "request_timestamp_epoch", "UNIX" ]
timezone => "GMT"
locale => "en"
target => "[event][created]"
}
mutate {
add_field => { "[cloud][provider]" => "mcafee" }
#add_field => { "[event][module]" => "mcafee" }
#add_field => { "[event][dataset]" => "mcafee.mcp" }
add_field => { "[log][source][hostname]" => "api_mcp"}
}
mutate {
lowercase => [ "[event][action]" ]
}
if [event][action] =~ "observed" {
mutate {
replace => { "[event][action]" => "allowed" }
}
}
if [event][action] =~ "denied" {
mutate {
replace => { "[event][action]" => "denied" }
}
}
mutate {
add_field => { "[service][state]" => "ssl_scanned: %{ssl_scanned}" }
}
if [av_scanned_up] {
mutate {
add_field => { "[service][state]" => "av_scanned_up: %{av_scanned_up}" }
}
}
if [av_scanned_down] {
mutate {
add_field => { "[service][state]" => "av_scanned_down: %{av_scanned_down}" }
}
}
if [rbi] {
mutate {
add_field => { "[service][state]" => "rbi: %{rbi}" }
}
}
if [dlp] {
mutate {
add_field => { "[service][state]" => "dlp: %{dlp}" }
}
mutate {
split => { "[rule][category]" => ", " }
}
# Message is csv, this creates [event][original] to include fields names
mutate {
add_field => { "[event][original]" => "request_timestamp_epoch: %{[event][time]}, num: %{num}, usr: %{usr}, source.nat.ip: %{[source][ip]}, http.request.method: %{[http][request][method]}, destination.bytes: %{[destination][bytes]}, source.bytes: %{[source][bytes]}, url.domain: %{[url][domain]}, url.path: %{[url][path]}, event.action: %{[event][action]}, rule.name: %{[rule][name]}, request_timestamp_epoch: %{request_timestamp_epoch}, url.scheme: %{[url][scheme]}, rule.category: %{[rule][category]}, http.request.body.content: %{[http][request][body][content]}, service.name: %{[network][application]}, event.severity_name: %{[event][severity_name]}, last_rule: %{[rule][uuid]}, http_status_code: %{[http][response][status_code]}, client_ip: %{[source][nat][ip]}, location: %{[rule][description]}, block_reason: %{[rule][ruleset]}, user_agent_comment: %{[user_agent][version]}, user_agent_product: %{[user_agent][name]}, user_agent_version: %{[user_agent][original]}, process_name: %{[process][name]}, destination_ip: %{[destination][ip]}, destination_port: %{[destination][port]}, event.risk_score: %{[event][risk_score]}, discarded_host: %{discarded_host}, tls.client.x509.version_number: %{[tls][client][x509][version_number]}, tls.version: %{[tls][version]}" }
}
mutate {
gsub => [ "[event][original]", "%\{.*?}(,)? ", "" ]
gsub => [ "[event][original]", "%\{.*?}", "" ]
}

if [usr] and [usr] =~ ".*?\\.*?" {
grok {
match => { "usr" => "(?<[user][domain]>.*?)\\(?<[user][name]>.*?)$" }
timeout_millis => 500
}
} else {
mutate {
rename => { "[usr]" => "[user][name]" }
}
}
if [source][ip] and [source][ip] =~ "source_ip" {
mutate {
remove_field => ["[source][ip]"]
}
}
# Create [url][full]
mutate {
add_field => { "[url][full]" => "%{[[url][scheme]]}://%{[[url][domain]]}%{[[url][path]]}" }
}
date {
match => [ "request_timestamp_epoch", "UNIX" ]
timezone => "GMT"
locale => "en"
target => "[event][created]"
}
mutate {
add_field => { "[cloud][provider]" => "mcafee" }
#add_field => { "[event][module]" => "mcafee" }
#add_field => { "[event][dataset]" => "mcafee.mcp" }
add_field => { "[log][source][hostname]" => "api_mcp"}
}
mutate {
lowercase => [ "[event][action]" ]
}
if [event][action] =~ "observed" {
mutate {
replace => { "[event][action]" => "allowed" }
}
}
if [event][action] =~ "denied" {
mutate {
replace => { "[event][action]" => "denied" }
}
}
mutate {
add_field => { "[service][state]" => "ssl_scanned: %{ssl_scanned}" }
}
if [av_scanned_up] {
mutate {
add_field => { "[service][state]" => "av_scanned_up: %{av_scanned_up}" }
}
}
if [av_scanned_down] {
mutate {
add_field => { "[service][state]" => "av_scanned_down: %{av_scanned_down}" }
}
}
if [rbi] {
mutate {
add_field => { "[service][state]" => "rbi: %{rbi}" }
}
}
if [dlp] {
mutate {
add_field => { "[service][state]" => "dlp: %{dlp}" }
}
}

}
mutate {
# host field is added by file input plugin
Expand Down

0 comments on commit cd795ab

Please sign in to comment.