From b38dde3d66d60d8004997100b5213cc81538df52 Mon Sep 17 00:00:00 2001 From: MehmedSalihbasic Date: Mon, 20 May 2024 17:42:20 -0500 Subject: [PATCH] adding error.code field --- config/enrichments/89_cmdb_app.conf | 162 ++++++++++++++++++ .../processors/api_gitguardian_incidents.conf | 1 + config/processors/api_security_azure.atp.conf | 1 + .../api_security_azure.graph_api.conf | 1 + ...y_azure.graph_identity_protection_api.conf | 1 + .../processors/syslog_as400_audit_test.conf | 64 +++++++ .../processors/vuln_gitguardian_internal.conf | 1 + 7 files changed, 231 insertions(+) create mode 100755 config/enrichments/89_cmdb_app.conf create mode 100755 config/processors/syslog_as400_audit_test.conf diff --git a/config/enrichments/89_cmdb_app.conf b/config/enrichments/89_cmdb_app.conf new file mode 100755 index 00000000..71316c35 --- /dev/null +++ b/config/enrichments/89_cmdb_app.conf @@ -0,0 +1,162 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +##### mapping logs with CMDB memcache +# If ip,hostname exists, fetch it's stored value from memcached +# JSON to extract fields +# Remove the fetched field so that same key can be used to get new value in next `if block`. +filter { + if ("cmdb_enrichment" in [tags]) { + mutate { + remove_tag => ["cmdb_enrichment"] + } + if [source][ip] { + fingerprint { + source => ["[source][ip]"] + target => "source.ip_hashed" + method => "MD5" + } + memcached { + hosts => [VAR_MEMCACHED_ADDRESS] + namespace => "cmdb_hostip" + get => { "%{source.ip_hashed}" => "memcache_value" } + add_tag => [ "memcached_get_success_source.ip" ] + } + if "memcached_get_success_source.ip" in [tags] { + mutate { + add_tag => ["log_enriched_by_cmdb", "cmdb_sourceip"] + remove_tag => ["memcached_get_success_source.ip"] + } + json { + source => "memcache_value" + } + } + mutate { + remove_field => ["memcache_value", "source.ip_hashed"] + } + } + if [destination][ip] { + fingerprint { + source => ["[destination][ip]"] + target => "destination.ip_hashed" + method => "MD5" + } + memcached { + hosts => [VAR_MEMCACHED_ADDRESS] + namespace => "cmdb_hostip" + get => { "%{destination.ip_hashed}" => "memcache_value" } + add_tag => [ "memcached_get_success_destination.ip" ] + } + if "memcached_get_success_destination.ip" in [tags] { + mutate { + add_tag => ["log_enriched_by_cmdb", "cmdb_destinationip"] + remove_tag => ["memcached_get_success_destination.ip"] + } + json { + source => "memcache_value" + } + } + mutate { + remove_field => ["memcache_value", "destination.ip_hashed"] + } + } + if [host][hostname]{ + if [host][hostname] =~ '.' { + fingerprint { + source => ["[host][hostname]"] + target => "host.hostname_hashed" + method => "MD5" + } + } + else if [host][domain]{ + fingerprint { + source => ["[host][hostname].[host][domain]"] + target => "host.hostname_hashed" + method => "MD5" + } + } + memcached { + hosts => [VAR_MEMCACHED_ADDRESS] + namespace => "cmdb_hostname" + get => { "%{host.hostname_hashed}" => "memcache_value" } + add_tag => [ "memcached_get_success_host.hostname" ] + } + if "memcached_get_success_host.hostname" in [tags] { + mutate { + add_tag => ["log_enriched_by_cmdb"] + remove_tag => ["memcached_get_success_host.hostname"] + } + json { + source => "memcache_value" + } + } + mutate { + remove_field => ["memcache_value", "host.hostname_hashed"] + } + } + if [host][name] { + fingerprint { + source => ["[host][name]"] + target => "host.name_hashed" + method => "MD5" + } + memcached { + hosts => [VAR_MEMCACHED_ADDRESS] + namespace => "related.hosts" + get => { "%{host.name_hashed}" => "memcache_value" } + add_tag => [ "memcached_get_success_host.name" ] + } + if "memcached_get_success_host.name" in [tags] { + mutate { + add_tag => ["log_enriched_by_memcached"] + remove_tag => ["memcached_get_success_host.name"] + } + json { + source => "memcache_value" + } + } + mutate { + remove_field => ["memcache_value", "host.name_hashed"] + } + } + # memcache JSON output fields will be in dot notation which needs to be changed to nested. So, mutate filter to copy the fields to nested and removing dot notation fields. + if "cmdb_sourceip" in [tags] { + mutate { + rename => {"[cmdb.application.name]" => "[cmdb][source][application][name]"} + rename => {"[cmdb.application.criticality]" => "[cmdb][source][application][criticality]"} + rename => {"[cmdb.application.owner.name]" => "[cmdb][source][application][owner][name]"} + rename => {"[cmdb.application.owner.id]" => "[cmdb][source][application][owner][id]"} + rename => {"[cmdb.application.sensitivity]" => "[cmdb][source][application][sensitivity]"} + rename => {"[cmdb.application.message]" => "[cmdb][source][application][message]"} + } + mutate { + remove_tag => ["cmdb_sourceip"] + } + } + else if "cmdb_destinationip" in [tags] { + mutate { + rename => {"[cmdb.application.name]" => "[cmdb][destination][application][name]"} + rename => {"[cmdb.application.criticality]" => "[cmdb][destination][application][criticality]"} + rename => {"[cmdb.application.owner.name]" => "[cmdb][destination][application][owner][name]"} + rename => {"[cmdb.application.owner.id]" => "[cmdb][destination][application][owner][id]"} + rename => {"[cmdb.application.sensitivity]" => "[cmdb][destination][application][sensitivity]"} + rename => {"[cmdb.application.message]" => "[cmdb][destination][application][message]"} + } + mutate { + remove_tag => ["cmdb_destinationip"] + } + } + else{ + mutate { + rename => {"[cmdb.application.name]" => "[cmdb][application][name]"} + rename => {"[cmdb.application.criticality]" => "[cmdb][application][criticality]"} + rename => {"[cmdb.application.owner.name]" => "[cmdb][application][owner][name]"} + rename => {"[cmdb.application.owner.id]" => "[cmdb][application][owner][id]"} + rename => {"[cmdb.application.sensitivity]" => "[cmdb][application][sensitivity]"} + rename => {"[cmdb.application.message]" => "[cmdb][application][message]"} + } + } + mutate { + remove_field => [ "[cmdb.application.id]", "[cmdb.application.type]", "[cmdb.application.status]"] + } + } +} \ No newline at end of file diff --git a/config/processors/api_gitguardian_incidents.conf b/config/processors/api_gitguardian_incidents.conf index bd570469..1e0564e0 100644 --- a/config/processors/api_gitguardian_incidents.conf +++ b/config/processors/api_gitguardian_incidents.conf @@ -17,6 +17,7 @@ filter { target => "tmp" } mutate { + rename => { "[tmp][error_code]" => "[error][code]" } rename => { "[tmp][id]" => "[event][id]" } rename => { "[tmp][url]" => "[event][url]" } rename => { "[tmp][created_at]" => "[event][created]" } diff --git a/config/processors/api_security_azure.atp.conf b/config/processors/api_security_azure.atp.conf index f7aa9456..5dc5a5da 100644 --- a/config/processors/api_security_azure.atp.conf +++ b/config/processors/api_security_azure.atp.conf @@ -19,6 +19,7 @@ filter { add_field => { "[log][source][hostname]" => "%{[az][aadTenantId]}"} } mutate { + rename => { "[az][error_code]" => "[error][code]" } rename => {"[az][category]" => "[rule][category]"} rename => {"[az][id]" => "[event][id]"} # rename => {"[az][incidentId]" => ""} diff --git a/config/processors/api_security_azure.graph_api.conf b/config/processors/api_security_azure.graph_api.conf index 74a0248a..e5c3c02b 100644 --- a/config/processors/api_security_azure.graph_api.conf +++ b/config/processors/api_security_azure.graph_api.conf @@ -23,6 +23,7 @@ filter { } mutate { tag_on_failure => "mutate 1 failure" + rename => { "[az][error_code]" => "[error][code]" } rename => { "[az][category]" => "[rule][category]" } rename => { "[az][closeddatetime]" => "[event][end]" } rename => { "[az][confidence]" => "[event][risk_score_norm]" } # 2020-08-18t14:41:19.000z diff --git a/config/processors/api_security_azure.graph_identity_protection_api.conf b/config/processors/api_security_azure.graph_identity_protection_api.conf index 714bb1fe..9e03909c 100644 --- a/config/processors/api_security_azure.graph_identity_protection_api.conf +++ b/config/processors/api_security_azure.graph_identity_protection_api.conf @@ -19,6 +19,7 @@ filter { } mutate { tag_on_failure => "mutate 1 failure" + rename => { "[az][error_code]" => "[error][code]" } rename => { "[az][riskState]" => "[event][outcome]" } rename => { "[az][detectionTimingType]" => "[rule][category]" } rename => { "[az][id]" => "[event][id]" } diff --git a/config/processors/syslog_as400_audit_test.conf b/config/processors/syslog_as400_audit_test.conf new file mode 100755 index 00000000..5deb55d9 --- /dev/null +++ b/config/processors/syslog_as400_audit_test.conf @@ -0,0 +1,64 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { + "[event][module]" => "as400" + "[event][dataset]" => "as400.audit" + } + lowercase => [ "message" ] + } + grok { + tag_on_failure => "_parsefailure_header" + match => { "message" => "^(.*?{name=.*?}(\s)?)?(<(?\d+)>)?(\s)?(?.*)$|(^(?.*)$)" } + timeout_millis => 500 + } + syslog_pri { + syslog_pri_field_name => "pri" + remove_field => [ "pri" ] + } + json { + source => "actual_msg" + target => "tmp" + } + mutate { + rename => { + "[tmp][a_date]" => "date" + "[tmp][b_time]" => "time" + "[tmp][c_system]" => "[host][name]" + "[tmp][d_event]" => "[event][id]" + "[tmp][e_command]" => "[process][command_line]" + "[tmp][f_remote_ip_add]" => "[host][ip]" + "[tmp][g_job_name]" => "[rule][name]" + "[tmp][h_job_number]" => "[rule][id]" + "[tmp][i_job_user]" => "[user][name]" + } + } + if [host][name] { + mutate { + add_field => { "[log][source][hostname]" => "%{[host][name]}" } + } + } + mutate { + add_field => { "[event][created]" => "%{date} %{time}" } + } + #2023-02-08 07.03.36 + date { + match => [ "[event][created]" , "yyyy-MM-dd HH.mm.ss" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "actual_msg", "tmp", "date", "time", "[log][original]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/vuln_gitguardian_internal.conf b/config/processors/vuln_gitguardian_internal.conf index c897fe6b..2385e344 100644 --- a/config/processors/vuln_gitguardian_internal.conf +++ b/config/processors/vuln_gitguardian_internal.conf @@ -16,6 +16,7 @@ filter { "[event][dataset]" => "vuln.gitguardian_api" "[log][source][hostname]" => "vuln_gitguardian" } + rename => { "[tmp][error_code]" => "[error][code]" } rename => { "[tmp][date]" => "[event][created]" } rename => { "[tmp][gitguardian_url]" => "[event][reference]" } rename => { "[tmp][id]" => "[event][id]" }