From 3a002b4b094afe67b3b31645137cc062cb4b04ba Mon Sep 17 00:00:00 2001 From: MehmedSalihbasic Date: Thu, 11 Jan 2024 21:35:27 -0600 Subject: [PATCH] moving files from internal repo per Brian --- config/processors/api_aws_app.conf | 127 ++++ .../api_https_audit_sap_tcodes.conf | 61 ++ .../api_ois_sap_security_bridge.conf | 104 +++ config/processors/api_security_sap_btp.conf | 56 ++ .../processors/api_security_skyhigh.scp.conf | 119 ++++ .../api_sqlite_known_good_hash_nist.conf | 38 ++ ...t_hub_audit_azure.event_hub_anomalies.conf | 103 +++ ..._hub_audit_azure.event_hub_azureaudit.conf | 108 ++++ ...hub_audit_azure.event_hub_emailevents.conf | 65 ++ ...it_azure.event_hub_interactive_signin.conf | 160 +++++ ..._audit_azure.event_hub_officeactivity.conf | 8 +- ...t_hub_audit_azure.event_hub_riskyuser.conf | 62 ++ ...b_audit_azure.event_hub_securityalert.conf | 82 +++ ...udit_azure.event_hub_securityincident.conf | 85 +++ ..._audit_azure.event_hub_userriskevents.conf | 5 +- ...at_file_log_audit_sap.cloud_connector.conf | 64 ++ .../ftp_gzip_audit_infoblox_dns.conf | 33 +- config/processors/log_cdp_hadoop.conf | 47 ++ config/processors/log_cdp_rundeck.conf | 45 ++ config/processors/log_cdp_tanium.conf | 45 ++ .../log_security_proofpoint.trap.conf | 108 ++++ config/processors/s3_log_aws.cloudfront.conf | 77 +++ config/processors/s3_log_aws.lb.conf | 97 +++ .../snowflake_acc_access_history.conf | 40 ++ .../processors/syslog_as400_audit_test.conf | 64 ++ .../processors/syslog_audit_aix_rsyslog.conf | 65 ++ ...slog_securityonion_securityonion.zeek.conf | 603 ++++++++++++++++++ ...slog_virtustream_trendmicro_forwarder.conf | 54 ++ .../processors/vuln_gitguardian_internal.conf | 54 ++ config/processors/vuln_sysdig_scanning.conf | 45 ++ config/processors/vuln_tenable_asm.conf | 177 +++++ config/processors/vuln_tenable_sc.conf | 123 ++++ 32 files changed, 2899 insertions(+), 25 deletions(-) create mode 100644 config/processors/api_aws_app.conf create mode 100644 config/processors/api_https_audit_sap_tcodes.conf create mode 100644 config/processors/api_ois_sap_security_bridge.conf create mode 100644 config/processors/api_security_sap_btp.conf create mode 100644 config/processors/api_security_skyhigh.scp.conf create mode 100644 config/processors/api_sqlite_known_good_hash_nist.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_anomalies.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_azureaudit.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_emailevents.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_riskyuser.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_securityalert.conf create mode 100644 config/processors/event_hub_audit_azure.event_hub_securityincident.conf create mode 100644 config/processors/flat_file_log_audit_sap.cloud_connector.conf create mode 100644 config/processors/log_cdp_hadoop.conf create mode 100644 config/processors/log_cdp_rundeck.conf create mode 100644 config/processors/log_cdp_tanium.conf create mode 100644 config/processors/log_security_proofpoint.trap.conf create mode 100644 config/processors/s3_log_aws.cloudfront.conf create mode 100644 config/processors/s3_log_aws.lb.conf create mode 100644 config/processors/snowflake_acc_access_history.conf create mode 100644 config/processors/syslog_as400_audit_test.conf create mode 100644 config/processors/syslog_audit_aix_rsyslog.conf create mode 100644 config/processors/syslog_securityonion_securityonion.zeek.conf create mode 100644 config/processors/syslog_virtustream_trendmicro_forwarder.conf create mode 100644 config/processors/vuln_gitguardian_internal.conf create mode 100644 config/processors/vuln_sysdig_scanning.conf create mode 100644 config/processors/vuln_tenable_asm.conf create mode 100644 config/processors/vuln_tenable_sc.conf diff --git a/config/processors/api_aws_app.conf b/config/processors/api_aws_app.conf new file mode 100644 index 00000000..914c67f4 --- /dev/null +++ b/config/processors/api_aws_app.conf @@ -0,0 +1,127 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + remove_field => ["event", "host"] + } + json { + source => "message" + target => "tmp" + } + mutate { + add_field => { + "[event][module]" => "aws" + "[event][dataset]" => "aws.app" + "[log][source][hostname]" => "aws_app" + } + } + # cmd + # "[tmp][__monotonic_timestamp]" => "1289776813424" + # "[tmp][__realtime_timestamp]" => "1704485287095345" + # "[tmp][_source_monotonic_timestamp]" => "1989560529012" + # "[tmp][_cap_effective]" => "1ffffffffff" + mutate { + rename => { + "[tmp][_source_realtime_timestamp]" => "[event][created]" + "[tmp][_exe]" => "[process][executable]" + "[tmp][_comm]" => "[process][name]" + "[tmp][_boot_id]" => "[process][entity_id]" + "[tmp][unit]" => "[process][parent][command_line]" + "[tmp][code_file]" => "[process][parent][name]" + "[tmp][service]" => "[service][name]" + "[tmp][_gid]" => "[process][pgid]" + "[tmp][_pid]" => "[process][pid]" + "[tmp][_cmdline]" => "[process][command_line]" + "[tmp][_uid]" => "[file][uid]" + "[tmp][_systemd_cgroup]" => "[group][name]" + "[tmp][_transport]" => "[network][transport]" + "[tmp][_machine_id]" => "[host][id]" + "[tmp][code_function]" => "[log][origin][function]" + "[tmp][code_line]" => "[log][origin][file][line]" + "[tmp][host]" => "[host][hostname]" + "[tmp][syslog_identifier]" => "[log][syslog][facility][name]" + "[tmp][priority]" => "[log][syslog][priority]" + "[tmp][syslog_facility]" => "[log][syslog][facility][code]" + } + } + # k8s + mutate { + rename => { + "[tmp][cluster_name]" => "[cloud][instance][name]" + "[tmp][container_id]" => "[container][id]" + "[tmp][ddsource]" => "[container][runtime]" + "[tmp][ddtags]" => "[container][image][tag]" + "[tmp][hostname]" => "[host][hostname]" + "[tmp][kubernetes][container_image]" => "[container][image][name]" + "[tmp][kubernetes][container_name]" => "[container][name]" + "[tmp][kubernetes][pod_ip]" => "[cloud][instance][id]" + "[tmp][kubernetes][namespace_name]" => "[cloud][project][id]" + "[tmp][kubernetes][pod_name]" => "[cloud][project][name]" + "[tmp][kubernetes][pod_owner]" => "[cloud][account][name]" + "[tmp][service]" => "[cloud][service][name]" + "[tmp][source]" => "[cloud][machine][type]" + "[tmp][source_type]" => "[event][kind]" + "[tmp][stream]" => "[event][provider]" + "[tmp][timestamp]" => "[event][ingested]" + } + } + # k8s or json message + if [tmp][message] =~ "^{.*?}$" { + json { + source => "[tmp][message]" + target => "jtmp" + } + mutate { + rename => { + "[jtmp][class]" => "com.cargill.fps.server.service.impl.pricingserviceimpl" + "[jtmp][method]" => "[http][request][mime_type]" + "[jtmp][@severity]" => "[log][syslog][severity][name]" + "[jtmp][level]" => "[log][level]" + "[jtmp][level_value]" => "[log][syslog][severity][code]" + "[jtmp][logger_name]" => "[log][logger]" + "[jtmp][thread_name]" => "[process][thread][name]" + "[jtmp][@timestamp]" => "[event][created]" + "[jtmp][file]" => "[file][name]" + "[jtmp][line]" => "[log][origin][file][line]" + "[jtmp][@message]" => "[error][message]" + "[jtmp][dd.trace_id]" => "[trace][id]" + "[jtmp][dd.service]" => "[service][name]" + "[jtmp][dd.span_id]" => "[span][id]" + } + add_tag => [ "%{[jtmp][dd.env]}" ] + } + } else { + mutate { + rename => { + "[tmp][message]" => "[error][message]" + } + } + } + # [event][created] "1704485287095345" + # "[event][created]" "2024-01-04t17:45:30.921z" + date { + match => ["[event][created]", "yyyy-MM-dd'T'HH:mm:sss'Z'","yyyy-MM-dd't'HH:mm:sss'z'", "ISO8601", "UNIX", "UNIX_MS" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + } + # "[event][ingested]" = "2024-01-04t17:45:30.921235225z" + date { + match => ["[event][ingested]", "yyyy-MM-dd'T'HH:mm:sssssssss'Z'","yyyy-MM-dd't'HH:mm:sssssssss'z'", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + } + + mutate { + remove_field => ["tmp", "jtmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/api_https_audit_sap_tcodes.conf b/config/processors/api_https_audit_sap_tcodes.conf new file mode 100644 index 00000000..c47b5386 --- /dev/null +++ b/config/processors/api_https_audit_sap_tcodes.conf @@ -0,0 +1,61 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + remove_field => ["event", "host", "log" ] + } + json { + source => "message" + target => "tmp" + } + mutate { + add_field => { + "[event][module]" => "sap" + "[event][dataset]" => "sap.t-codes" + } + } + mutate { + add_tag => [ "%{[tmp][tags]}" ] + rename => { + "[tmp][timestamp]" => "[event][created]" + "[tmp][reason_id]" => "[event][reason]" + "[tmp][agent][type]" => "[agent][type]" + "[tmp][agent][name]" => "[agent][name]" + "[tmp][client_pc]" => "[source][ip]" + "[tmp][username]" => "[user][name]" + "[tmp][log_client]" => "[group][id]" + "[tmp][edm_data][log_sapgui][value_header][0][pprogram]" => "[process][name]" + "[tmp][edm_data][log_sapgui][value_header][0][gui_title]" => "[rule][name]" + "[tmp][edm_data][log_sapgui][value_header][0][tcode]" => "[rule][id]" + "[tmp][trx_name]" => "[rule][description]" + "[tmp][tid]" => "[rule][category]" + "[tmp][sysid]" => "[host][id]" + "[tmp][technology]" => "[host][os][type]" + "[tmp][host_name]" => "[host][hostname]" + "[tmp][log][source][ip]" => "[log][source][ip]" + "[tmp][log][source][hostname]" => "[log][source][hostname]" + } + } + mutate { + strip => ["[event][created]"] + } + + # "[event][created]" = "20240108175253.178 " + date { + match => ["[event][created]", "yyyyMMddHHmmss.SSS" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + } + mutate { + remove_field => ["tmp"] + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/api_ois_sap_security_bridge.conf b/config/processors/api_ois_sap_security_bridge.conf new file mode 100644 index 00000000..9fb1c1f7 --- /dev/null +++ b/config/processors/api_ois_sap_security_bridge.conf @@ -0,0 +1,104 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { + "[event][module]" => "sap" + "[event][dataset]" => "sap.security_bridge" + "[log][source][hostname]" => "api_ois_sap_security_bridge" + } + } + json{ + source=>"message" + target=>"tmp" + } + mutate{ + rename=>{ + "[tmp][IPv4Address]" => "[source][ip]" + "[tmp][severity]" => "[event][severity]" + "[tmp][eventUserType]" => "[user][roles]" + "[tmp][eventUserGroup]" => "[user][group][name]" + "[tmp][systemType]" => "[observer][type]" + "[tmp][eventMsg]" => "[event][reason]" + "[tmp][program]" => "[package][type]" + "[tmp][account]" => "[user][name]" + "[tmp][action]" => "[event][action]" + "[tmp][guid]" => "[event][id]" + "[tmp][transactionCode]" => "[transaction][id]" + "[tmp][terminal]" => "[group][name]" + "[tmp][object]" => "[event][code]" + "[tmp][sid]" => "[group][id]" + "[tmp][client]" => "[service][id]" + "[tmp][qSid]" => "[cloud][instance][name]" + } + } + if [tmp][eventAttr1] { + mutate{ + add_field => { + "[event][type]" => "%{[tmp][eventAttr1]}" + } + } + } + if [tmp][eventAttr2] { + mutate{ + add_field => { + "[event][type]" => "%{[tmp][eventAttr2]}" + } + } + } + if [tmp][eventAttr3] { + mutate{ + add_field => { + "[event][type]" => "%{[tmp][eventAttr3]}" + } + } + } + if [tmp][eventAttr4] { + mutate{ + add_field => { + "[event][type]" => "%{[tmp][eventAttr4]}" + } + } + } + if [tmp][eventAttr5] { + mutate{ + add_field => { + "[event][type]" => "%{[tmp][eventAttr5]}" + } + } + } + grok { + match => { "[tmp][timestamp]" => "^(\/Date\()(?<[event][created]>.*?)(\)\/)$" } + match => { "[tmp][recTimestamp]" => "^(\/Date\()(?<[event][modified]>.*?)(\)\/)$" } + } + date { + match => [ "[event][created]", "UNIX_MS" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][modified]", "UNIX_MS" ] + timezone => "GMT" + locale => "en" + target => "[event][modified]" + tag_on_failure => "_dateparsefailure_es" + } + if [tmp][retroactive] { + mutate { + add_tag => [ "retroactive" ] + } + } + mutate { + remove_field => ["tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/api_security_sap_btp.conf b/config/processors/api_security_sap_btp.conf new file mode 100644 index 00000000..556957b0 --- /dev/null +++ b/config/processors/api_security_sap_btp.conf @@ -0,0 +1,56 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + remove_field => ["event", "host", "log" ] + } + mutate { + add_field => { + "[event][module]" => "sap" + "[event][dataset]" => "sap.btp" + } + } + json { + source => "message" + target => "tmp" + } + + mutate{ + add_field =>{ "[log][source][hostname]" => "%{[tmp][tenant]}" } + lowercase => [ "tmp" ] + } + mutate { + rename => { + "[tmp][als_service_id]" => "[service][id]" + "[tmp][user]" => "[user][name]" + "[tmp][category]" => "[event][category]" + "[tmp][message_uuid]" => "[rule][uuid]" + "[tmp][org_id]" => "[cloud][project][id]" + "[tmp][time]" => "[event][created]" + "[tmp][space_id]" => "[cloud][instance][id]" + "[tmp][tenant]" => "[cloud][account][id]" + "[tmp][object][type]" => "[event][type]" + "[tmp][id]" => "[event][id]" + "[tmp][data]" => "[error][message]" + "[tmp][ip]" => "[source][ip]" + } + } + date { + match => [ "[event][created]", "ISO8601" ] + timezone => "GMT" + locale => "ec" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/api_security_skyhigh.scp.conf b/config/processors/api_security_skyhigh.scp.conf new file mode 100644 index 00000000..17c4f8d5 --- /dev/null +++ b/config/processors/api_security_skyhigh.scp.conf @@ -0,0 +1,119 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + ### McAfee MCP API, tested against API v1-v5 + mutate { + add_field => { "[event][module]" => "skyhigh" } + add_field => { "[event][dataset]" => "skyhigh.scp" } + strip => ["message"] + } + if ![message] or [message] == "" { + drop {} + } + csv { + source => "message" + columns => ["num","usr","[source][nat][ip]","[http][request][method]","[destination][bytes]","[source][bytes]","[url][domain]","[url][path]","[event][action]","[rule][name]","request_timestamp_epoch","[event][time]","[url][scheme]","[rule][category]","[http][request][body][content]","[service][name]","[event][severity_name]", "[rule][uuid]", "[http][response][status_code]", "[source][ip]", "[rule][description]", "[rule][ruleset]", "[user_agent][name]", "[user_agent][version]", "[user_agent][original]", "[process][name]", "[destination][ip]", "[destination][port]", "[observer][geo][country_iso_code]", "[http][request][referrer]", "[ssl_scanned]", "[av_scanned_up]", "[av_scanned_down]", "[rbi]", "[dlp]", "[source][address]", "[file][name]", "[observer][egress][ip]", "[observer][ingress][ip]", "[source][nat][port]", "[event][risk_score]", "discarded_host", "[tls][client][x509][version_number]", "[tls][version]" ] + convert => { + "[destination][bytes]" => "integer" + "[source][bytes]" => "integer" + } + skip_empty_columns => true + skip_empty_rows => true + } + # row empty + if "_csvskippedemptyfield" in [tags] { + drop {} + } + + mutate { + split => { "[rule][category]" => ", " } + } + # Message is csv, this creates [event][original] to include fields names + mutate { + add_field => { "[event][original]" => "request_timestamp_epoch: %{[event][time]}, num: %{num}, usr: %{usr}, source.nat.ip: %{[source][ip]}, http.request.method: %{[http][request][method]}, destination.bytes: %{[destination][bytes]}, source.bytes: %{[source][bytes]}, url.domain: %{[url][domain]}, url.path: %{[url][path]}, event.action: %{[event][action]}, rule.name: %{[rule][name]}, request_timestamp_epoch: %{request_timestamp_epoch}, url.scheme: %{[url][scheme]}, rule.category: %{[rule][category]}, http.request.body.content: %{[http][request][body][content]}, service.name: %{[network][application]}, event.severity_name: %{[event][severity_name]}, last_rule: %{[rule][uuid]}, http_status_code: %{[http][response][status_code]}, client_ip: %{[source][nat][ip]}, location: %{[rule][description]}, block_reason: %{[rule][ruleset]}, user_agent_comment: %{[user_agent][version]}, user_agent_product: %{[user_agent][name]}, user_agent_version: %{[user_agent][original]}, process_name: %{[process][name]}, destination_ip: %{[destination][ip]}, destination_port: %{[destination][port]}, event.risk_score: %{[event][risk_score]}, discarded_host: %{discarded_host}, tls.client.x509.version_number: %{[tls][client][x509][version_number]}, tls.version: %{[tls][version]}" } + } + mutate { + gsub => [ "[event][original]", "%\{.*?}(,)? ", "" ] + gsub => [ "[event][original]", "%\{.*?}", "" ] + } + + if [usr] and [usr] =~ ".*?\\.*?" { + grok { + match => { "usr" => "(?<[user][domain]>.*?)\\(?<[user][name]>.*?)$" } + timeout_millis => 500 + } + } else { + mutate { + rename => { "[usr]" => "[user][name]" } + } + } + if [source][ip] and [source][ip] =~ "source_ip" { + mutate { + remove_field => ["[source][ip]"] + } + } + # Create [url][full] + mutate { + add_field => { "[url][full]" => "%{[[url][scheme]]}://%{[[url][domain]]}%{[[url][path]]}" } + } + date { + match => [ "request_timestamp_epoch", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + } + mutate { + add_field => { "[cloud][provider]" => "mcafee" } + #add_field => { "[event][module]" => "mcafee" } + #add_field => { "[event][dataset]" => "mcafee.mcp" } + add_field => { "[log][source][hostname]" => "api_mcp"} + } + mutate { + lowercase => [ "[event][action]" ] + } + if [event][action] =~ "observed" { + mutate { + replace => { "[event][action]" => "allowed" } + } + } + if [event][action] =~ "denied" { + mutate { + replace => { "[event][action]" => "denied" } + } + } + mutate { + add_field => { "[service][state]" => "ssl_scanned: %{ssl_scanned}" } + } + if [av_scanned_up] { + mutate { + add_field => { "[service][state]" => "av_scanned_up: %{av_scanned_up}" } + } + } + if [av_scanned_down] { + mutate { + add_field => { "[service][state]" => "av_scanned_down: %{av_scanned_down}" } + } + } + if [rbi] { + mutate { + add_field => { "[service][state]" => "rbi: %{rbi}" } + } + } + if [dlp] { + mutate { + add_field => { "[service][state]" => "dlp: %{dlp}" } + } + } + mutate { + # host field is added by file input plugin + remove_field => [ "request_timestamp_epoch", "usr", "num", "[event][time]", "path", "ssl_scanned", "av_scanned_up", "av_scanned_down", "rbi", "dlp", "discarded_host", "[event][original]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/api_sqlite_known_good_hash_nist.conf b/config/processors/api_sqlite_known_good_hash_nist.conf new file mode 100644 index 00000000..fb6e1de2 --- /dev/null +++ b/config/processors/api_sqlite_known_good_hash_nist.conf @@ -0,0 +1,38 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "known_good_file_hash" } + add_field => { "[event][dataset]" => "known_good_file_hash.nist" } + add_field => { "[log][source][hostname]" => "api_sqlite_hash.nist" } + } + json { + source => "message" + target => "tmp" + } + mutate { + rename => { "[tmp][sha256]" => "[file][hash][sha256]" } + rename => { "[tmp][sha1]" => "[file][hash][sha1]" } + rename => { "[tmp][md5]" => "[file][hash][md5]" } + rename => { "[tmp][crc32]" => "[file][gid]" } + rename => { "[tmp][file_name]" => "[file][name]" } + rename => { "[tmp][file_size]" => "[file][size]" } + rename => { "[tmp][package_id]" => "[file][uid]" } + } + + mutate { + copy => { "[file][hash][md5]" => "[@metadata][document_id]" } + } + + mutate { + remove_field => [ "[tmp]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/event_hub_audit_azure.event_hub_anomalies.conf b/config/processors/event_hub_audit_azure.event_hub_anomalies.conf new file mode 100644 index 00000000..6d700030 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_anomalies.conf @@ -0,0 +1,103 @@ +# Copyright [2022] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => ["message"] + } + json{ + source=>"message" + target=>"tmp" + } + + split{ + field => "[tmp][records]" + } + json_encode{ + source=>"tmp" + target => "message" + } + mutate { + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.anomalies" } + add_field => { "[log][source][hostname]" => "%{[tmp][records][tenantid]}" } + } + mutate{ + rename => { + "[tmp][records][username]" => "[user][name]" + "[tmp][records][userprincipalname]" => "[user][email]" + "[tmp][records][userinsights][accountdisplayname]" => "[user][full_name]" + "[tmp][records][userinsights][accountdomain]" => "[user][domain]" + "[tmp][records][userinsights][accountobjectid]" => "[user][id]" + "[tmp][records][timegenerated]" => "[event][created]" + "[tmp][records][starttime]" => "[event][start]" + "[tmp][records][endtime]" => "[event][end]" + "[tmp][records][rulename]" => "[rule][name]" + "[tmp][records][ruleid]" => "[rule][id]" + "[tmp][records][extendedlinks][label]" => "[event][kind]" + "[tmp][records][deviceinsights][useragentfamily]" => "[user_agent][device][name]" + "[tmp][records][entities][address]" => "[source][ip]" + "[tmp][records][anomalydetails][score]" => "[event][risk_score]" + } + } + if [tmp][records][activityinsights][actionuncommonlyperformedbyuser] { + mutate{ + add_tag => [ "actionuncommonlyperformedbyuser" ] + } + } + if [tmp][records][activityinsights][unusualnumberofaadconditionalaccessfailures] { + mutate{ + add_tag => [ "unusualnumberofaadconditionalaccessfailures" ] + } + } + if [tmp][records][activityinsights][unusualnumberoffailedsigninofthisuser] { + mutate{ + add_tag => [ "unusualnumberoffailedsigninofthisuser" ] + } + } + if [tmp][records][tactics][0] != '' { + mutate{ + rename =>{ + "[tmp][records][tactics]" => "[threat][tactic][name]" + } + } + } + if [tmp][records][techniques][0] != '' { + mutate{ + rename =>{ + "[tmp][records][techniques]" => "[threat][technique][id]" + } + } + } + date { + match => [ "[event][created]", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][start]", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][start]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][end]", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][end]" + tag_on_failure => "_dateparsefailure_ee" + } + mutate{ + remove_field => ["tmp"] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/event_hub_audit_azure.event_hub_azureaudit.conf b/config/processors/event_hub_audit_azure.event_hub_azureaudit.conf new file mode 100644 index 00000000..47cbd5b6 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_azureaudit.conf @@ -0,0 +1,108 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate{ + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.audit" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { + rename => { "[az][aadoperationtype]" => "[event][action]" } + rename => { "[az][aadtenantid]" => "[rule][id]" } + rename => { "[az][activitydatetime]" => "[event][created]" } + rename => { "[az][activitydisplayname]" => "[rule][name]" } + rename => { "[az][category]" => "[event][category]" } + rename => { "[az][durationms]" => "[event][duration]" } + rename => { "[az][id]"=>"[event][id]" } + rename => { "[az][initiatedby][user][id]"=>"[source][user][id]" } + rename => { "[az][initiatedby][user][displayname]"=>"[source][user][name]" } + rename => { "[az][initiatedby][user][userprincipalname]"=>"[source][user][email]" } + rename => { "[az][initiatedby][user][ipaddress]"=>"[source][ip]" } + rename => { "[az][level]" => "[event][severity]" } + rename => { "[az][operationversion]"=>"[rule][version]" } + rename => { "[az][result]" => "[event][outcome]" } + rename => { "[az][resultdescription]" => "[rule][description]" } + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][type]" => "[event][type]" } + } + ruby { + code => " + id_lst = [] + name_lst = [] + email_lst = [] + + if event.get('[az][targetresources]').kind_of?(Array) + event.get('[az][targetresources]').each do |item| + if item['id'] != '' + id_lst.push(item['id']) + end + if item['displayname'] != '' + name_lst.push(item['displayname']) + end + if item['userprincipalname'] != '' + email_lst.push(item['userprincipalname']) + end + end + end + + if !id_lst.empty? + event.set('[destination][user][id]', id_lst) + end + if !name_lst.empty? + event.set('[destination][user][name]', name_lst) + end + if !email_lst.empty? + event.set('[destination][user][email]', email_lst) + end + " + tag_on_exception => "ruby_block_exception" + } + mutate { + remove_field => [ "az", "tmp" ] + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } + date { + match => [ "[event][created]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/event_hub_audit_azure.event_hub_emailevents.conf b/config/processors/event_hub_audit_azure.event_hub_emailevents.conf new file mode 100644 index 00000000..c292de18 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_emailevents.conf @@ -0,0 +1,65 @@ +# Copyright [2022] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } + else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate { + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.emailevents" } + add_field => { "[log][source][hostname]" => "api_azure_emailevents" } + rename => { "[az][reportid]" => "[rule][id]" } + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][url]" => "[url][original]" } + rename => { "[az][urldomain]" => "[url][domain]"} + rename => { "[az][type]" => "[event][type]" } + rename => { "[az][filename]" => "[file][name]" } + rename => { "[az][filetype]" => "[file][type]" } + rename => { "[az][senderdisplayname]" => "[source][user][name]" } + rename => { "[az][senderfromaddress]" => "[source][user][email]" } + rename => { "[az][senderobjectid]" => "[source][user][id]" } + rename => { "[az][filesize]" => "[file][size]" } + rename => { "[az][recipientemailaddress]" => "[destination][user][email]" } + rename => { "[az][recipientobjectid]" => "[database][user][id]" } + rename => { "[az][filesize]" => "[file][size]" } + } + mutate { + remove_field => [ "az", "tmp" ] + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf b/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf new file mode 100644 index 00000000..d226d833 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf @@ -0,0 +1,160 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate{ + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.interactivesignin" } + add_field => { "[log][source][hostname]" => "%{[az][TenantId]}" } + } + mutate { + rename => { "[az][TenantId]" => "[cloud][account][id]" } + rename => { "[az][TimeGenerated]" => "[event][ingested]" } + rename => { "[az][OperationName]" => "[event][reference]" } + rename => { "[az][Category]" => "[event][category]" } + # rename => { "host" => "[agent][name]" } + rename => { "[az][DurationMs]" => "[event][duration]"} + rename => { "[az][ResultType]" => "[error][code]" } + rename => { "[az][CorrelationId]" => "[event][id]" } + rename => { "[az][Identity]" => "[file][group]" } + rename => { "[az][CreatedDateTime]" => "[event][created]" } + rename => { "[az][IPAddress]" => "[source][ip]" } + rename => { "[az][UserAgent]" => "[user_agent][original]" } + rename => { "[az][UserPrincipalName]" => "[user][email]" } + rename => { "[az][AppDisplayName]" => "[process][name]" } + rename => { "[az][UserId]" => "[user][id]" } + rename => { "[az][OperationVersion]" => "[service][version]" } + rename => { "[az][AppId]" => "[process][entity_id]" } + rename => { "[az][Status][failureReason]" => "[event][reason]" } + rename => { "[az][AutonomousSystemNumber]" => "[source][as][number]" } + rename => { "[az][UserDisplayName]" => "[user][full_name]" } + rename => { "[az][ConditionalAccessStatus]" => "[event][action]" } + rename => { "[az][ResourceServicePrincipalId]" => "[cloud][instance][id]" } + rename => { "[az][Status][additionalDetails]" => "[rule][category]"} + rename => { "[az][Status][errorCode]" => "[rule][id]" } + rename => { "[az][AuthenticationRequirement]" => "[event][kind]"} + rename => { "[az][ClientAppUsed]" => "[event][provider]"} + rename => { "[az][MfaDetail]" => "[event][outcome]"} + rename => { "[az][ResourceDisplayName]" => "[group][name]"} + rename => { "[az][Type]" => "[event][type]"} + rename => { "[az][DeviceDetail][deviceId]" => "[host][id]"} + rename => { "[az][DeviceDetail][operatingSystem]" => "[host][os][name]"} + rename => { "[az][DeviceDetail][displayName]" => "[host][hostname]"} + } + mutate{ + gsub => ["[az][NetworkLocationDetails]", "[\r\n ]", ""] + gsub => ["[az][NetworkLocationDetails]", '\"', '"'] + } + json { + source => "[az][NetworkLocationDetails]" + target => "[az][NetworkLocationDetails]" + } + ruby { + code => " + networktype_list = [] + networknames_list = [] + + if event.get('[az][NetworkLocationDetails]').kind_of? Array + event.get('[az][NetworkLocationDetails]').each do |item| + if item['networkType'] != '' + networktype_list.push(item['networkType']) + end + if item['networkNames'] != '' + networknames_list.push(item['networkNames']) + end + end + end + + if !networktype_list.empty? + event.set('[network][type]', networktype_list) + end + if !networknames_list.empty? + event.set('[network][name]', networknames_list) + end + " + tag_on_exception => "ruby_block_exception" + } + json { + source => "[az][LocationDetails]" + target => "[az][LocationDetails]" + skip_on_invalid_json => true + } + json { + source => "[az][AuthenticationDetails]" + target => "[az][AuthenticationDetails]" + skip_on_invalid_json => true + } + # json { + # source => "[az][DeviceDetail]" + # target => "[az][DeviceDetail]" + # skip_on_invalid_json => true + # } + json { + source => "[az][AuthenticationProcessingDetails]" + target => "[az][AuthenticationProcessingDetails]" + skip_on_invalid_json => true + } + + # json { + # source => "[az][ConditionalAccessPolicies]" + # target => "[user][roles]" + # skip_on_invalid_json => true + # } + mutate { + remove_field => [ "tmp", "[az]" ] + # "tmp", "az", + } + # split { + # field => "[user][roles]" + # target => "[user][roles]" + # } + + mutate { + gsub => [ + "[event][created]", "^(.*?)T(.*?\.\d\d\d).*?$", "\1 \2" , + "[event][ingested]", "^(.*?)T(.*?\.\d\d\d).*?$", "\1 \2" + ] + } + date { + match => [ "[event][created]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } +} +output { + pipeline { send_to => [enrichments] } +} + diff --git a/config/processors/event_hub_audit_azure.event_hub_officeactivity.conf b/config/processors/event_hub_audit_azure.event_hub_officeactivity.conf index 299de8db..90402255 100644 --- a/config/processors/event_hub_audit_azure.event_hub_officeactivity.conf +++ b/config/processors/event_hub_audit_azure.event_hub_officeactivity.conf @@ -29,9 +29,12 @@ filter { skip_on_invalid_json => true } } - mutate { + mutate{ add_field => { "[event][module]" => "azure" } add_field => { "[event][dataset]" => "azure.officeactivity" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { rename => { "[az][timegenerated]" => "[event][ingested]" } rename => { "[az][site_]" => "[server][user][group][id]" } rename => { "[az][organizationid]" => "[organization][id]" } @@ -76,4 +79,5 @@ filter { } output { pipeline { send_to => [enrichments] } -} \ No newline at end of file +} + diff --git a/config/processors/event_hub_audit_azure.event_hub_riskyuser.conf b/config/processors/event_hub_audit_azure.event_hub_riskyuser.conf new file mode 100644 index 00000000..6ed3ba78 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_riskyuser.conf @@ -0,0 +1,62 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate{ + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.riskyuser" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { + rename => { "[az][riskdetail]" => "[rule][description]" } + rename => { "[az][risklastupdateddatetime]" => "[event][end]" } + rename => { "[az][risklevel]" => "[event][severity_name]"} + rename => { "[az][riskstate]" => "[event][outcome]" } + rename => { "[az][userdisplayname]" => "[user][name]" } + rename => { "[az][userprincipalname]" => "[user][email]" } + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][operationname]"=>"[rule][name]" } + rename => { "[az][correlationid]"=>"[event][id]" } + rename => { "[az][type]" => "[event][type]" } + } + mutate { + remove_field => [ "az", "tmp" ] + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/event_hub_audit_azure.event_hub_securityalert.conf b/config/processors/event_hub_audit_azure.event_hub_securityalert.conf new file mode 100644 index 00000000..d1c64114 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_securityalert.conf @@ -0,0 +1,82 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate{ + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.securityalert" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { + rename => { "[az][alertname]" => "[rule][name]" } + rename => { "[az][alertseverity]" => "[event][severity_name]" } + rename => { "[az][alerttype]" => "[rule][category]" } + rename => { "[az][description]" => "[rule][description]" } + rename => { "[az][displayname]" => "[user][name]" } + rename => { "[az][endtime]" => "[event][end]" } + rename => { "[az][providername]" => "[event][provider]" } + rename => { "[az][starttime]" => "[event][start]" } + rename => { "[az][status]" => "[event][outcome]" } + rename => { "[az][systemalertid]" => "[event][id]" } + rename => { "[az][tactics]" => "[threat][tactic][name]" } + rename => { "[az][techniques]" => "[threat][technique][name]" } + rename => { "[az][tenantid]" => "[rule][id]" } + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][vendorname]" => "[observer][vendor]" } + rename => { "[az][type]" => "[event][type]" } + } + mutate { + remove_field => [ "az", "tmp" ] + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } + date { + match => [ "[event][start]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][start]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][end]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][end]" + tag_on_failure => "_dateparsefailure_ee" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/event_hub_audit_azure.event_hub_securityincident.conf b/config/processors/event_hub_audit_azure.event_hub_securityincident.conf new file mode 100644 index 00000000..6f87e068 --- /dev/null +++ b/config/processors/event_hub_audit_azure.event_hub_securityincident.conf @@ -0,0 +1,85 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + if [message] =~ '^{"records": \[' { + json { + source => "message" + target => "tmp" + skip_on_invalid_json => true + } + split { + field => "[tmp][records]" + target => "az" + } + mutate { + replace => { "message" => "%{az}" } + } + } else { + json { + source => "message" + target => "az" + skip_on_invalid_json => true + } + } + mutate{ + add_field => { "[event][module]" => "azure" } + add_field => { "[event][dataset]" => "azure.securityincident" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { + rename => { "[az][createdtime]" => "[event][created]" } + rename => { "[az][description]" => "[rule][description]" } + rename => { "[az][firstactivitytime]" => "[event][start]" } + rename => { "[az][incidentnumber]" => "[event][id]" } + rename => { "[az][incidentname]" => "[rule][name]" } + rename => { "[az][lastmodifiedtime]" => "[event][modified]" } + rename => { "[az][providername]" => "[event][provider]" } + rename => { "[az][severity]" => "[event][severity_name]"} + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][type]" => "[event][type]" } + rename => { "[az][tenantid]" => "[rule][id]" } + rename => { "[az][title]" => "[event][kind]" } + } + mutate { + remove_field => [ "az", "tmp" ] + } + date { + match => [ "[event][created]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + date { + match => [ "[event][ingested]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } + date { + match => [ "[event][start]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][start]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][modified]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][modified]" + tag_on_failure => "_dateparsefailure_em" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/event_hub_audit_azure.event_hub_userriskevents.conf b/config/processors/event_hub_audit_azure.event_hub_userriskevents.conf index b6dca68c..2a855c3b 100644 --- a/config/processors/event_hub_audit_azure.event_hub_userriskevents.conf +++ b/config/processors/event_hub_audit_azure.event_hub_userriskevents.conf @@ -29,9 +29,12 @@ filter{ skip_on_invalid_json => true } } - mutate { + mutate{ add_field => { "[event][module]" => "azure" } add_field => { "[event][dataset]" => "azure.userriskevents" } + add_field => { "[log][source][hostname]" => "%{[az][tenantid]}" } + } + mutate { rename => { "[az][id]" => "[event][id]" } rename => { "[az][userid]" => "[user][id]" } rename => { "[az][ipaddress]" => "[client][ip]" } diff --git a/config/processors/flat_file_log_audit_sap.cloud_connector.conf b/config/processors/flat_file_log_audit_sap.cloud_connector.conf new file mode 100644 index 00000000..093bfaed --- /dev/null +++ b/config/processors/flat_file_log_audit_sap.cloud_connector.conf @@ -0,0 +1,64 @@ +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + if [message] =~ "^Date & Time" { + drop {} + } + mutate { + add_field => { + "[event][module]" => "sap" + "[event][dataset]" => "sap.cloud_connector" + } + copy => { "[host][name]" => "[log][source][hostname]" } + lowercase => [ "message" ] + } + dissect { + mapping => { + "message" => "%{[event][created]} %{+[event][created]} %{?data},%{?data},%{?data},%{?data},%{[event][action]},%{[rest_msg]}" + } + tag_on_failure => "_dissectfailure" + } + if [event][action] and [event][action] =~ "unspecified" { + grok { + match => { + "rest_msg" => "^.*? file (?<[file][name]>.*?) .*? serial (?<[file][x509][serial_number]>.*?) .*?" + } + tag_on_failure => "_grokfailure_1" + } + } + if [event][action] and [event][action] =~ "service_stopped|service_started" { + grok { + match => { + "rest_msg" => "(?<[process][name]>.*?) \[(?<[destination][address]>.*?)\] .*? account://(?<[user][name]>.*?) .*?" + } + tag_on_failure => "_grokfailure_2" + } + } + if [event][action] and [event][action] =~ "op_access_allowed" { + grok { + match => { + "rest_msg" => "^.*? user (?<[user][name]>.*?) to resource (?<[process][name]>.*?) on system (?<[destination][address]>.*?):(?<[destination][port]>.*?) .*?" + } + tag_on_failure => "_grokfailure_3" + } + } + mutate { + gsub => [ "[event][created]", "^(.*?) (.*?\.\d\d\d).*?$", "\1T\2" ] + } + date { + match => ["[event][created]" , "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "[agent][hostname]", "fields", "input", "type", "data" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/ftp_gzip_audit_infoblox_dns.conf b/config/processors/ftp_gzip_audit_infoblox_dns.conf index b215f03e..c3ade1e5 100644 --- a/config/processors/ftp_gzip_audit_infoblox_dns.conf +++ b/config/processors/ftp_gzip_audit_infoblox_dns.conf @@ -6,31 +6,20 @@ input { } } filter { - mutate { - add_field => { "[event][module]" => "infoblox" } - add_field => { "[event][dataset]" => "infoblox.dns_query" } - } - grok { - match => { "message" => "^(?<[event][created]>.*? .*?) client (?<[source][ip]>.*?)#(?<[source][port]>\d+): ((?<[network][transport]>\w+): )?query: (?<[dns][question][name]>.*?) (?<[dns][question][class]>.*?) (?<[dns][question][type]>.*?)(( \+)|( \-))? (\((?<[observer][ip]>.*?)\))?(response: (?<[dns][response_code]>.*?) (?<[dns][header_flags]>.*?)( (?.*?))?(;|$)|$)?" } - timeout_millis => 500 - } - if [response] { - mutate { - split => { "response" => "; " } - } - mutate { - gsub => [ "response", ";$", "$" ] - } - grok { - match => { "response" => "^((?<[dns][resolved_ip]>\d+\.\d+\.\d+\.\d+)|(?<[dns][answers][name]>.*?)) (?<[dns][id]>\d+) (?<[dns][answers][class]>.*?) (?<[dns][answers][type]>.*?) ((\()?(?<[observer][ip]>\d+\.\d+\.\d+\.\d+)(\))|(?<[observer][hostname]>.*?)(;|$))?" } - timeout_millis => 500 - } + grok { + match => { "message" => "^(?<[event][created]>.*? .*?) client (?<[source][ip]>.*?)#(?<[source][port]>\d+):( ((?<[network][transport]>\w+):))? query: (?<[dns][question][name]>.*?) IN (?<[dns][question][class]>.*?) (\((?<[observer][ip]>.*?)\)|response: (?<[dns][response_code]>.*?) (?<[dns][header_flags]>.*?)$)" } } + mutate { + add_field => { "[log][source][hostname]" => "ftp_infoblox_api" } + add_field => { "[event][module]" => "infoblox" } + add_field => { "[event][dataset]" => "infoblox.dns" } + } + mutate { copy => { "[dns][question][name]" => "[url][full]" } - remove_field => [ "response", "agent", "host" ] + remove_field => [ "response", "host", "path" ] } - # "[event][created]" => "07-Apr-2013 20:16:49.083", + # "[event][created]" => "08-Jul-2022 13:11:32.893", date { match => ["[event][created]", "dd-MMM-yyyy HH:mm:ss.SSS"] timezone => "GMT" @@ -39,7 +28,7 @@ filter { tag_on_failure => "_dateparsefailure_ec" } if "_dateparsefailure_ec" in [tags] { - mutate { + mutate { remove_field => ["[event][created]"] } } diff --git a/config/processors/log_cdp_hadoop.conf b/config/processors/log_cdp_hadoop.conf new file mode 100644 index 00000000..a1c3e3ed --- /dev/null +++ b/config/processors/log_cdp_hadoop.conf @@ -0,0 +1,47 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +# Ansible alert web hook to logstash https listener +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "cdp" } + add_field => { "[event][dataset]" => "cdp.hadoop" } + rename => { "[beat][hostname]" => "[log][source][hostname]" } + update => { "message" => "%{[event][original]}" } + remove_tag => [ "hadoop-log4j", "jobhistory", "beats", "beats_input_codec_plain_applied", "siem-kafka" ] + } + mutate { + rename => { + "@timestamp" => "[event][created]" + "[input][type]" => "[file][type]" + "program" => "[event][kind]" + "source" => "[file][path]" + "destination" => "[file][target_path]" + "tags" => "[tags]" + "severity" => "[event][severity_name]" + "[beat][name]" => "[agent][name]" + "[beat][hostname]" => "[host][hostname]" + "[beat][version]" => "[agent][version]" + "service" => "[service][name]" + "thread" => "[process][thread][id]" + "role" => "[user][roles]" + "[fields][environment]" => "[group][name]" + "[user]" => "[user][name]" + } + } + date { + match => [ "[event][created]", "ISO8601" ] + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "beat" , "input", "topic_name", "offset", "@version", "fields", "prospector", "level", "[event][original]"] + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/log_cdp_rundeck.conf b/config/processors/log_cdp_rundeck.conf new file mode 100644 index 00000000..92c7717d --- /dev/null +++ b/config/processors/log_cdp_rundeck.conf @@ -0,0 +1,45 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +# Ansible alert web hook to logstash https listener +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "cdp" } + add_field => { "[event][dataset]" => "cdp.rundeck" } + copy => { "[beat][hostname]" => "[log][source][hostname]" } + update => { "message" => "[event][original]" } + } + mutate { + rename => { + "@timestamp" => "[event][created]" + "[input][type]" => "[file][type]" + "program" => "[event][kind]" + "source" => "[file][path]" + "destination" => "[file][target_path]" + "tags" => "[tags]" + "severity" => "[event][severity_name]" + "[beat][name]" => "[agent][name]" + "[beat][hostname]" => "[host][hostname]" + "[beat][version]" => "[agent][version]" + "service" => "[service][name]" + "thread" => "[process][thread][id]" + "role" => "[user][roles]" + "[fields][environment]" => "[group][name]" + } + } + date { + match => [ "[event][created]", "ISO8601" ] + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "beat" , "input", "topic_name", "offset", "@version", "fields", "prospector", "level", "[event][original]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/log_cdp_tanium.conf b/config/processors/log_cdp_tanium.conf new file mode 100644 index 00000000..88f7c711 --- /dev/null +++ b/config/processors/log_cdp_tanium.conf @@ -0,0 +1,45 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +# Ansible alert web hook to logstash https listener +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "cdp" } + add_field => { "[event][dataset]" => "cdp.tanium" } + copy => { "[beat][hostname]" => "[log][source][hostname]" } + update => { "message" => "[event][original]" } + } + mutate { + rename => { + "@timestamp" => "[event][created]" + "[input][type]" => "[file][type]" + "program" => "[event][kind]" + "source" => "[file][path]" + "destination" => "[file][target_path]" + "tags" => "[tags]" + "severity" => "[event][severity_name]" + "[beat][name]" => "[agent][name]" + "[beat][hostname]" => "[host][hostname]" + "[beat][version]" => "[agent][version]" + "service" => "[service][name]" + "thread" => "[process][thread][id]" + "role" => "[user][roles]" + "[fields][environment]" => "[group][name]" + } + } + date { + match => [ "[event][created]", "ISO8601" ] + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "beat" , "input", "topic_name", "offset", "@version", "fields", "prospector", "level", "[event][original]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} diff --git a/config/processors/log_security_proofpoint.trap.conf b/config/processors/log_security_proofpoint.trap.conf new file mode 100644 index 00000000..3ea607ee --- /dev/null +++ b/config/processors/log_security_proofpoint.trap.conf @@ -0,0 +1,108 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "proofpoint" } + add_field => { "[event][dataset]" => "proofpoint.trap" } + add_field => { "[log][source][hostname]" => "api_proofpoint_trap" } + } + json { + source => "message" + target => "tmp" + } + mutate { + rename => { "[tmp][summary]" => "[event][reason]" } + rename => { "[tmp][event_sources]" => "[event][category]" } + rename => { "[tmp][created_at]" => "[event][created]" } + rename => { "[tmp][event_ids]" => "[event][id]" } + rename => { "[tmp][hosts][cnc]" => "[related][ip]" } + rename => { "[tmp][hosts][attacker]" => "[threat][list][ioc]" } + rename => { "[tmp][hosts][forensics]" => "[threat][list][event_analysis]" } + rename => { "[tmp][description]" => "[rule][description]" } + rename => { "[tmp][id]" => "[trace][id]" } + rename => { "[tmp][score]" => "[event][risk_score]" } + rename => { "[tmp][state]" => "[event][outcome]" } + rename => { "[tmp][closed_at]" => "[event][end]" } + rename => { "[tmp][updated_at]" => "[event][modified]" } + rename => { "[tmp][team]" => "[team][name]" } + rename => { "[tmp][assignee]" => "[assignee][user][name]" } + rename => { "[tmp][event_count]" => "[event][sequence]" } + rename => { "[tmp][close_detail]" => "[event][recommendation]" } + rename => { "[tmp][failed_quarantines]" => "[event][risk_score_norm]" } + + } + ruby { + code => " + detected_user_list = [] + detected_email_list=[] + if event.get('[tmp][users]').kind_of?(Array) + event.get('[tmp][users]').each do |item| + if item != '' + detected_user_list.push(item) + end + end + end + if event.get('[tmp][quarantine_results]').kind_of?(Array) + event.get('[tmp][quarantine_results]').each do |item| + if item['recipient'] != '' or 'null' + detected_email_list.push(item['recipient']) + end + end + end + if event.get('[tmp][incident_field_values]').kind_of?(Array) + event.get('[tmp][incident_field_values]').each do |item| + if item['name'] == 'Classification' + event.set('[threat][list][category]', item['value']) + end + if item['name'] == 'Attack Vector' + event.set('[threat][list][attribute_tag]', item['value']) + end + if item['name'] == 'Severity' + event.set('[event][severity_name]', item['value']) + end + end + end + if !detected_user_list.empty? + event.set('[source][user][name]', detected_user_list) + end + if !detected_email_list.empty? + event.set('[user][email]', detected_email_list) + end + " + tag_on_exception => "msgparts_ruby_block" + } + + mutate { + remove_field => [ "tmp" ] + } + date { + match => ["[event][created]","yyyy-MM-dd'T'HH:mm:ss'Z'"] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + date { + match => ["[event][end]","yyyy-MM-dd'T'HH:mm:ss'Z'"] + timezone => "GMT" + locale => "en" + target => "[event][end]" + tag_on_failure => "_dateparsefailure_ee" + } + date { + match => ["[event][modified]","yyyy-MM-dd'T'HH:mm:ss'Z'"] + timezone => "GMT" + locale => "en" + target => "[event][modified]" + tag_on_failure => "_dateparsefailure_em" + } + +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/s3_log_aws.cloudfront.conf b/config/processors/s3_log_aws.cloudfront.conf new file mode 100644 index 00000000..383a77e4 --- /dev/null +++ b/config/processors/s3_log_aws.cloudfront.conf @@ -0,0 +1,77 @@ +# Copyright [2022] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + if [message] == "" or ![message] or [message] =~ "^#" { + drop {} + } + mutate { + add_field => { + "[event][module]" => "aws" + "[event][dataset]" => "aws.cloudfront" + "[cloud][provider]" => "aws" + "[log][source][hostname]" => "aws_cloudfront" + } + } + grok { + match => { + "message" => '(-|(?<[tmp][date]>.*?)) (-|(?<[tmp][time]>.*?)) (-|(?<[tmp][x_edge_location]>.*?)) (-|(?<[tmp][sc_bytes]>.*?)) (-|(?<[tmp][c_ip]>.*?)) (-|(?<[tmp][cs_method]>.*?)) (-|(?<[tmp][cs_host]>.*?)) (-|(?<[tmp][cs_uri_stem]>.*?)) (-|(?<[tmp][sc_status]>.*?)) (-|(?<[tmp][cs_referer]>.*?)) (-|(?<[tmp][cs_user_agent]>.*?)) (-|(?<[tmp][cs_uri_query]>.*?)) (-|(?<[tmp][cs_cookie]>.*?)) (-|(?<[tmp][x_edge_result_type]>.*?)) (-|(?<[tmp][x_edge_request_id]>.*?)) (-|(?<[tmp][x_host_header]>.*?)) (-|(?<[tmp][cs_protocol]>.*?)) (-|(?<[tmp][cs_bytes]>.*?)) (-|(?<[tmp][time_taken]>.*?)) (-|(?<[tmp][x_forwarded_for]>.*?)) (-|(?<[tmp][ssl_protocol]>.*?)) (-|(?<[tmp][ssl_cipher]>.*?)) (-|(?<[tmp][x_edge_response_result_type]>.*?)) (-|(?<[tmp][cs_protocol_version]>.*?)) (-|(?<[tmp][fle_status]>.*?)) (-|(?<[tmp][fle_encrypted_fields]>.*?)) (-|(?<[tmp][c_port]>.*?)) (-|(?<[tmp][time_to_first_byte]>.*?)) (-|(?<[tmp][x_edge_detailed_result_type]>.*?)) (-|(?<[tmp][sc_content_type]>.*?)) (-|(?<[tmp][sc_content_len]>.*?)) (-|(?<[tmp][sc_range_start]>.*?)) (-|(?<[tmp][sc_range_end]>.*?))' + } + } + ## Did not map + # "[tmp][time_to_first_byte]" => "0.183" + # "[tmp][sc_content_len]" => "" + # "[tmp][sc_range_start]" => "" + # "[tmp][sc_range_end]" => "" + # "[tmp][x_edge_location]" => "" = Lookig to add location enrichment + # "[tmp][fle_status]" => "" + # "[tmp][fle_encrypted_fields]" => "" + mutate { + add_field => { + "[event][created]" => "%{[tmp][date]} %{[tmp][time]}" + } + rename => { + "[tmp][sc_bytes]" => "[http][response][bytes]" + "[tmp][c_ip]" => "[source][ip]" + "[tmp][cs_method]" => "[http][request][method]" + "[tmp][cs_host]" => "[cloud][instance][id]" + "[tmp][cs_uri_stem]" => "[url][path]" + "[tmp][sc_status]" => "[http][response][status_code]" + "[tmp][cs_referer]" => "[http][request][referrer]" + "[tmp][sc_content_type]" => "[http][request][mime_type]" + "[tmp][x_edge_response_result_type]" => "http.response.status_code" + "[tmp][x_edge_result_type]" => "[event][outcome]" + "[tmp][x_edge_detailed_result_type]" => "[event][reason]" + "[tmp][cs_user_agent]" => "[user_agent][original]" + "[tmp][cs_uri_query]" => "[url][query]" + "[tmp][cs_cookie]" => "[http][cookie][name]" + "[tmp][x_edge_request_id]" => "[event][id]" + "[tmp][x_host_header]" => "[url][domain]" + "[tmp][cs_protocol]" => "[url][scheme]" + "[tmp][cs_bytes]" => "[http][request][bytes]" + "[tmp][time_taken]" => "[event][duration]" + "[tmp][x_forwarded_for]" => "[network][forwarded_ip]" + "[tmp][ssl_protocol]" => "[tls][version_protocol]" + "[tmp][ssl_cipher]" => "[tls][cipher]" + "[tmp][cs_protocol_version]" => "[tls][next_protocol]" + "[tmp][c_port]" => "[source][port]" + } + } + # "event.created" => "2020-08-13 11:19:43" + date { + match => ["[event][created]", "yyyy-MM-dd HH:mm:ss"] + timezone => "GMT" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "[tmp]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/s3_log_aws.lb.conf b/config/processors/s3_log_aws.lb.conf new file mode 100644 index 00000000..896967ca --- /dev/null +++ b/config/processors/s3_log_aws.lb.conf @@ -0,0 +1,97 @@ +# Copyright [2022] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + if [message] == "" or ![message] { + drop {} + } + mutate { + add_field => { + "[event][module]" => "aws" + "[event][dataset]" => "aws.lb" + "[cloud][provider]" => "aws" + } + } + grok { + match => { + "message" => '^((?<[tmp][type]>.*?) )?(?<[tmp][ingested]>.*?) (?<[tmp][instance_id]>.*?) (?<[tmp][client_ip]>.*?):(?<[tmp][client_port]>\d+) (-|((?<[tmp][dst_ip]>.*?):(?<[tmp][dst_port]>\d+))) (-|(?<[tmp][request_processing_time]>\.*?))(-|(?<[tmp][target_processing_time]>.*?)) (-|(?<[tmp][response_processing_time]>.*?)) (-|(?<[tmp][elb_status_code]>.*?)) (-|(?<[tmp][target_status_code]>.*?)) (-|(?<[tmp][received_bytes]>.*?)) (-|(?<[tmp][sent_bytes]>.*?)) "(-|(?<[tmp][method]>.*?)) (?<[tmp][url_full]>.*?) (?<[tmp][next_protocol]>.*?)" "(-|(?<[tmp][user_agent]>.*?))" (-|(?<[tmp][ssl_cipher]>.*?)) (-|(?<[tmp][ssl_protocol]>.*?))( (-|(?<[tmp][target_group_arn]>.*?)) "(-|(?<[tmp][trace_id]>.*?))" "(-|(?<[tmp][domain_name]>.*?))" "(-|(?<[tmp][chosen_cert_arn]>.*?))" (-|(?<[tmp][matched_rule_priority]>.*?)) (-|(?<[tmp][request_creation_time]>.*?)) "(-|(?<[tmp][actions_executed]>.*?))" "(-|(?<[tmp][redirect_url]>.*?))" "(-|(?<[tmp][error_reason]>.*?))" "(-|(?<[tmp][target_port_list]>.*?))" "(-|(?<[tmp][target_status_code_list]>.*?))" "(-|(?<[tmp][classification]>.*?))" "(-|(?<[tmp][classification_reason]>.*?))")?$' + } + } + mutate{ + add_field => { + "[log][source][hostname]" => "%{[tmp][instance_id]}" + } + } + + # did not mapp + # "[tmp][target_processing_time]" + # "[tmp][response_processing_time]" + # "[tmp][target_group_arn]" + mutate { + rename => { + "[tmp][request_creation_time]" => "[event][created]" + "[tmp][ingested]" => "[event][ingested]" + "[tmp][request_processing_time]" => "[event][duration]" + "[tmp][matched_rule_priority]" => "[event][severity]" + "[tmp][classification]" => "[event][category]" + "[tmp][error_reason]" => "[event][reason]" + "[tmp][instance_id]" => "[cloud][instance][name]" + "[tmp][type]" => "[url][scheme]" + "[tmp][url_full]" => "[url][full]" + "[tmp][domain_name]" => "[url][domain]" + "[tmp][elb_status_code]" => "[http][response][status_code]" + "[tmp][received_bytes]" => "[http][request][bytes]" + "[tmp][sent_bytes]" => "[http][response][bytes]" + "[tmp][method]" => "[http][request][method]" + "[tmp][redirect_url]" => "[http][request][referrer]" + "[tmp][user_agent]" => "[user_agent][name]" + "[tmp][ssl_cipher]" => "[tls][cipher]" + "[tmp][ssl_protocol]" => "[tls][version_protocol]" + "[tmp][chosen_cert_arn]" => "[tls][client][server_name]" + "[tmp][trace_id]" => "[trace][id]" + "[tmp][actions_executed]" => "[rule][name]" + "[tmp][classification_reason]" => "[rule][description]" + "[tmp][client_port]" => "[source][port]" + "[tmp][client_ip]" => "[source][ip]" + "[tmp][dst_ip]" => "[destination][ip]" + "[tmp][dst_port]" => "[destination][port]" + "[tmp][next_protocol]" => "[tls][next_protocol]" + } + } + if [tmp][target_port_list] != "-" { + mutate { + split => { "[tmp][target_port_list]" => " " } + } + grok { + match => { "[tmp][target_port_list]" => "(?<[destination][ip]>.*?):(?<[destination][port]>.*?)$" } + } + } + if [tmp][target_status_code] != "-" { + mutate { + split => { "[tmp][target_status_code]" => " " } + } + } + # "time" => "2023-03-11T11:30:53.264914Z" + date { + match => ["[event][created]", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] + timezone => "GMT" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + date { + match => ["[event][ingested]", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] + timezone => "GMT" + target => "[event][ingested]" + tag_on_failure => "_dateparsefailure_ei" + } + mutate { + remove_field => [ "[tmp]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/snowflake_acc_access_history.conf b/config/processors/snowflake_acc_access_history.conf new file mode 100644 index 00000000..fdf9681d --- /dev/null +++ b/config/processors/snowflake_acc_access_history.conf @@ -0,0 +1,40 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { + "[event][module]" => "snowflake" + "[event][dataset]" => "snowflake.access_history" + "[log][source][hostname]" => "sql_snowflake_access_history" + } + } + json { + source => "message" + target => "tmp" + } + mutate { + rename => { + "[tmp][QUERY_ID]" => "[database][user][id]" + "[tmp][QUERY_START_TIME]" => "[event][start]" + "[tmp][USER_NAME]" => "[user][name]" + } + } + date { + match => [ "[event][start]", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[event][start]" + tag_on_failure => "_dateparsefailure_es" + } + mutate { + remove_field => [ "tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/syslog_as400_audit_test.conf b/config/processors/syslog_as400_audit_test.conf new file mode 100644 index 00000000..5deb55d9 --- /dev/null +++ b/config/processors/syslog_as400_audit_test.conf @@ -0,0 +1,64 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { + "[event][module]" => "as400" + "[event][dataset]" => "as400.audit" + } + lowercase => [ "message" ] + } + grok { + tag_on_failure => "_parsefailure_header" + match => { "message" => "^(.*?{name=.*?}(\s)?)?(<(?\d+)>)?(\s)?(?.*)$|(^(?.*)$)" } + timeout_millis => 500 + } + syslog_pri { + syslog_pri_field_name => "pri" + remove_field => [ "pri" ] + } + json { + source => "actual_msg" + target => "tmp" + } + mutate { + rename => { + "[tmp][a_date]" => "date" + "[tmp][b_time]" => "time" + "[tmp][c_system]" => "[host][name]" + "[tmp][d_event]" => "[event][id]" + "[tmp][e_command]" => "[process][command_line]" + "[tmp][f_remote_ip_add]" => "[host][ip]" + "[tmp][g_job_name]" => "[rule][name]" + "[tmp][h_job_number]" => "[rule][id]" + "[tmp][i_job_user]" => "[user][name]" + } + } + if [host][name] { + mutate { + add_field => { "[log][source][hostname]" => "%{[host][name]}" } + } + } + mutate { + add_field => { "[event][created]" => "%{date} %{time}" } + } + #2023-02-08 07.03.36 + date { + match => [ "[event][created]" , "yyyy-MM-dd HH.mm.ss" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "actual_msg", "tmp", "date", "time", "[log][original]" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/syslog_audit_aix_rsyslog.conf b/config/processors/syslog_audit_aix_rsyslog.conf new file mode 100644 index 00000000..8915f400 --- /dev/null +++ b/config/processors/syslog_audit_aix_rsyslog.conf @@ -0,0 +1,65 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "aix" } + add_field => { "[event][dataset]" => "aix.rsyslog" } + } + grok { + tag_on_failure => "_parsefailure_header" + match => { "message" => "(^(.*?)(<(?\d+)>)(\s)?(?.*$))|(^(?.*)$)" } + timeout_millis => 500 + } + syslog_pri { + syslog_pri_field_name => "pri" + remove_field => [ "pri" ] + } + dissect { + mapping => { + "actual_msg" => "%{?data}} %{rest_message}" + } + tag_on_failure => [ "_dissectfailure_1" ] + } + mutate { + gsub => [ "rest_message", '\"', '"' ] + } + json { + source => "rest_message" + target => "tmp" + } + mutate { + gsub => [ "[tmp][message]", "(\s+)", " " ] + } + mutate { + rename => { "[tmp][severity]" => "[event][severity]" } + rename => { "[tmp][type]" => "[event][type]" } + rename => { "[tmp][message]" => "[event][original]" } + } + mutate { + lowercase => [ "[event][original]" ] + } + dissect { + mapping => { + "[event][original]" => "%{?data} %{[event][created]} %{+[event][created]} %{+[event][created]} %{+[event][created]} %{?data} %{?data} %{[user][name]} %{?data} %{?data} %{?data} %{?data} %{[process][command_line]}" + } + tag_on_failure => [ "_dissectfailure_2" ] + } + date { + match => [ "[event][created]", "MMM dd HH:mm:ss yyyy" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "actual_msg", "rest_message", "tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/syslog_securityonion_securityonion.zeek.conf b/config/processors/syslog_securityonion_securityonion.zeek.conf new file mode 100644 index 00000000..63ac3356 --- /dev/null +++ b/config/processors/syslog_securityonion_securityonion.zeek.conf @@ -0,0 +1,603 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + if [dataset] == "syslog" { + drop {} + } + mutate { + add_field => { + "[event][module]" => "securityonion" + "[event][dataset]" => "securityonion.zeek" + } + } + json { + source => "message" + target => "tmp" + } + mutate { + copy => { "[host][name]" => "[log][source][hostname]" } + rename => { "[category]" => "[event][category]" } + rename => { "[dataset]" => "[event][kind]" } + rename => { "[tmp][ts]" => "[event][created]" } + rename => { "[tmp][proto]" => "[network][protocol]" } + rename => { "[tmp][id.orig_h]" => "[source][ip]" } + rename => { "[tmp][id.orig_p]" => "[source][port]" } + rename => { "[tmp][id.resp_h]" => "[destination][ip]" } + rename => { "[tmp][id.resp_p]" => "[destination][port]" } + rename => { "[tmp][orig_bytes]" => "[source][bytes]" } + rename => { "[tmp][resp_bytes]" => "[destination][bytes]" } + rename => { "[tmp][orig_pkts]" => "[source][packets]" } + rename => { "[tmp][resp_pkts]" => "[destination][packets]" } + rename => { "[tmp][duration]" => "[event][duration]" } + rename => { "[tmp][community_id]" => "[rule][uuid]" } + rename => { "[tmp][orig_mac_oui]" => "[network][name]" } + rename => { "[tmp][times.modified]" => "[file][mtime]" } + rename => { "[tmp][times.created]" => "[file][created]" } + rename => { "[tmp][times.changed]" => "[file][ctime]" } + rename => { "[tmp][times.accessed]" => "[file][accessed]" } + rename => { "[tmp][protoqtype_name]" => "[dns][question][type]" } + rename => { "[tmp][query]" => "[dns][question][name]" } + rename => { "[tmp][answers]" => "[dns][answers]" } + rename => { "[tmp][method]" => "[http][request][method]" } + rename => { "[tmp][user_agent]" => "[user_agent][original]" } + rename => { "[tmp][status_code]" => "[http][response][status_code]" } + rename => { "[tmp][status_msg]" => "[event][action]" } + rename => { "[tmp][resp_mime_type]" => "[http][request][mime_type]" } + rename => { "[tmp][proxied]" => "[event][code]" } + rename => { "[tmp][size]" => "[file][size]" } + rename => { "[tmp][path]" => "[file][path]" } + rename => { "[tmp][action]" => "[event][action]" } + rename => { "[tmp][server_name]" => "[destination][address]" } + rename => { "[tmp][ja3]" => "[tls][client][ja3]" } + rename => { "[tmp][ja3s]" => "[tls][server][ja3s]" } + rename => { "[tmp][validation_status]" => "[event][action]" } + rename => { "[tmp][tunnel_type]" => "[service][type]" } + rename => { "[tmp][endpoint]" => "[event][action]" } + rename => { "[tmp][host]" => "[url][domain]" } + rename => { "[tmp][uid]" => "[event][id]" } + rename => { "[tmp][conn_state]" => "[service][state]" } + rename => { "[tmp][orig_ip_bytes]" => "[host][network][egress][bytes]" } + rename => { "[tmp][resp_ip_bytes]" => "[host][network][ingress][bytes]" } + rename => { "[tmp][client_addr]" => "[source][ip]" } + rename => { "[tmp][server_addr]" => "[destination][ip]" } + rename => { "[tmp][host_name]" => "[host][hostname]" } + rename => { "[tmp][domain]" => "[host][domain]" } + rename => { "[tmp][assigned_addr]" => "[host][ip]" } + rename => { "[tmp][user]" => "[user][name]" } + rename => { "[tmp][command]" => "[process][command_line]" } + rename => { "[tmp][file_size]" => "[file.size]" } + rename => { "[tmp][reply_code]" => "[http.response.status_code]" } + rename => { "[tmp][value]" => "[http][request][body][content]" } + rename => { "[tmp][request_type]" => "[process][name]" } + rename => { "[tmp][till]" => "[tls.client][not_after]" } + rename => { "[tmp][cipher]" => "[tls][cipher]" } + rename => { "[tmp][func]" => "[event][outcome]" } + rename => { "[tmp][result]" => "[event][outcome]" } + rename => { "[tmp][ttl]" => "[event][duration]" } + rename => { "[tmp][security_protocol]" => "[event][outcome]" } + rename => { "[tmp][request_from]" => "[source][address]" } + rename => { "[tmp][request_to]" => "[destination][address]" } + rename => { "[tmp][community]" => "[network][community_id]" } + rename => { "[tmp][up_since]" => "[process][start]" } + rename => { "[tmp][request_p]" => "[client][port]" } + rename => { "[tmp][bound.host]" => "[server][address]" } + rename => { "[tmp][bound_p]" => "[server][port]" } + rename => { "[tmp][request.host]" => "[client][address]" } + rename => { "[tmp][machine]" => "[file][pe][architecture]" } + rename => { "[tmp][section_names]" => "[file][pe][original_file_name]" } + rename => { "[tmp][compile_ts]" => "[file][created]" } + rename => { "[tmp][os]" => "[host][os][full]" } + rename => { "[tmp][subsystem]" => "[file][pe][product]" } + rename => { "[tmp][fingerprint]" => "[file][hash][sha256]" } + rename => { "[tmp][certificate.version]" => "[file][x509][version_number]" } + rename => { "[tmp][certificate.serial]" => "[file][x509][serial_number]" } + rename => { "[tmp][certificate.not_valid_before]" => "[file][x509][not_before]" } + rename => { "[tmp][certificate.not_valid_after]" => "[file][x509][not_after]" } + rename => { "[tmp][certificate.key_alg]" => "[file][x509][public_key_algorithm]" } + rename => { "[tmp][certificate.key_length]" => "[file][x509][public_key_size]" } + rename => { "[tmp][certificate.sig_alg]" => "[file][x509][signature_algorithm]" } + rename => { "[tmp][certificate.curve]" => "[file][x509][public_key_curve]" } + rename => { "[tmp][certificate.exponent]" => "[file][x509][public_key_exponent]" } + rename => { "[tmp][san.dns]" => "[file][x509][alternative_names]" } + rename => { "[tmp][certificate.subject]" => "[file][x509][subject][distinguished_name]" } + rename => { "[tmp][certificate.issuer]" => "[file][x509][issuer][distinguished_name]" } + rename => { "[tmp][fuid]" => "[file][uid]" } + rename => { "[tmp][note]" => "[event][code]" } + rename => { "[tmp][msg]" => "[error][message]" } + rename => { "[tmp][sub]" => "[event][original]" } + rename => { "[tmp][peer_descr]" => "[observer][name]" } + rename => { "[tmp][actions]" => "[event][action]" } + rename => { "[tmp][email_dest]" => "[destination][user][email]" } + rename => { "[tmp][suppress]" => "[event][duration]" } + rename => { "[tmp][analyzer]" => "[process][name]" } + rename => { "[tmp][failure_reason]" => "[error][message]" } + rename => { "[tmp][peer]" => "[observer][name]" } + rename => { "[tmp][source]" => "[service][name]" } + rename => { "[tmp][share_type]" => "[file][device]" } + rename => { "[tmp][reply_msg]" => "[event][outcome]" } + rename => { "[tmp][fc_request]" => "[dns][op_code]" } + rename => { "[tmp][fc_reply]" => "[dns][response_code]" } + rename => { "[tmp][lease_time]" => "[process][uptime]" } + rename => { "[tmp][msg_types]" => "[event.outcome]" } + rename => { "[tmp][trans_id]" => "[transaction][id]" } + rename => { "[tmp][rtt]" => "[event][duration]" } + rename => { "[tmp][qclass_name]" => "[dns][question][class]" } + rename => { "[tmp][qtype_name]" => "[dns][question][type]" } + rename => { "[tmp][rcode_name]" => "[dns][response_code]" } + rename => { "[tmp][filename]" => "[file][name]" } + rename => { "[tmp][total_bytes]" => "[source][bytes]]" } + rename => { "[tmp][seen_bytes]" => "[destination][bytes]" } + rename => { "[tmp][md5]" => "[file][hash][md5]" } + rename => { "[tmp][sha1]" => "[file][hash][sha1]" } + rename => { "[tmp][mime_type]" => "[file][mime_type]" } + rename => { "[tmp][extracted_size]" => "[file][size]" } + rename => { "[tmp][data_channel.orig_h]" => "[source][nat][ip]" } + rename => { "[tmp][data_channel.resp_h]" => "[destination][nat][ip]" } + rename => { "[tmp][data_channel.resp_p]" => "[destination][nat][port]" } + rename => { "[tmp][referrer]" => "[http][request][referrer]" } + rename => { "[tmp][file_desc]" => "[file][pe][description]" } + rename => { "[tmp][file_mime_type]" => "[file][mime_type]" } + rename => { "[tmp][seen.node]" => "[service][node][name]" } + rename => { "[tmp][sources]" => "[event][provider]" } + rename => { "[tmp][seen.where]" => "[event][reference]" } + rename => { "[tmp][seen.indicator]" => "[rule][name]" } + rename => { "[tmp][seen.indicator_type]" => "[rule][reference]" } + rename => { "[tmp][dcc_file_name]" => "[file][name]" } + rename => { "[tmp][dcc_file_size]" => "[file][size]" } + rename => { "[tmp][dcc_mime_type]" => "[file][mime_type]" } + rename => { "[tmp][nick]" => "[user][effective][name]" } + rename => { "[tmp][addl]" => "[rule][description]" } + rename => { "[tmp][error_msg]" => "[error][message]" } + rename => { "[tmp][from]" => "[source][user][email]" } + rename => { "[tmp][server_cert_subject]" => "[tls][server][certificate]" } + rename => { "[tmp][exception]" => "[error][message]" } + rename => { "[tmp][cmd]" => "[database][instance]" } + rename => { "[tmp][mime]" => "[file][mime_type]" } + rename => { "[tmp][desc]" => "[file][pe][description]" } + rename => { "[tmp][src]" => "[source][nat][ip]" } + rename => { "[tmp][dst]" => "[destination][nat][ip]" } + rename => { "[tmp][client_name]" => "[client][user][name]" } + rename => { "[tmp][authentication_method]" => "[event][type]" } + rename => { "[tmp][event_msg]" => "[event][reason]" } + rename => { "[tmp][sig_id]" => "[rule][id]" } + rename => { "[tmp][sub_msg]" => "[rule][description]" } + rename => { "[tmp][helo]" => "[client][user][domain]" } + rename => { "[tmp][useragent]" => "[user_agent][original]" } + rename => { "[tmp][fuids]" => "[file][uid]" } + rename => { "[tmp][analyzers]" => "[process][name]" } + rename => { "[tmp][subject]" => "[email][subject]" } + rename => { "[tmp][mailfrom]" => "[source][user][email]" } + rename => { "[tmp][direction]" => "[network][direction]" } + rename => { "[tmp][hostname]" => "[host][hostname]" } + rename => { "[tmp][domainname]" => "[host][domain]" } + rename => { "[tmp][server_dns_computer_name]" => "[server][address]" } + rename => { "[tmp][server_tree_name]" => "[server][domain]" } + rename => { "[tmp][server_nb_computer_name]" => "[observer][name]" } + rename => { "[tmp][status]" => "[event][action]" } + rename => { "[tmp][history]" => "[service][id]" } + rename => { "[tmp][request_body_len]" => "[http][request][body][bytes]" } + rename => { "[tmp][response_body_len]" => "[http][response][body][bytes]" } + rename => { "[tmp][matched]" => "[threat][list][type]" } + rename => { "[tmp][suppress_for]" => "[event][duration]" } + rename => { "[tmp][software_type]" => "[package][type]" } + rename => { "[tmp][missed_bytes]" => "[network][bytes]" } + rename => { "[tmp][named_pipe]" => "[group][name]" } + rename => { "[tmp][operation]" => "[service][name]" } + rename => { "[tmp][reply_code]" => "[event][code]" } + rename => { "[tmp][rows]" => "[event][sequence]" } + rename => { "[tmp][id]" => "[event][id]" } + rename => { "[tmp][cert_count]" => "[process][args_count]" } + rename => { "[tmp][desktop_name]" => "[service][id]" } + rename => { "[tmp][last_reply]" => "[event][action]" } + rename => { "[tmp][trans_depth]" => "[event][risk_score]" } + rename => { "[tmp][kex_alg]" => "[tls][server][x509][signature_algorithm]" } + rename => { "[tmp][client]" => "[tls][client][x509][version_number]" } + rename => { "[tmp][server]" => "[tls][server][x509][version_number]" } + rename => { "[tmp][auth_attempts]" => "[event][sequence]" } + rename => { "[tmp][host_key_alg]" => "[tls][server][x509][public_key_algorithm]" } + lowercase => [ "[event][kind]" ] + } + if [event][kind] == "smtp" { + ruby { + code => " + email_recipient = [] + if event.get('[tmp][rcptto]').kind_of? Array + event.get('[tmp][rcptto]').each do |item| + if item!= '' + email_recipient.push(item) + end + end + end + if event.get('[tmp][cc]').kind_of? Array + event.get('[tmp][cc]').each do |item| + if item!= '' + email_recipient.push(item) + end + end + end + if !email_recipient.empty? + event.set('[destination][user][email]', email_recipient) + end + " + tag_on_exception => "ruby_block_exception" + } + } + + if [event][kind] == "x509" { + # KV subject distinguished_name + kv { + source => "[file][x509][subject][distinguished_name]" + target => "[tmp][subject_split]" + transform_key => "lowercase" + value_split => "=" + field_split => "," + } + # issuer distinguished_name + kv { + source => "[file][x509][issuer][distinguished_name]" + target => "[tmp][issuer_split]" + transform_key => "lowercase" + value_split => "=" + field_split => "," + } + # rename issuer and subject distinguished_name subfields + mutate { + rename => { "[tmp][subject_split][countryname]" => "[file][x509][subject][country]" } + rename => { "[tmp][subject_split][c]" => "[file][x509][subject][country]" } + rename => { "[tmp][subject_split][organizationname]" => "[file][x509.subject][organization]" } + rename => { "[tmp][subject_split][o]" => "[file][x509][subject][organization]" } + rename => { "[tmp][subject_split][organizationalunitname]" => "[file][x509][subject][organizational_unit]" } + rename => { "[tmp][subject_split][ou]" => "[file][x509][subject][organizational_unit]" } + rename => { "[tmp][subject_split][commonname]" => "[file][x509][subject][common_name]" } + rename => { "[tmp][subject_split][cn]" => "[file][x509][subject][common_name]" } + rename => { "[tmp][subject_split][stateorprovincename]" => "[file][x509][subject][state_or_province]" } + rename => { "[tmp][subject_split][st]" => "[file][x509][subject][state_or_province]" } + rename => { "[tmp][subject_split][locality]" => "[file][x509][subject][locality]" } + rename => { "[tmp][subject_split][l]" => "[file][x509][subject][locality]" } + rename => { "[tmp][subject_split][role]" => "[user][roles]" } + rename => { "[tmp][subject_split][title]" => "[user][title]" } + rename => { "[tmp][issuer_split][countryname]" => "[file][x509][issuer][country]" } + rename => { "[tmp][issuer_split][c]" => "[file][x509][issuer][country]" } + rename => { "[tmp][issuer_split][organizationname]" => "[file][x509.subject][organization]" } + rename => { "[tmp][issuer_split][o]" => "[file][x509][issuer][organization]" } + rename => { "[tmp][issuer_split][organizationalunitname]" => "[file][x509][issuer][organizational_unit]" } + rename => { "[tmp][issuer_split][ou]" => "[file][x509][issuer][organizational_unit]" } + rename => { "[tmp][issuer_split][commonname]" => "[file][x509][issuer][common_name]" } + rename => { "[tmp][issuer_split][cn]" => "[file][x509][issuer][common_name]" } + rename => { "[tmp][issuer_split][stateorprovincename]" => "[file][x509][issuer][state_or_province]" } + rename => { "[tmp][issuer_split][st]" => "[file][x509][issuer][state_or_province]" } + rename => { "[tmp][issuer_split][locality]" => "[file][x509][issuer][locality]" } + rename => { "[tmp][issuer_split][l]" => "[file][x509][issuer][locality]" } + rename => { "[tmp][issuer_split][role]" => "[source][user][roles]" } + rename => { "[tmp][issuer_split][title]" => "[source][user][title]" } + } + # create full_name field if subject has user info + if [tmp][subject_split][givenname] { + mutate { + add_field => { "[user][full_name]" => "%{[tmp][subject_split][givenname]} %{[tmp][subject_split][surname]}" } + } + } else if [tmp][subject_split][gn] { + mutate { + add_field => { "[user][full_name]" => "%{[tmp][subject_split][gn]} %{[tmp][subject_split][sn]}" } + } + } + # create full_name field if issuer has user info + if [tmp][issuer_split][givenname] { + mutate { + add_field => { "[source][user][full_name]" => "%{[tmp][issuer_split][givenname]} %{[tmp][issuer_split][surname]}" } + } + } else if [tmp][issuer_split][gn] { + mutate { + add_field => { "[source][user][full_name]" => "%{[tmp][issuer_split][gn]} %{[tmp][issuer_split][sn]}" } + } + } + } + + if [event][kind] == "sip" { + grok { + match => { "[tmp][uri]" => "^(?<[url][query]>sip:(?.*?)@(?<[client][ip]>.*?)(?<[client][port]>.*?)?;transport=(?<[network][transport]>).*?)$" } + } + } else { + mutate { + rename => { "[tmp][uri]" => "[url][query]" } + } + } + if [event][kind] == "rdp" { + grok { + match => { "[tmp][cookie]" => "^(?<[user][domain]>.*?)\\+(?<[user][name]>.*?)$" } + } + } + + if [event][kind] == "mysql" { + mutate { + rename => { "[tmp][arg]" => "[process][command_line]" } + } + } else { + mutate { + rename => { "[tmp][arg]" => "[url][full]" } + } + } + + if [event][kind] == "radius" or [event][kind] == "ntlm" { + mutate { + rename => { "[tmp][username]" => "[user][name]" } + } + } else if [tmp][username] { + grok { + match => { "[tmp][username]" => "^((?<[user][domain]>.*?)\\+)?(?<[user][name]>.*?)$" } + } + } + if [event][kind] != "ssh" and [tmp][client]{ + grok { + match => { "[tmp][client]" => "^(?<[user][name]>.*?)\/(?<[user][domain]>.*?)$" } + } + } + if [event][kind] == "kerberos" { + grok { + match => { "[tmp][service]" => "^(?<[service][name]>.*?)\/(?<[destination][domain]>.*?)$" } + } + } else { + mutate { + rename => { "[tmp][service]" => "[service][name]" } + } + } + if [event][kind] == "dhcp" { + mutate { + rename => { "[tmp][mac]" => "[host][mac]" } + } + } else { + mutate { + rename => { "[tmp][mac]" => "[source][mac]" } + } + } + if [event][kind] == "tls" { + mutate { + rename => { "[tmp][version]" => "[tls][version]" } + } + } else { + mutate { + rename => { "[tmp][version]" => "[service][version]" } + } + } + if [tmp][success] { + mutate { + add_tag => [ "success" ] + } + } + if [event][kind] == "smb_file" { + grok { + match => { "[tmp][name]" => "^(?<[url][full]>.*\\(?<[file][name]>.*?))$" } + } + } else { + mutate { + rename => { "[tmp][name]" => "[rule][name]" } + } + } + # add tags + if [tmp][sni_matches_cert] { + mutate { + add_tag => [ "sni_matches_cert" ] + } + } + if [tmp][local_orig] { + mutate { + add_tag => [ "local_orig" ] + } + } + if [tmp][local_resp] { + mutate { + add_tag => [ "local_resp" ] + } + } + if [tmp][forwardable] { + mutate { + add_tag => [ "forwardable" ] + } + } + if [tmp][renewable] { + mutate { + add_tag => [ "renewable" ] + } + } + if [tmp][is_webmail] { + mutate { + add_tag => [ "is_webmail" ] + } + } + if [tmp][is_64it] { + mutate { + add_tag => [ "is_64it" ] + } + } + if [tmp][is_exe] { + mutate { + add_tag => [ "is_exe" ] + } + } + if [tmp][uses_aslr] { + mutate { + add_tag => [ "uses_aslr" ] + } + } + if [tmp][uses_dep] { + mutate { + add_tag => [ "uses_dep" ] + } + } + if [tmp][uses_code_integrity] { + mutate { + add_tag => [ "uses_code_integrity" ] + } + } + if [tmp][uses_seh] { + mutate { + add_tag => [ "uses_seh" ] + } + } + if [tmp][has_import_table] { + mutate { + add_tag => [ "has_import_table" ] + } + } + if [tmp][has_export_table] { + mutate { + add_tag => [ "has_export_table" ] + } + } + if [tmp][has_cert_table] { + mutate { + add_tag => [ "has_cert_table" ] + } + } + if [tmp][has_debug_data] { + mutate { + add_tag => [ "has_debug_data" ] + } + } + if [tmp][host_cert] { + mutate { + add_tag => [ "host_cert" ] + } + } + if [tmp][client_cert] { + mutate { + add_tag => [ "client_cert" ] + } + } + if [tmp][notice] { + mutate { + add_tag => [ "notice" ] + } + } + if [tmp][auth_success] { + mutate { + add_tag => [ "auth_success" ] + } + } + if [tmp][auth] { + mutate { + add_tag => [ "auth_success" ] + } + } + if [tmp][share_flag] { + mutate { + add_tag => [ "share_flag" ] + } + } + if [tmp][tls] { + mutate { + add_tag => [ "tls" ] + } + } + if [event][kind] == "smtp" { + mutate { + remove_field => ["[file][path]" ] + } + } + if [tmp][dropped] { + mutate { + add_tag => [ "dropped" ] + } + } + if [event][kind] == "dns" { + if [tmp][AA]{ + mutate { + add_tag => [ "authoritative" ] + } + } + if [tmp][TC]{ + mutate { + add_tag => [ "truncated" ] + } + } + if [tmp][RD]{ + mutate { + add_tag => [ "recursion desired" ] + } + } + if [tmp][RA]{ + mutate { + add_tag => [ "recursion available" ] + } + } + if [tmp][rejected]{ + mutate { + add_tag => [ "rejected" ] + } + } + } + mutate { + remove_field => [ "tmp", "type", "metadata.beat", "metadata.ip_address", "metadata.type", "metadata.version", "module" ] + } + date { + match => [ "[process][start]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[process][start]" + tag_on_failure => "_dateparsefailure_ps" + } + date { + match => [ "[tls.client][not_after]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[tls.client][not_after]" + tag_on_failure => "_dateparsefailure_tcnf" + } + date { + match => [ "[file][mtime]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][mtime]" + tag_on_failure => "_dateparsefailure_fmt" + } + date { + match => [ "[file][created]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][created]" + tag_on_failure => "_dateparsefailure_fc" + } + date { + match => [ "[file][ctime]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][ctime]" + tag_on_failure => "_dateparsefailure_fct" + } + date { + match => [ "[file][accessed]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][accessed]" + tag_on_failure => "_dateparsefailure_fa" + } + date { + match => [ "[event][created]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[file][x509][not_before]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][x509][not_before]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[file][x509][not_after]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][x509][not_after]" + tag_on_failure => "_dateparsefailure_es" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/syslog_virtustream_trendmicro_forwarder.conf b/config/processors/syslog_virtustream_trendmicro_forwarder.conf new file mode 100644 index 00000000..5c88bbf9 --- /dev/null +++ b/config/processors/syslog_virtustream_trendmicro_forwarder.conf @@ -0,0 +1,54 @@ +# Copyright [2021] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + add_field => { "[event][module]" => "virtustream" } + add_field => { "[event][dataset]" => "virtustream.trendmicro" } + } + json { + source => "message" + target => "tmp_msg" + } + dissect { + mapping => { + "[tmp_msg][message]" => "<%{?data}>%{[[event][created]]->} %{+[[event][created]]} %{+[[event][created]]} %{[[host][hostname]]} %{?data}|%{[[event][provider]]}|%{?data}|%{[[rule][version]]}|%{?data}|%{[[event][type]]}|%{[[event][severity]]}|%{rest_msg}" + } + } + kv { + source => "rest_msg" + field_split => " " + value_split => "=" + target => "tmp" + } + mutate { + rename => { "[tmp][src]" => "[source][ip]" } + rename => { "[tmp][dst]" => "[destination][ip]" } + rename => { "[tmp][spt]" => "[source][port]" } + rename => { "[tmp][dpt]" => "[destination][port]" } + rename => { "[tmp][smac]" => "[source][mac]" } + rename => { "[tmp][dmac]" => "[destination][mac]" } + rename => { "[tmp][proto]" => "[network][transport]" } + rename => { "[tmp][cn1]" => "[host][id]" } + rename => { "[tmp][in]" => "[source][bytes]" } + rename => { "[tmp][act]" => "[event][action]" } + rename => { "[tmp][dvchost]" => "[observer][hostname]" } + rename => { "[tmp][TrendMicroDsFrameType]" => "[observer][egress][zone]" } + copy => { "[host][hostname]" => "[log][source][hostname]"} + } + date { + match => [ "[event][created]", "MMM dd HH:mm:ss" ] + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } + mutate { + remove_field => [ "rest_msg", "tmp", "tmp_msg" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/vuln_gitguardian_internal.conf b/config/processors/vuln_gitguardian_internal.conf new file mode 100644 index 00000000..c897fe6b --- /dev/null +++ b/config/processors/vuln_gitguardian_internal.conf @@ -0,0 +1,54 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + json { + source => "message" + target => "tmp" + } + mutate { + add_field => { + "[event][module]" => "gitguardian" + "[event][dataset]" => "vuln.gitguardian_api" + "[log][source][hostname]" => "vuln_gitguardian" + } + rename => { "[tmp][date]" => "[event][created]" } + rename => { "[tmp][gitguardian_url]" => "[event][reference]" } + rename => { "[tmp][id]" => "[event][id]" } + rename => { "[tmp][tags]" => "[tags]" } + rename => { "[tmp][regression]" => "[vulnerability][regression]" } + rename => { "[tmp][resolved_at]" => "[event][end]" } + rename => { "[tmp][secret_hash]" => "[related][hash]" } + rename => { "[tmp][status]" => "[vulnerability][status]" } + rename => { "[tmp][validity]" => "[vulnerability][exploitable]" } + rename => { "[tmp][severity]" => "[vulnerability][severity]" } + rename => { "[tmp][detect_group_name]" => "[rule][ruleset]" } + rename => { "[tmp][repository_url]" => "[repository][name]" } + rename => { "[tmp][matched_on]" => "[rule][name]" } + rename => { "[tmp][commit_author]" => "[user][email]" } + rename => { "[tmp][file_path]" => "[file][path]" } + rename => { "[tmp][full_name]" => "[network][application]" } + rename => { "[tmp][visibility]" => "[repository][visibility]" } + remove_field => [ "[tmp][assignee_email]" ] + remove_field => [ "[tmp][ignore_reason]" ] + remove_field => [ "[tmp][ignored_at]" ] + remove_field => [ "[tmp][occurrences_count]" ] + remove_field => [ "[tmp][secret_revoked]" ] + remove_field => [ "[tmp][share_url]" ] + } + date { + match => ["[event][created]", "ISO8601"] + timezone => "UTC" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } +} + +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/vuln_sysdig_scanning.conf b/config/processors/vuln_sysdig_scanning.conf new file mode 100644 index 00000000..b53b9772 --- /dev/null +++ b/config/processors/vuln_sysdig_scanning.conf @@ -0,0 +1,45 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + json { + source => "message" + target => "tmp" + } + mutate { + add_field => { + "[event][module]" => "sysdig" + "[event][dataset]" => "sysdig.scan.api" + "[log][source][hostname]" => "vuln_sysdig_scanning" + } + rename => { "[tmp][application_name]" => "[network][application]" } + rename => { "[tmp][team]" => "[team][name]" } + rename => { "[tmp][environment]" => "[tags]" } + rename => { "[tmp][cloud_platform_external]" => "[network][dmz]" } + rename => { "[tmp][image]" => "[file][name]" } + rename => { "[tmp][github_repository]" => "[repository][name]" } + rename => { "[tmp][build_author]" => "[user][email]" } + rename => { "[tmp][build_date]" => "[event][created]" } + rename => { "[tmp][team_distribution_list]" => "[team][email]" } + rename => { "[tmp][team_escalation_contact]" => "[escalation][contact]" } + rename => { "[tmp][vuln_id]" => "[vulnerability][id]" } + rename => { "[tmp][vuln_severity]" => "[vulnerability][severity]" } + rename => { "[tmp][vuln_exception]" => "[vulnerability][exception]" } + rename => { "[tmp][dns_records]" => "[url][full]" } + remove_field => [ "[tmp][build_link]" ] + } + date { + match => ["[event][created]", "yyyy-MM-dd HH:mm:ss Z 'UTC'"] + timezone => "UTC" + locale => "en" + target => "[event][created]" + tag_on_failure => "_dateparsefailure_ec" + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/vuln_tenable_asm.conf b/config/processors/vuln_tenable_asm.conf new file mode 100644 index 00000000..7e7dc65f --- /dev/null +++ b/config/processors/vuln_tenable_asm.conf @@ -0,0 +1,177 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + mutate { + lowercase => [ "message" ] + } + json { + source => "message" + target => "tmp" + } + mutate{ + add_field => { "[log][source][hostname]" => "%{[tmp][bd.hostname]}" } + } + mutate { + add_field => { + "[event][module]" => "tenable" + "[event][dataset]" => "tenable.asm_api" + } + rename => { + "[tmp][id]" => "[event][id]" + "[tmp][bd.addedtoportfolio]" => "[cloud][account][id]" + "[tmp][bd.hostname]" => "[host][name]" + "[tmp][bd.ip_address]" => "[host][ip]" + "[tmp][bd.original_hostname]" => "[host][hostname]" + "[tmp][bd.record_type]" => "[dns][answers][class]" + "[tmp][bd.domain]" => "[host][domain]" + "[tmp][domaininfo.administrativecontact_email]" => "[destination][user][email]" + "[tmp][domaininfo.administrativecontact_name]" => "[destination][user][name]" + "[tmp][domaininfo.administrativecontact_organization]" => "[destination][as][organization][name]" + "[tmp][domaininfo.administrativecontact_telephone]" => "[destination][as][number]" + "[tmp][domaininfo.registrant_country]" => "[destination][geo][country_name]" + "[tmp][domaininfo.registrant_name]" => "[destination][user][business][unit]" + "[tmp][domaininfo.registrarname]" => "[observer][name]" + "[tmp][domaininfo.status]" => "[event][action]" + "[tmp][domaininfo.fdns_nameservers]" => "[group][domain]" + "[tmp][app_updates][rbls]" => "[threat][list][created]" + "[tmp][ipgeo.cloud]" => "[cloud][provider]" + "[tmp][ipgeo.asn_number]" => "[service][id]" + "[tmp][ipgeo.asn]" => "[service][name]" + "[tmp][ipgeo.time_zone]" => "[cloud][region]" + "[tmp][ipgeo.registered_country]" => "[destination][geo][country_name]" + } + } + if [tmp][domaininfo.administrativecontact_email] and [tmp][domaininfo.administrativecontact_email]!= "nil" { + mutate { + rename => { "[tmp][domaininfo.contactemail]" => "[destination][user][email]" } + } + } + if [tmp][domaininfo.administrativecontact_organization] and [tmp][domaininfo.administrativecontact_organization]!= "nil" { + mutate { + rename => { "[tmp][domaininfo.registrant_organization]" => "[destination][as][organization][name]" } + } + } + if [tmp][domaininfo.administrativecontact_name] and [tmp][domaininfo.administrativecontact_name]!= "nil" { + mutate { + rename => { "[tmp][domaininfo.registrant_name]" => "[destination][user][name]" } + } + } + ruby { + code => " + ports = [] + ports_services = [] + ports_banners = [] + ports_cves = [] + ports_cvss3_scores = [] + ports_cvss3_vectors = [] + rbls = [] + if event.get('[tmp][ports.ports]').kind_of? Array + event.get('[tmp][ports.ports]').each do |item| + if item!= '' + ports.push(item) + end + end + end + if event.get('[tmp][ports.services]').kind_of? Array + event.get('[tmp][ports.services]').each do |item| + if item!= '' + ports_services.push(item) + end + end + end + if event.get('[tmp][ports.banners]').kind_of? Array + event.get('[tmp][ports.banners]').each do |item| + if item!= '' + ports_banners.push(item) + end + end + end + if event.get('[tmp][ports.cves]').kind_of? Array + event.get('[tmp][ports.cves]').each do |item| + if item!= '' + ports_cves.push(item) + end + end + end + if event.get('[tmp][ports.cvss3_scores]').kind_of? Array + event.get('[tmp][ports.cvss3_scores]').each do |item| + if item!= '' + ports_cvss3_scores.push(item) + end + end + end + if event.get('[tmp][ports.cvss3_vectors]').kind_of? Array + event.get('[tmp][ports.cvss3_vectors]').each do |item| + if item!= '' + ports_cvss3_vectors.push(item) + end + end + end + if event.get('[tmp][rbls.rbls]').kind_of? Array + event.get('[tmp][rbls.rbls]').each do |item| + if item!= '' + rbls.push(item) + end + end + end + if !ports_cvss3_vectors.empty? + event.set('[vulnerability][classification]', ports_cvss3_vectors) + end + if !ports_cvss3_scores.empty? + event.set('[vulnerability][score][base]', ports_cvss3_scores) + end + if !ports_cves.empty? + event.set('[vulnerability][id]', ports_cves) + end + if !ports_services.empty? + event.set('[rule][name]', ports_services) + end + if !ports_banners.empty? + event.set('[rule][description]', ports_banners) + end + if !ports.empty? + event.set('[destination][port]', ports) + end + if !rbls.empty? + event.set('[vulnerability][reference]', rbls) + + end + " + tag_on_exception => "ruby_block_exception" + } + if [threat][list][created] { + date { + match => [ "[threat][list][created]", "ISO8601" ] + timezone => "GMT" + locale => "en" + target => "[threat][list][created]" + tag_on_failure => "_dateparsefailure_tcnf" + } + } + if [tmp][bd.subdomain]{ + mutate { + add_tag => [ "subdomain" ] + } + } + if [tmp][ipgeo.insideeu]{ + mutate { + add_tag => [ "inside_EU" ] + } + } + if [tmp][ipgeo.cloudhosted]{ + mutate { + add_tag => [ "cloud_hosted" ] + } + } + mutate { + remove_field => [ "tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file diff --git a/config/processors/vuln_tenable_sc.conf b/config/processors/vuln_tenable_sc.conf new file mode 100644 index 00000000..d92d8c3f --- /dev/null +++ b/config/processors/vuln_tenable_sc.conf @@ -0,0 +1,123 @@ +# Copyright [2023] [Cargill, Incorporated.] +# SPDX-License-Identifier: Apache-2.0 +input { + pipeline { + address => VAR_PIPELINE_NAME + } +} +filter { + json { + source => "message" + target => "tmp" + } + mutate{ + add_field => { "[log][source][hostname]" => "%{[tmp][netbiosName]}" } + } + mutate { + add_field => { + "[event][module]" => "tenable" + "[event][dataset]" => "tenable.sc_api" + } + rename => { + "[tmp][severity][id]" => "[event][severity]" + "[tmp][severity][name]" => "[event][severity_name]" + "[tmp][ip]" => "[host][ip]" + "[tmp][uuid]" => "[event][id]" + "[tmp][port]" => "[destination][port]" + "[tmp][protocol]" => "[network][transport]" + "[tmp][firstSeen]" => "[event][start]" + "[tmp][lastSeen]" => "[event][end]" + "[tmp][cve]" => "[vulnerability][id]" + "[tmp][dnsName]" => "[host][name]" + "[tmp][macAddress]" => "[host][mac]" + "[tmp][netbiosName]" => "[host][hostname]" + "[tmp][operatingSystem]" => "[host][os][full]" + "[tmp][repository][dataFormat]" => "[network][type]" + "[tmp][seeAlso]" => "[vulnerability][reference]" + "[tmp][pluginInfo]" => "[vulnerability][description]" + "[tmp][pluginName]" => "[service][name]" + "[tmp][version]" => "[service][version]" + "[tmp][synopsis]" => "[rule][description]" + "[tmp][cvssV3BaseScore]" => "[vulnerability][score][base]" + "[tmp][cvssV3TemporalScore]" => "[vulnerability][score][temporal]" + "[tmp][vprScore]" => "[vulnerability][score][environmental]" + "[tmp][repository][name]" => "[repository][name]" + "[tmp][xref]" => "[rule][reference]" + "[tmp][baseScore]" => "[event][risk_score]" + "[tmp][cvssVector]" => "[vulnerability][classification]" + "[tmp][temporalScore]" => "[event][risk_score_norm]" + "[tmp][pluginID]" => "[service][id]" + "[tmp][solution]" => "[event][recommendation]" + "[tmp][pluginPubDate]" => "[file][created]" + "[tmp][pluginModDate]" => "[file][mtime]" + "[tmp][exploitFrameworks]" => "[threat][framework]" + "[tmp][exploitEase]" => "[threat][list][attribute_tag]" + } + } + json{ + source=> "[tmp][vprContext]" + target=> "[tmp][vprContext]" + } + ruby { + code => " + if event.get('[tmp][vprContext]').kind_of? Array + event.get('[tmp][vprContext]').each do |item| + if item['id'] == 'threat_intensity_last_28' + event.set('[threat][list][severity_name]', item['value']) + end + if item['id'] == 'threat_sources_last_28' + event.set('[threat][list][event_info]' ,item['value']) + end + if item['id'] == 'exploit_code_maturity' + event.set('[threat][list][comment]' ,item['value']) + end + end + end + " + tag_on_exception => "ruby_block_exception" + } + date { + match => [ "[event][start]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[event][start]" + tag_on_failure => "_dateparsefailure_es" + } + date { + match => [ "[event][end]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[event][end]" + tag_on_failure => "_dateparsefailure_ee" + } + date { + match => [ "[file][created]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][created]" + tag_on_failure => "_dateparsefailure_ee" + } + date { + match => [ "[file][mtime]", "UNIX" ] + timezone => "GMT" + locale => "en" + target => "[file][mtime]" + tag_on_failure => "_dateparsefailure_ee" + } + if [tmp][hasBeenMitigated] == "1" { + mutate{ + add_tag => [ "mitigated" ] + } + } + if [tmp][exploitAvailable] == "yes" { + mutate{ + add_tag => [ "exploit_available" ] + } + } + mutate { + remove_field => [ "tmp" ] + } +} +output { + pipeline { send_to => [enrichments] } +} \ No newline at end of file