moving files from internal repo per Brian

config/processors/api_aws_app.conf

@@ -0,0 +1,127 @@
+# Copyright [2023] [Cargill, Incorporated.]
+# SPDX-License-Identifier: Apache-2.0
+input {
+  pipeline {
+    address => VAR_PIPELINE_NAME
+  }
+}
+filter {
+  mutate {
+    remove_field => ["event", "host"]
+  }
+  json {
+    source => "message"
+    target => "tmp"
+  }
+  mutate {
+    add_field => { 
+      "[event][module]" => "aws"
+      "[event][dataset]" => "aws.app" 
+      "[log][source][hostname]" => "aws_app" 
+    }
+  }
+	  # cmd 
+      # "[tmp][__monotonic_timestamp]" => "1289776813424"
+      # "[tmp][__realtime_timestamp]" => "1704485287095345"
+	  # "[tmp][_source_monotonic_timestamp]" => "1989560529012"
+      # "[tmp][_cap_effective]" => "1ffffffffff"
+  mutate {
+    rename => {
+      "[tmp][_source_realtime_timestamp]" => "[event][created]"
+      "[tmp][_exe]" => "[process][executable]"
+      "[tmp][_comm]" => "[process][name]" 
+      "[tmp][_boot_id]" => "[process][entity_id]"
+      "[tmp][unit]" => "[process][parent][command_line]"
+      "[tmp][code_file]" => "[process][parent][name]"
+      "[tmp][service]" => "[service][name]"
+      "[tmp][_gid]" => "[process][pgid]"
+      "[tmp][_pid]" => "[process][pid]"	
+      "[tmp][_cmdline]" => "[process][command_line]"
+      "[tmp][_uid]" => "[file][uid]"
+      "[tmp][_systemd_cgroup]" => "[group][name]"
+      "[tmp][_transport]" => "[network][transport]"
+      "[tmp][_machine_id]" => "[host][id]"
+      "[tmp][code_function]" => "[log][origin][function]"
+      "[tmp][code_line]" => "[log][origin][file][line]"
+      "[tmp][host]" => "[host][hostname]"
+      "[tmp][syslog_identifier]" => "[log][syslog][facility][name]"
+      "[tmp][priority]" => "[log][syslog][priority]"
+      "[tmp][syslog_facility]" => "[log][syslog][facility][code]"
+    }
+  }
+   # k8s
+  mutate {
+   rename => {
+      "[tmp][cluster_name]" => "[cloud][instance][name]"
+      "[tmp][container_id]" => "[container][id]"
+      "[tmp][ddsource]" => "[container][runtime]"
+      "[tmp][ddtags]" => "[container][image][tag]"
+      "[tmp][hostname]" => "[host][hostname]"
+      "[tmp][kubernetes][container_image]" => "[container][image][name]"
+      "[tmp][kubernetes][container_name]" => "[container][name]"
+      "[tmp][kubernetes][pod_ip]" => "[cloud][instance][id]"
+      "[tmp][kubernetes][namespace_name]" => "[cloud][project][id]"
+      "[tmp][kubernetes][pod_name]" => "[cloud][project][name]"
+      "[tmp][kubernetes][pod_owner]" => "[cloud][account][name]"
+      "[tmp][service]" => "[cloud][service][name]"
+      "[tmp][source]" => "[cloud][machine][type]"
+      "[tmp][source_type]" => "[event][kind]"
+      "[tmp][stream]" => "[event][provider]"
+      "[tmp][timestamp]" => "[event][ingested]"
+    }
+  }
+  # k8s or json message
+  if [tmp][message] =~ "^{.*?}$" {
+    json {
+      source => "[tmp][message]"
+      target => "jtmp"
+    }
+    mutate {
+      rename => {
+        "[jtmp][class]" => "com.cargill.fps.server.service.impl.pricingserviceimpl"
+        "[jtmp][method]" => "[http][request][mime_type]"
+        "[jtmp][@severity]" => "[log][syslog][severity][name]"
+        "[jtmp][level]" => "[log][level]"
+        "[jtmp][level_value]" => "[log][syslog][severity][code]"
+        "[jtmp][logger_name]" => "[log][logger]"
+        "[jtmp][thread_name]" => "[process][thread][name]"
+        "[jtmp][@timestamp]" => "[event][created]"
+        "[jtmp][file]" => "[file][name]"
+        "[jtmp][line]" => "[log][origin][file][line]"
+        "[jtmp][@message]" => "[error][message]"
+        "[jtmp][dd.trace_id]" => "[trace][id]"
+        "[jtmp][dd.service]" => "[service][name]"
+        "[jtmp][dd.span_id]" => "[span][id]"
+      }
+      add_tag => [ "%{[jtmp][dd.env]}" ]
+    }
+  } else {
+    mutate {
+      rename => {
+    	"[tmp][message]" => "[error][message]"
+      }
+    }
+  }
+  # [event][created] "1704485287095345"
+  # "[event][created]" "2024-01-04t17:45:30.921z"
+  date {
+    match => ["[event][created]", "yyyy-MM-dd'T'HH:mm:sss'Z'","yyyy-MM-dd't'HH:mm:sss'z'", "ISO8601", "UNIX", "UNIX_MS" ]
+    timezone => "GMT"
+    locale => "en"
+    target => "[event][created]"
+  }
+  # "[event][ingested]" = "2024-01-04t17:45:30.921235225z"
+  date {
+    match => ["[event][ingested]", "yyyy-MM-dd'T'HH:mm:sssssssss'Z'","yyyy-MM-dd't'HH:mm:sssssssss'z'", "ISO8601" ]
+    timezone => "GMT"
+    locale => "en"
+    target => "[event][ingested]"
+  }
+
+  mutate {
+    remove_field => ["tmp", "jtmp" ]
+  } 
+}
+output {
+  pipeline { send_to => [enrichments] }
+}

config/processors/api_https_audit_sap_tcodes.conf

@@ -0,0 +1,61 @@
+# Copyright [2021] [Cargill, Incorporated.]
+# SPDX-License-Identifier: Apache-2.0
+input {
+  pipeline {
+    address => VAR_PIPELINE_NAME
+  }
+}
+filter {
+  mutate {
+    remove_field => ["event", "host", "log" ]
+  }
+  json {
+    source => "message"
+    target => "tmp"
+  }  
+  mutate {
+    add_field => { 
+      "[event][module]" => "sap" 
+      "[event][dataset]" => "sap.t-codes" 
+    }
+  }
+  mutate {
+    add_tag => [ "%{[tmp][tags]}" ]
+    rename => {
+      "[tmp][timestamp]" => "[event][created]"
+      "[tmp][reason_id]" => "[event][reason]"
+      "[tmp][agent][type]" => "[agent][type]"
+      "[tmp][agent][name]" => "[agent][name]"
+      "[tmp][client_pc]" => "[source][ip]"
+      "[tmp][username]" => "[user][name]"
+      "[tmp][log_client]" => "[group][id]"
+      "[tmp][edm_data][log_sapgui][value_header][0][pprogram]" => "[process][name]"
+      "[tmp][edm_data][log_sapgui][value_header][0][gui_title]" => "[rule][name]"
+      "[tmp][edm_data][log_sapgui][value_header][0][tcode]" => "[rule][id]"	  
+      "[tmp][trx_name]" => "[rule][description]"
+      "[tmp][tid]" => "[rule][category]"
+      "[tmp][sysid]" =>  "[host][id]"
+      "[tmp][technology]" => "[host][os][type]"
+      "[tmp][host_name]" => "[host][hostname]"
+      "[tmp][log][source][ip]" => "[log][source][ip]"	
+      "[tmp][log][source][hostname]" => "[log][source][hostname]"	  
+    }
+  }
+  mutate {
+    strip => ["[event][created]"]
+  }
+
+  # "[event][created]" = "20240108175253.178 "
+  date {
+    match => ["[event][created]", "yyyyMMddHHmmss.SSS" ]
+    timezone => "GMT"
+    locale => "en"
+    target => "[event][created]"
+  }
+  mutate {
+    remove_field => ["tmp"]
+  } 
+}
+output {
+  pipeline { send_to => [enrichments] }
+}

config/processors/api_ois_sap_security_bridge.conf

@@ -0,0 +1,104 @@
+# Copyright [2023] [Cargill, Incorporated.] 
+# SPDX-License-Identifier: Apache-2.0
+input {
+  pipeline {
+    address => VAR_PIPELINE_NAME
+  }
+}
+filter {
+  mutate {
+    add_field => {
+      "[event][module]" => "sap"
+      "[event][dataset]" => "sap.security_bridge"
+      "[log][source][hostname]" => "api_ois_sap_security_bridge"
+    }
+  }
+  json{
+    source=>"message"
+    target=>"tmp"
+  }
+  mutate{
+    rename=>{
+      "[tmp][IPv4Address]" => "[source][ip]"
+      "[tmp][severity]" => "[event][severity]"
+      "[tmp][eventUserType]" => "[user][roles]"
+      "[tmp][eventUserGroup]" => "[user][group][name]"
+      "[tmp][systemType]" => "[observer][type]"
+      "[tmp][eventMsg]" => "[event][reason]"
+      "[tmp][program]" => "[package][type]"
+      "[tmp][account]" => "[user][name]"
+      "[tmp][action]" => "[event][action]"
+      "[tmp][guid]" => "[event][id]"
+      "[tmp][transactionCode]" => "[transaction][id]"
+      "[tmp][terminal]" => "[group][name]"
+      "[tmp][object]" => "[event][code]"
+      "[tmp][sid]" => "[group][id]"
+      "[tmp][client]" => "[service][id]"
+      "[tmp][qSid]" => "[cloud][instance][name]"
+    }
+  }
+  if [tmp][eventAttr1] {
+    mutate{
+      add_field => {
+        "[event][type]" => "%{[tmp][eventAttr1]}"
+      }
+    }
+  }
+  if [tmp][eventAttr2] {
+    mutate{
+      add_field => {
+        "[event][type]" => "%{[tmp][eventAttr2]}"
+      }
+    }
+  }
+  if [tmp][eventAttr3] {
+    mutate{
+      add_field => {
+        "[event][type]" => "%{[tmp][eventAttr3]}"
+      }
+    }
+  }
+  if [tmp][eventAttr4] {
+    mutate{
+      add_field => {
+        "[event][type]" => "%{[tmp][eventAttr4]}"
+      }
+    }
+  }
+  if [tmp][eventAttr5] {
+    mutate{
+      add_field => {
+        "[event][type]" => "%{[tmp][eventAttr5]}"
+      }
+    }
+  }
+  grok {
+      match => { "[tmp][timestamp]" => "^(\/Date\()(?<[event][created]>.*?)(\)\/)$" }
+      match => { "[tmp][recTimestamp]" => "^(\/Date\()(?<[event][modified]>.*?)(\)\/)$" }
+  }
+  date {
+    match => [ "[event][created]", "UNIX_MS" ]
+    timezone => "GMT"
+    locale => "en"
+    target => "[event][created]"
+    tag_on_failure => "_dateparsefailure_es"
+  }
+  date {
+    match => [ "[event][modified]", "UNIX_MS" ]
+    timezone => "GMT"
+    locale => "en"
+    target => "[event][modified]"
+    tag_on_failure => "_dateparsefailure_es"
+  }
+  if [tmp][retroactive] {       
+    mutate {
+      add_tag => [ "retroactive" ]
+    }
+  }
+  mutate {
+    remove_field => ["tmp" ]
+  }
+}
+output {
+  pipeline { send_to => [enrichments] }
+}

config/processors/api_security_sap_btp.conf

@@ -0,0 +1,56 @@
+# Copyright [2021] [Cargill, Incorporated.]
+# SPDX-License-Identifier: Apache-2.0
+input {
+  pipeline {
+    address => VAR_PIPELINE_NAME
+  }
+}
+filter {
+  mutate {
+    remove_field => ["event", "host", "log" ]
+  }
+  mutate {
+    add_field => { 
+      "[event][module]" => "sap"
+      "[event][dataset]" => "sap.btp"
+    }
+  }
+  json {
+    source => "message"
+    target => "tmp"
+  }
+
+  mutate{
+    add_field =>{ "[log][source][hostname]" => "%{[tmp][tenant]}" }
+    lowercase => [ "tmp" ]
+  }
+  mutate {
+    rename => { 
+      "[tmp][als_service_id]" => "[service][id]"
+      "[tmp][user]" => "[user][name]"
+      "[tmp][category]" => "[event][category]"
+      "[tmp][message_uuid]" => "[rule][uuid]"
+      "[tmp][org_id]" => "[cloud][project][id]"
+      "[tmp][time]" => "[event][created]"
+      "[tmp][space_id]" => "[cloud][instance][id]"
+      "[tmp][tenant]" => "[cloud][account][id]"
+      "[tmp][object][type]" => "[event][type]"
+      "[tmp][id]" => "[event][id]"
+      "[tmp][data]" => "[error][message]"
+      "[tmp][ip]" => "[source][ip]"
+    }    
+  }
+  date {
+    match => [ "[event][created]", "ISO8601" ]
+    timezone => "GMT"
+    locale => "ec"
+    target => "[event][created]"
+    tag_on_failure => "_dateparsefailure_ec"
+  }
+  mutate {
+    remove_field => [ "tmp" ]
+  }
+}
+output {
+  pipeline { send_to => [enrichments] }
+}