From 0c2249a3472d3a7df49c298210febb345b500370 Mon Sep 17 00:00:00 2001
From: Brian Grabau <brian_grabau@cargill.com>
Date: Wed, 23 Oct 2024 17:26:49 -0500
Subject: [PATCH] Added ecs checks after enrichments to remove bad fields from
 enrichments

---
 config/enrichments/96_ecs_data_type.conf      | 321 ++++++++++++++++++
 .../{96_output.conf => 97_output.conf}        |   0
 2 files changed, 321 insertions(+)
 create mode 100644 config/enrichments/96_ecs_data_type.conf
 rename config/enrichments/{96_output.conf => 97_output.conf} (100%)

diff --git a/config/enrichments/96_ecs_data_type.conf b/config/enrichments/96_ecs_data_type.conf
new file mode 100644
index 00000000..54093697
--- /dev/null
+++ b/config/enrichments/96_ecs_data_type.conf
@@ -0,0 +1,321 @@
+# Copyright [2021] [Cargill, Incorporated.] 
+# SPDX-License-Identifier: Apache-2.0
+# remove empty fields and truncate all fields to 1024 bytes
+filter {
+    mutate {
+      rename => { "log.source.ip" => "[log][source][ip]" }
+      rename => { "log.source.hostname" => "[log][source][hostname]" }
+    }
+  if "disable_ecs_data_type_enrichment" in [tags] {
+    mutate {
+      remove_tag => ["disable_ecs_data_type_enrichment"]
+    }
+  } else {
+    # Remove custom shipper topic name
+    mutate {
+      remove_field => ["log_source_type"]
+    }
+    # remove fields which are not convertible to ecs datatypes
+    ruby {
+      code => '
+        long_fields = [
+          "[client][bytes]",
+          "[client][packets]",
+          "[client][port]",
+          "[destination][bytes]",
+          "[destination][packets]",
+          "[destination][port]",
+          "[event][duration]",
+          "[event][severity]",
+          "[file][x509][public_key_exponent]",
+          "[network][bytes]",
+          "[network][packets]",
+          "[process][parent][pid]",
+          "[process][pid]",
+          "[server][bytes]",
+          "[server][packets]",
+          "[server][port]",
+          "[source][bytes]",
+          "[source][packets]",
+          "[source][port]",
+          "[client][as][number]",
+          "[client][nat][port]",
+          "[destination][as][number]",
+          "[destination][nat][port]",
+          "[dns][answers][ttl]",
+          "[event][sequence]",
+          "[file][size]",
+          "[file][x509][public_key_size]",
+          "[host][disk][read][bytes]",
+          "[host][disk][write][bytes]",
+          "[host][network][egress][bytes]",
+          "[host][network][egress][packets]",
+          "[host][network][ingress][bytes]",
+          "[host][network][ingress][packets]",
+          "[host][uptime]",
+          "[http][request][body][bytes]",
+          "[http][request][bytes]",
+          "[http][response][body][bytes]",
+          "[http][response][bytes]",
+          "[http][response][status_code]",
+          "[log][origin][file][line]",
+          "[log][syslog][facility][code]",
+          "[log][syslog][priority]",
+          "[log][syslog][severity][code]",
+          "[observer][ingress][interface][host_count]",
+          "[package][size]",
+          "[process][args_count]",
+          "[process][exit_code]",
+          "[process][parent][args_count]",
+          "[process][parent][exit_code]",
+          "[process][parent][pgid]",
+          "[process][parent][ppid]",
+          "[process][parent][thread][id]",
+          "[process][parent][uptime]",
+          "[process][pgid]",
+          "[process][ppid]",
+          "[process][thread][id]",
+          "[process][uptime]",
+          "[server][as][number]",
+          "[server][nat][port]",
+          "[source][as][number]",
+          "[source][nat][port]",
+          "[tls][client][x509][public_key_exponent]",
+          "[tls][client][x509][public_key_size]",
+          "[tls][server][x509][public_key_exponent]",
+          "[tls][server][x509][public_key_size]",
+          "[url][port]"
+        ]
+        float_fields = [
+          "[event][risk_score]",
+          "[event][risk_score_norm]",
+          "[host][cpu][usage]",
+          "[vulnerability][score][base]",
+          "[vulnerability][score][environmental]",
+          "[vulnerability][score][temporal]"
+          ]
+
+        float_fields.each do |k|
+          field_value = event.get(k)
+          if field_value != nil
+            begin
+              converted_v = Float(field_value)
+              event.set(k, converted_v)
+            rescue
+              event.remove(k)
+            end
+          end
+        end
+
+        long_fields.each do |k|
+          field_value = event.get(k)
+          if field_value != nil
+            begin
+              converted_v = Integer(field_value)
+              event.set(k, converted_v)
+            rescue
+              event.remove(k)
+            end
+          end
+        end
+      '
+      tag_on_exception => "enrichment_data_type_ruby_exception"
+    }
+    ### Validate IP data types
+    mutate {
+      strip => ["[client][ip]", "[client][nat][ip]", "[destination][ip]", "[destination][nat][ip]", "[dns][resolved_ip]", "[host][ip]", "[network][forwarded_ip]", "[observer][ip]", "[server][ip]", "[server][nat][ip]", "[source][ip]", "[source][nat][ip]"]
+    }
+
+    # [log][source][ip]
+    if [log][source][ip] {
+      if [log][source][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![log][source][ip][0] {
+        if ![log][source][hostname] {
+          mutate {
+            copy => { "[log][source][ip]" => "[log][source][hostname]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [log][source][ip]" ]
+          remove_field => ["[log][source][ip]"]
+        }
+      }
+    }
+    # [source][ip]	[source][address]
+    if [source][ip] {
+      if [source][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![source][ip][0] {
+        if ![source][address] {
+          mutate {
+            copy => { "[source][ip]" => "[source][address]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [source][ip]" ]
+          remove_field => ["[source][ip]"]
+        }
+      }
+    }
+    # [client][ip]	[client][address]
+    if [client][ip] {
+      if [client][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![client][ip][0] {
+        if ![client][address] {
+          mutate {
+            copy => { "[client][ip]" => "[client][address]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [client][ip]" ]
+          remove_field => ["[client][ip]"]
+        }
+      }
+    }
+    # [client][nat][ip]	drop field
+    if [client][nat][ip] {
+      if [client][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![client][nat][ip][0] {
+        mutate {
+          remove_field => ["[client][nat][ip]"]
+          add_tag => [ "invalid ip format [client][nat][ip]" ]
+        }
+      }
+    }
+    # [destination][ip]	[destination][address]
+    if [destination][ip] {
+      if [destination][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![destination][ip][0] {
+        if ![destination][address] {
+          mutate {
+            copy => { "[destination][ip]" => "[destination][address]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [destination][ip]" ]
+          remove_field => ["[destination][ip]"]
+        }
+      }
+    }
+    # [destination][nat][ip] drop field
+    if [destination][nat][ip] {
+      if [destination][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![destination][nat][ip][0] {
+        mutate {
+          remove_field => ["[destination][nat][ip]"]
+          add_tag => [ "invalid ip format [destination][nat][ip]" ]
+        }
+      }
+    }
+    # [dns][resolved_ip]	[dns][answers][name]
+    if [dns][resolved_ip] {
+      if [dns][resolved_ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![dns][resolved_ip][0]{
+        if ![dns][answers][name] {
+          mutate {
+            copy => { "[dns][resolved_ip]" => "[dns][answers][name]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [dns][resolved_ip]" ]
+          remove_field => ["[dns][resolved_ip]"]
+        }
+      }
+    }
+    # [host][ip]	[host][hostname]
+    if [host][ip] {
+      if [host][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![host][ip][0] {
+        if ![host][hostname] {
+          mutate {
+            copy => { "[host][ip]" => "[host][hostname]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [host][ip]" ]
+          remove_field => ["[host][ip]"]
+        }
+      }
+    }
+    # network.forwarded_ip	drop field
+    if [network][forwarded_ip] {
+      if [network][forwarded_ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![network][forwarded_ip][0] {
+        mutate {
+          remove_field => ["[network][forwarded_ip]"]
+          add_tag => [ "invalid ip format [network][forwarded_ip]" ]
+        }
+      }
+    }
+    # [observer][ip]	[observer][name]
+    if [observer][ip] {
+      if [observer][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![observer][ip][0] {
+        if ![observer][name] {
+          mutate {
+            copy => { "[observer][ip]" => "[observer][name]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [observer][ip]" ]
+          remove_field => ["[observer][ip]"]
+        }
+      }
+    }
+    # related.ip	n/a if all fields are fixed before adding to related field then will not have issue
+
+    # [server][ip]	[server][address]
+    if [server][ip] {
+      if [server][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][ip][0] {
+        if ![server][name] {
+          mutate {
+            copy => { "[server][ip]" => "[server][name]" }
+          }
+        }
+        mutate {
+          add_tag => [ "invalid ip format [server][ip]" ]
+          remove_field => ["[server][ip]"]
+        }
+      }
+    }
+    # [server][nat][ip]	drop field
+    if [server][nat][ip] {
+      if [server][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][nat][ip][0] {
+        mutate {
+          remove_field => ["[server][nat][ip]"]
+          add_tag => [ "invalid ip format [server][nat][ip]" ]
+        }
+      }
+    }
+    # [source][nat][ip]	drop field
+    if [source][nat][ip] {
+      if [source][nat][ip] !~ "^\d+\.\d+\.\d+\.\d+$" and ![server][nat][ip][0] {
+        mutate {
+          remove_field => ["[source][nat][ip]"]
+          add_tag => [ "invalid ip format [source][nat][ip]" ]
+        }
+      }
+    }
+
+    ### Drop unparsed date fields and add tag
+    ruby {
+      code => '
+        # field names to check with their corresponding failure tags
+        date_fields_hash = {
+          "[event][created]" => "_dateparsefailure_ec",
+          "[event][end]" => "_dateparsefailure_ee",
+          "[event][ingested]" => "_dateparsefailure_ei",
+          "[event][start]" => "_dateparsefailure_es",
+          "[file][accessed]" => "_dateparsefailure_fa",
+          "[file][created]" => "_dateparsefailure_fc",
+          "[file][ctime]" => "_dateparsefailure_fct",
+          "[file][mtime]" => "_dateparsefailure_fm",
+          "[package][installed]" => "_dateparsefailure_pi",
+          "[process][parent][start]" => "_dateparsefailure_pps",
+          "[process][start]" => "_dateparsefailure_ps",
+          "[tls][client][not_after]" => "_dateparsefailure_tcna",
+          "[tls][client][not_before]" => "_dateparsefailure_tcnb",
+          "[tls][server][not_after]" => "_dateparsefailure_tsna",
+          "[tls][server][not_before]" => "_dateparsefailure_tsnb"
+        }
+        # for each field in the above has, check if the value in the event is a string
+        # if yes, remove that field and add the corresponding tag,
+        date_fields_hash.each do |k,v|
+          if event.get(k).is_a? String
+            event.remove(k)
+            event.tag(v)
+          end
+        end
+      '
+    }
+  }
+}
diff --git a/config/enrichments/96_output.conf b/config/enrichments/97_output.conf
similarity index 100%
rename from config/enrichments/96_output.conf
rename to config/enrichments/97_output.conf