Merge pull request #487 from Cargill/azure_fw_proxy_dns_ids_flow

azure fw dns ids proxy flow

config/processors/event_hub_azure.fw_dns_ids_proxy_flow.conf

@@ -0,0 +1,104 @@
+# Copyright [2023] [Cargill, Incorporated.]
+# SPDX-License-Identifier: Apache-2.0
+input {
+  pipeline {
+    address => VAR_PIPELINE_NAME
+  }
+}
+filter {
+  mutate {
+	remove_field => ["event", "host", "log","user" ]
+  }
+  mutate {
+    add_field => { 
+      "[event][module]" => "azure"
+    }
+  }
+  json {
+    source => "message"
+    target => "tmp"
+  } 
+  mutate {
+    rename => { 
+      "[tmp][resourceId]" => "[log][source][hostname]"
+      "[tmp][time]" => "[event][created]"
+      "[tmp][category]" => "[event][kind]"
+      "[tmp][properties][msg]" => "[error][message]"
+    }
+    # category":"azfwdnsquery"
+    rename => { 
+      "[tmp][properties][QueryClass]" => "[dns][question][class]"
+      "[tmp][properties][ErrorMessage]" => "[error][message]"
+      "[tmp][properties][Protocol]" => "[network][transport]"
+      "[tmp][properties][QueryId]" => "[dns][id]"
+      "[tmp][properties][QueryName]" => "[dns][question][name]"
+      "[tmp][properties][QueryType]" => "[dns][question][type]"
+      "[tmp][properties][ResponseCode]" => "[dns][response_code]"
+      "[tmp][properties][ResponseFlags]" => "[dns][header_flags]"
+      "[tmp][properties][SourceIp]" => "[source][ip]"
+      "[tmp][properties][SourcePort]" => "[source][port]"
+    }
+    # AZFWIdpsSignature"
+    rename => { 
+      "[tmp][properties][Description]" => "[rule][description]"
+      "[tmp][properties][Action]" => "[event][action]"
+      "[tmp][properties][DestinationIp]" => "[destination][ip]"
+      "[tmp][properties][DestinationPort]" => "[destination][port]"
+      "[tmp][properties][Category]" => "[rule][category]"
+      "[tmp][properties][Severity]" => "[event][severity]"
+      "[tmp][properties][SignatureId]" => "[rule][id]"
+    }
+  # AZFWApplicationRule
+    rename => { 
+      "[tmp][properties][RuleCollectionGroup]" => "[rule][description]"
+      "[tmp][properties][Policy]" => "[rule][category]"
+      "[tmp][properties][IsTlsInspected]" => "[tls][established]"
+      "[tmp][properties][TargetUrl]" => "[url][registered_domain]"
+      "[tmp][properties][RuleCollection]" => "[rule][ruleset]"
+      "[tmp][properties][Rule]" => "[rule][name]"
+      "[tmp][properties][IsExplicitProxyRequest]" => false
+      "[tmp][properties][WebCategory]" => "[event][category]"
+      "[tmp][properties][ActionReason]" => "[event][reason]"
+      "[tmp][properties][Fqdn]" => "[url][full]"
+    }
+  }
+  if [event][kind] == "AzureFirewallDnsProxy" {
+    grok {
+      match => { "[error][message]" => "DNS Request: (?<[source][ip]>\d+\.\d+\.\d+\.\d+):(?<[source][port]>\d+) .*? (?<[dns][id]>\d+) (?<[dns][question][type]>\w+) (?<[dns][question][class]>\w+) (?<[dns][question][name]>.*?)\. (?<[network][transport]>\w+) \d+ \w+ \d+ (?<[dns][response_code]>\w+) (?<[dns][header_flags]>(\w+,)+\w+) \d+" }
+    }
+  }  
+  if [event][kind] == "AzureFirewallNetworkRule" or [event][kind] == "AzureFirewallApplicationRule" {
+    grok {
+      match => { "[error][message]" => "^(?<[network][protocol]>.*?)\w+ .*?from (?<[source][ip]>\d+.\d+.\d+.\d+):(?<[source][port]>\d+) to ((?<[destination][ip]>\d+.\d+.\d+.\d+)|(?<[destination][address]>.*?)):(?<[destination][port]>\d+).*?Action: (?<[event][action]>\w+).*?\.+ (Policy: ((?<[rule][category]>.*?) Rule Collection Group: (?<[rule][ruleset]>.*?) Rule Collection: (?<[observer][name]>.*?) Rule: (?<[rule][name]>.*?))|((?<[rule][name]>.*?)))$" }
+    }
+  } 
+  translate {
+    source => "[event][kind]"
+    target => "[event][dataset]"
+      dictionary => {
+        "AzureFirewallDnsProxy" => "azure.dns"
+        "AZFWDnsQuery" => "azure.dns"
+        "AzureFirewallNetworkRule" => "azure.fw"
+        "AzureFirewallApplicationRule" => "azure.fw"
+        "AZFWNetworkRule" => "azure.fw"
+        "AZFWIdpsSignature" => "azure.ids"
+        "AZFWApplicationRule" => "azure.proxy"
+        "AZFWFlowTrace" => "azure.flow"
+      }
+      fallback => "azure.fw_metrics"
+  }
+
+  # "[event][created]" "2024-02-29t16:41:47.9085510z"
+    date {
+    match => ["[event][created]" , "yyyy-MM-dd'T'HH:mm:ss.SSSSSSS'Z'", "yyyy-MM-dd't'HH:mm:ss.SSSSSSS'z'" ]
+    target => "[event][created]"
+    tag_on_failure => "_dateparsefailure_ec"
+  }
+  mutate {
+    remove_field => [ "tmp" ]
+    # copy => { "message" => "[event][original]" }
+  }
+}
+output {
+  pipeline { send_to => [enrichments] }
+}