handled urls with ts= and added error log parsing

config/processors/syslog_security_skyhigh.swg.conf

@@ -15,37 +15,54 @@ filter {
     gsub => [
       "message", "connection type=(.*?), ssl", "connection type=\1,  ssl"
     ]
+    remove_field => [ "host" ] 
   }
   grok {
-    match => { "message" => "^(.*?)(<(?<pri>\d+)>)(\s)?.*?mwg:(  )?(?<tmp_csv>.*?)$" }
+    match => { "message" => "^(.*?)(<(?<pri>\d+)>)(\s)?.*?mwg:(  )?(\s)?(?<tmp_csv>.*?)$" }
     timeout_millis => 500
   }
   syslog_pri {
     syslog_pri_field_name => "pri" 
     remove_field => [ "pri" ]
     ecs_compatibility => v8
   }
-if [tmp_csv] !~ "\w,\w" {
+  if [tmp_csv] =~ ",  " {
+    grok {
+      match => { "tmp_csv" => "^(?<tmp_csv_2>.*?)  method=(?<tmp_url>.*?)(,  ref=(?<tmp_referrer>.*?))?(\s+)?$" }
+      timeout_millis => 500
+      tag_on_failure => "_grokparsefailure"
+    }
     kv {
-      source => "tmp_csv"
+      source => "tmp_csv_2"
       target => "tmp"
-      field_split_pattern => ",(  | $)"
+      field_split => ",  "
       value_split => "="
       recursive => "false"
       trim_key => " "
       trim_value => " "
     }
-  } else {
+  } else if [tmp_csv] =~ "virus=" {
+    grok {
+      match => { "tmp_csv" => "^(?<tmp_csv_2>.*)method=(?<tmp_url>.*)(,ref=(?<tmp_referrer>.*?))?(\s+)?$" }
+      timeout_millis => 500
+      tag_on_failure => "_grokparsefailure"
+    }
     kv {
-      source => "tmp_csv"
+      source => "tmp_csv_2"
       target => "tmp"
       field_split => ","
       value_split => "="
       recursive => "false"
       trim_key => " "
       trim_value => " "
     }
-}
+  } else {
+    grok {
+      match => { "tmp_csv" => '^.*?" .*? (?<[error][id]>\d+) "(?<[error][code]>.*?)" ("(")?(?<[error][message]>.*?)"(")?)?(.*?Severity: (?<[event][severity]>\d+)")?' }
+      timeout_millis => 500
+      tag_on_failure => "_grokparsefailure"
+    }
+  }
   mutate {
     rename => {
       "[tmp][usr]" => "[user][name]"
@@ -55,12 +72,10 @@ if [tmp_csv] !~ "\w,\w" {
       "[tmp][sprt]" => "[source][port]"
       "[tmp][rule]" => "[rule][uuid]"
       "[tmp][stat]" => "[http][response][status_code]"
-      "[tmp][agent]" => "[user_agent][original]"
       "[tmp][sbytes]" => "[http][response][bytes]"
       "[tmp][blockid]" => "[rule][id]"
       "[tmp][sip]" => "[source][ip]"
       "[tmp][sev]" => "[event][severity_name]"
-      "[tmp][ref]" => "[http][request][referrer]"
       "[tmp][dip]" => "[destination][ip]"
       "[tmp][cat]" => "[rule][category]"
       "[tmp][ts]" => "[event][created]"
@@ -79,22 +94,23 @@ if [tmp_csv] !~ "\w,\w" {
       "[tmp][virus]" => "[rule][name]"
       "[tmp][ver]" => "[tls][next_protocol]"
       "[tmp][url]" => "[observer][ip]"
+      "[tmp][agent]" => "[user_agent][original]"
     }
   }
-  if [tmp_csv] !~ "ref=.*?$" {
-    mutate {
-      # identify long uri i.e. possible DNS exfiltration
-      add_tag => "long uri"
-    }
-  }
-
   # URI
-  grok {
-    match => { "[tmp][method]" => "^(?<[http][request][method]>.*?) (?<[url][full]>.*?)( |$)((?<[tls][next_protocol]>.*?).*$)?" }
-    tag_on_failure => "_grokparsefailure_uri"
-    timeout_millis => 500
+  if [tmp_url] {
+    grok {
+      match => { "tmp_url" => "^(?<[http][request][method]>.*?) (?<[url][full]>.*?)( |$)((?<[tls][next_protocol]>.*?).*$)?" }
+      tag_on_failure => "_grokparsefailure_uri"
+      timeout_millis => 500
+    }
+    # Long URI
+    if [tmp_csv] !~ "ref=.*?$" {
+      mutate {
+          add_tag => [ "long_uri"]
+        }
+    }
   }
-
   translate {
     source => "[[rule][id]]"
     target => "[rule][description]"
@@ -152,12 +168,23 @@ if [tmp_csv] !~ "\w,\w" {
       add_field => { "[event][action]" => "denied" }
     }
   }
+
+# Date
   mutate {
     gsub => [
       "[event][created]", "[\[\]]", ""
     ]
   }  
   if [event][created] {
+    if [event][created][0] {
+      mutate {
+       copy => { "[event][created][0]" => "[tmp][event][created]" }
+       remove_field => [ "[event][created]" ]
+      }
+      mutate {
+        rename => { "[tmp][event][created]" => "[event][created]" }
+      }
+    }
     date {
       # "26/aug/2020:19:35:09.533 +0000"
       # ts=[12/oct/2020:17:24:01 +0000]
@@ -168,14 +195,8 @@ if [tmp_csv] !~ "\w,\w" {
       tag_on_failure => "_dateparsefailure_ec"
     }
   }
-  if [http][request][referrer] == "," {
-    mutate {
-      remove_field => ["[http][request][referrer]"]
-    }
-  }
-
   mutate {
-    remove_field => [ "tmp_csv", "tmp" ]
+    remove_field => [ "tmp_csv_2", "tmp_url", "tmp_referrer", "tmp_csv", "tmp" ]
   }
 }
 output {