From 9c8a2b7b69e82259af97c7c3d8f5d0a465b5e959 Mon Sep 17 00:00:00 2001 From: Brian Grabau Date: Mon, 21 Oct 2024 15:48:02 -0500 Subject: [PATCH] Added addtional mappings and imprived parsing --- ...it_azure.event_hub_interactive_signin.conf | 267 ++++++++++-------- 1 file changed, 152 insertions(+), 115 deletions(-) diff --git a/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf b/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf index 784d09fa..8a9c7db4 100644 --- a/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf +++ b/config/processors/event_hub_audit_azure.event_hub_interactive_signin.conf @@ -5,85 +5,117 @@ input { address => VAR_PIPELINE_NAME } } -filter { +filter { mutate{ remove_field => [ "host", "event" ] } mutate{ add_field => { "[event][module]" => "azure" } add_field => { "[event][dataset]" => "azure.interactivesignin" } - add_field => { "[log][source][hostname]" => "%{[az][TenantId]}" } } - if [message] =~ '^{"records": \[' { - json { - source => "message" + json { + source => "message" target => "tmp" - skip_on_invalid_json => true + skip_on_invalid_json => false + tag_on_failure => "_jsonparsefailure_split_msg" } + if [message] =~ '^{"records": \[' { split { field => "[tmp][records]" target => "az" } - mutate { - replace => { "message" => "%{az}" } - } } else { - json { - source => "message" - target => "az" - skip_on_invalid_json => true + mutate { + rename => { "tmp" => "az"} } } - mutate { - rename => { "[az][TenantId]" => "[cloud][account][id]" } - rename => { "[az][TimeGenerated]" => "[event][ingested]" } - rename => { "[az][OperationName]" => "[event][reference]" } - rename => { "[az][Category]" => "[event][category]" } - # rename => { "host" => "[agent][name]" } - rename => { "[az][DurationMs]" => "[event][duration]"} - rename => { "[az][ResultType]" => "[error][code]" } - rename => { "[az][CorrelationId]" => "[event][id]" } - rename => { "[az][Identity]" => "[file][group]" } - rename => { "[az][CreatedDateTime]" => "[event][created]" } - rename => { "[az][IPAddress]" => "[source][ip]" } - rename => { "[az][UserAgent]" => "[user_agent][original]" } - rename => { "[az][UserPrincipalName]" => "[user][email]" } - rename => { "[az][AppDisplayName]" => "[process][name]" } - rename => { "[az][UserId]" => "[user][id]" } - rename => { "[az][OperationVersion]" => "[service][version]" } - rename => { "[az][AppId]" => "[process][entity_id]" } - rename => { "[az][Status][failureReason]" => "[event][reason]" } - rename => { "[az][ResultDescription]" => "[error][message]" } - rename => { "[az][AutonomousSystemNumber]" => "[source][as][number]" } - rename => { "[az][UserDisplayName]" => "[user][full_name]" } - rename => { "[az][ConditionalAccessStatus]" => "[event][action]" } - rename => { "[az][ResourceServicePrincipalId]" => "[cloud][instance][id]" } - rename => { "[az][Status][additionalDetails]" => "[rule][category]"} - rename => { "[az][Status][errorCode]" => "[rule][id]" } - rename => { "[az][AuthenticationRequirement]" => "[event][kind]"} - rename => { "[az][ClientAppUsed]" => "[event][provider]"} - rename => { "[az][MfaDetail]" => "[event][outcome]"} - rename => { "[az][ResourceDisplayName]" => "[group][name]"} - rename => { "[az][Type]" => "[event][type]"} - rename => { "[az][DeviceDetail][deviceId]" => "[host][id]"} - rename => { "[az][DeviceDetail][operatingSystem]" => "[host][os][name]"} - rename => { "[az][DeviceDetail][displayName]" => "[host][hostname]"} + json { + source => "[az][networklocationdetails]" + target => "[az][networklocationdetails]" + tag_on_failure => "_jsonparsefailure_network" } - mutate{ - gsub => ["[az][NetworkLocationDetails]", "[\r\n ]", ""] - gsub => ["[az][NetworkLocationDetails]", '\"', '"'] + json { + source => "[az][devicedetail]" + target => "[az][devicedetail]" + tag_on_failure => "_jsonparsefailure_device" } - json { - source => "[az][NetworkLocationDetails]" - target => "[az][NetworkLocationDetails]" + json { + source => "[az][locationdetails]" + target => "[az][locationdetails]" + tag_on_failure => "_jsonparsefailure_location" + } + json { + source => "[az][authenticationprocessingdetails]" + target => "[az][authenticationprocessingdetails]" + tag_on_failure => "_jsonparsefailure_auth_proc" } + json { + source => "[az][authenticationdetails]" + target => "[az][authenticationdetails]" + tag_on_failure => "_jsonparsefailure_auth_detail" + } + json { + source => "[az][authenticationrequirementpolicies]" + target => "[az][authenticationrequirementpolicies]" + tag_on_failure => "_jsonparsefailure_auth_policy" + } + json { + source => "[az][conditionalaccesspolicies]" + target => "[az][conditionalaccesspolicies]" + tag_on_failure => "_jsonparsefailure_conditional" + } + json { + source => "[az][status]" + target => "[az][status]" + tag_on_failure => "_jsonparsefailure_status" + } + ### lowercase all field names + ruby { + init => '@ignore = [ "path", "@timestamp", "@metadata", "host", "@version" ]' + code => ' + def processArray(a) + newArray = [] + a.each { |x| + newArray << processObject(x) + } + newArray + end + def processHash(h) + newHash = {} + h.each { |k, v| + newHash[k.downcase] = processObject(v) + } + newHash + end + def processObject(v) + if v.kind_of?(Array) + processArray(v) + elsif v.kind_of?(Hash) + processHash(v) + else + v + end + end + def filter(i_event) + i_event.to_hash.each { |k, v| + unless @ignore.include?(k) + i_event.remove(k) + i_event.set(k.downcase, processObject(v)) + end + } + [i_event] + end + filter(event) + ' + } + ruby { code => " networktype_list = [] networknames_list = [] - if event.get('[az][NetworkLocationDetails]').kind_of? Array - event.get('[az][NetworkLocationDetails]').each do |item| + if event.get('[az][networklocationdetails]').kind_of? Array + event.get('[az][networklocationdetails]').each do |item| if item['networkType'] != '' networktype_list.push(item['networkType']) end @@ -102,74 +134,75 @@ filter { " tag_on_exception => "ruby_block_exception" } - json { - source => "[az][LocationDetails]" - target => "[az][LocationDetails]" - skip_on_invalid_json => true - } - json { - source => "[az][AuthenticationDetails]" - target => "[az][AuthenticationDetails]" - skip_on_invalid_json => true - } - # json { - # source => "[az][DeviceDetail]" - # target => "[az][DeviceDetail]" - # skip_on_invalid_json => true - # } + # rename => { "[az][mfadetail]" => "[event][outcome]" } mutate { - gsub => [ - "[az][AuthenticationProcessingDetails]", '"""', "" - ] + rename => { "[az][tenantid]" => "[cloud][account][id]" } + rename => { "[az][timegenerated]" => "[event][ingested]" } + rename => { "[az][operationname]" => "[event][reference]" } + rename => { "[az][category]" => "[event][category]" } + rename => { "[az][durationms]" => "[event][duration]"} + rename => { "[az][resulttype]" => "[error][code]" } + rename => { "[az][correlationid]" => "[event][id]" } + rename => { "[az][identity]" => "[file][group]" } + rename => { "[az][createddatetime]" => "[event][created]" } + rename => { "[az][ipaddress]" => "[source][ip]" } + rename => { "[az][useragent]" => "[user_agent][original]" } + rename => { "[az][userprincipalname]" => "[user][email]" } + rename => { "[az][appdisplayname]" => "[process][name]" } + rename => { "[az][userid]" => "[user][id]" } + rename => { "[az][operationversion]" => "[service][version]" } + rename => { "[az][appid]" => "[process][entity_id]" } + rename => { "[az][status][failurereason]" => "[event][reason]" } + rename => { "[az][resultdescription]" => "[error][message]" } + rename => { "[az][autonomoussystemnumber]" => "[source][as][number]" } + rename => { "[az][userdisplayname]" => "[user][full_name]" } + rename => { "[az][conditionalaccessstatus]" => "[event][action]" } + rename => { "[az][resourceserviceprincipalid]" => "[cloud][instance][id]" } + rename => { "[az][status][additionaldetails]" => "[rule][category]"} + rename => { "[az][status][errorcode]" => "[rule][id]" } + rename => { "[az][authenticationrequirement]" => "[event][kind]" } + rename => { "[az][clientappused]" => "[event][provider]" } + rename => { "[az][resourcedisplayname]" => "[group][name]" } + rename => { "[az][type]" => "[event][type]" } + rename => { "[az][devicedetail][deviceid]" => "[host][id]"} + rename => { "[az][devicedetail][operatingsystem]" => "[host][os][name]"} + rename => { "[az][devicedetail][displayname]" => "[host][hostname]"} + rename => { "[az][authenticationdetails][authenticationmethod]" => "[http][request][method]" } + rename => { "[az][authenticationdetails][authenticationstepresultdetail]" => "[rule][description]" } + rename => { "[az][authenticationdetails][authenticationsteprequirement]" =>"[error][type]" } + rename => { "[az][authenticationdetails][0][authenticationmethod]" => "[http][request][method]" } + rename => { "[az][authenticationdetails][0][authenticationstepresultdetail]" => "[rule][description]" } + rename => { "[az][authenticationdetails][0][authenticationsteprequirement]" =>"[error][type]" } + rename => { "[az][devicedetail][browser]" => "[process][parent][name]" } + rename => { "[az][devicedetail][trusttype]" => "[tls][version_protocol]" } + rename => { "[az][authenticationdetails]" => "[event][outcome]" } + rename => { "[az][conditionalaccesspolicies]" => "[user][group][name]" } + copy => { "[az][authenticationprocessingdetails][0][key]" => "[tls][cipher]" } + } + json_encode { + source => "[event][outcome]" + } + json_encode { + source => "[user][group][name]" } - json { - source => "[az][AuthenticationProcessingDetails]" - target => "[az][AuthenticationProcessingDetails]" - skip_on_invalid_json => true + mutate { + add_field => { "[@metadata][ismanaged]" => "host_is_not_managed" } } - - # json { - # source => "[az][ConditionalAccessPolicies]" - # target => "[user][roles]" - # skip_on_invalid_json => true - # } mutate { - rename => { "[az][AuthenticationDetails][authenticationMethod]" => "[http][request][method]" } - rename => { "[az][AuthenticationDetails][succeeded]" => "[event][outcome]" } - rename => { "[az][AuthenticationDetails][authenticationStepResultDetail]" => "[rule][description]" } - rename => { "[az][AuthenticationDetails][authenticationStepRequirement]" =>"[error][type]" } - rename => { "[az][AuthenticationDetails][0][authenticationMethod]" => "[http][request][method]" } - rename => { "[az][AuthenticationDetails][0][succeeded]" => "[event][outcome]" } - rename => { "[az][AuthenticationDetails][0][authenticationStepResultDetail]" => "[rule][description]" } - rename => { "[az][AuthenticationDetails][0][authenticationStepRequirement]" =>"[error][type]" } - } - if [az][AuthenticationDetails][1] { - mutate { - merge => { "[http][request][method]" => "[az][AuthenticationDetails][1][authenticationMethod]" } - merge => { "[event][outcome]" => "[az][AuthenticationDetails][1][succeeded]" } - merge => { "[rule][description]" => "[az][AuthenticationDetails][1][authenticationStepResultDetail]" } - merge => { "[error][type]" => "[az][AuthenticationDetails][1][authenticationStepRequirement]" } - } + merge => { "[event][outcome]" => "[@metadata][ismanaged]" } + } + mutate { + add_field => { "[@metadata][iscompliant]" => "host_is_iscompliant" } } - if [az][AuthenticationDetails][2] { mutate { - merge => { "[http][request][method]" => "[az][AuthenticationDetails][2][authenticationMethod]" } - merge => { "[event][outcome]" => "[az][AuthenticationDetails][2][succeeded]" } - merge => { "[rule][description]" => "[az][AuthenticationDetails][2][authenticationStepResultDetail]" } - merge => { "[error][type]" => "[az][AuthenticationDetails][2][authenticationStepRequirement]" } - } - } - mutate { - remove_field => [ "tmp", "[az]" ] - } - # remove_field => [ "group", "service", "file", "log", "user", "host", "process", "cloud", "network" , "user_agent", "source", "as" ] - # remove_field => [ "[az][OriginalRequestId]", "[az][SessionLifetimePolicies]", "[az][ResourceTenantId]", "[az][OperationVersion]", "[az][RiskLevelDuringSignIn]", "[az][CorrelationId]", "[az][Identity]", "[az][Status]", "[az][ConditionalAccessPolicies]", "[az][AutonomousSystemNumber]", "[az][LocationDetails]", "[az][Resource]", "[az][ResultType]", "[az][CrossTenantAccessType]", "[az][AppId]", "[az][RiskEventTypes]", "[az][ResultDescription]", "[az][AuthenticationProtocol]", "[az][Level]", "[az][Id]", "[az][UserType]", "[az][UniqueTokenIdentifier]", "[az][CreatedDateTime]", "[az][AuthenticationRequirementPolicies]", "[az][Category]", "[az][DurationMs]", "[az][IsInteractive]", "[az][AuthenticationContextClassReferences]", "[az][ClientAppUsed]", "[az][ResourceDisplayName]", "[az][AuthenticationRequirement]", "[az][TokenIssuerType]", "[az][ProcessingTimeInMilliseconds]", "[az][HomeTenantId]", "[az][UserAgent]", "[az][NetworkLocationDetails]", "[az][ResourceServicePrincipalId]", "[az][AppDisplayName]", "[az][UserDisplayName]", "[az][ConditionalAccessStatus]", "[az][ResourceGroup]", "[az][RiskEventTypes]", "[az][ResourceIdentity]", "[az][MfaDetail]", "[az][TimeGenerated]", "[az][OperationName]", "[az][RiskDetail]", "[az][DeviceDetail]", "[az][RiskLevelAggregated]", "[az][TenantId]", "[az][Type]", "[az][RiskState]", "[az][_Internal_WorkspaceResourceId]", "[az][AADTenantId]", "[az][ResourceId]", "[az][UserId]", "[az][AuthenticationProcessingDetails]", "[az][UserPrincipalName]", "[az][IPAddress]", "[az][ResultSignature]", "[az][SourceSystem]", "[az][Location]", "[az][_ItemId]", "[az][RiskEventTypes_V2]" ] - # split { - # field => "[user][roles]" - # target => "[user][roles]" - # } + merge => { "[event][outcome]" => "[@metadata][iscompliant]" } + } mutate { + add_field => { "[log][source][hostname]" => "%{[host][hostname]}" } + } + + mutate { gsub => [ "[event][created]", "^(.*?)T(.*?\.\d\d\d).*?$", "\1 \2" , "[event][ingested]", "^(.*?)T(.*?\.\d\d\d).*?$", "\1 \2" @@ -189,6 +222,10 @@ filter { target => "[event][ingested]" tag_on_failure => "_dateparsefailure_ei" } + # + mutate { + remove_field => [ "tmp", "az" ] + } } output { pipeline { send_to => [enrichments] }