From b9808007c3bc2f774ebe76e43c67d0851f4ef5a8 Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Thu, 27 Apr 2023 10:03:32 +0530 Subject: [PATCH 1/9] XDR-4369: add DCR rule module (feat) --- modules/monitor-data-collection/README.md | 48 ++++ modules/monitor-data-collection/main.tf | 254 +++++++++++++++++++ modules/monitor-data-collection/outputs.tf | 4 + modules/monitor-data-collection/variables.tf | 181 +++++++++++++ 4 files changed, 487 insertions(+) create mode 100644 modules/monitor-data-collection/README.md create mode 100644 modules/monitor-data-collection/main.tf create mode 100644 modules/monitor-data-collection/outputs.tf create mode 100644 modules/monitor-data-collection/variables.tf diff --git a/modules/monitor-data-collection/README.md b/modules/monitor-data-collection/README.md new file mode 100644 index 0000000..fb3158f --- /dev/null +++ b/modules/monitor-data-collection/README.md @@ -0,0 +1,48 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.12 | +| [azurerm](#requirement\_azurerm) | ~> 3.53 | + +## Providers + +| Name | Version | +|------|---------| +| [azurerm](#provider\_azurerm) | 3.53.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [azurerm_monitor_data_collection_rule.dcr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_data_collection_rule) | resource | +| [azurerm_monitor_data_collection_rule_association.dcr_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_data_collection_rule_association) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [data\_flow](#input\_data\_flow) | One or more data\_flow blocks as defined below. |
list(object({
destinations = list(string)
streams = list(string)
built_in_transform = optional(string)
output_stream = optional(string)
transform_kql = optional(string)
}))
| n/a | yes | +| [data\_sources](#input\_data\_sources) | A data\_sources block as defined below. |
object({
data_import = optional(object({
event_hub_data_source = object({
name = string
stream = string
consumer_group = optional(string)
})
}))
extension = optional(list(object({
extension_name = string
name = string
streams = list(string)
extension_json = optional(string)
input_data_sources = optional(list(string))
})))
iis_log = optional(list(object({
name = string
streams = list(string)
log_directories = optional(list(string))
})))
log_file = optional(list(object({
name = string
streams = list(string)
file_patterns = list(string)
format = string
settings = optional(object({
text = object({
record_start_timestamp_format = string
})
}))
})))
performance_counter = optional(list(object({
name = string
counter_specifiers = list(string)
sampling_frequency_in_seconds = number
streams = list(string)
})))
platform_telemetry = optional(list(object({
name = string
streams = list(string)
})))
prometheus_forwarder = optional(list(object({
name = string
streams = list(string)
label_include_filter = optional(list(object({
label = string
value = string
})))
})))
syslog = optional(list(object({
name = string
log_levels = list(string)
facility_names = list(string)
streams = optional(list(string)) # After provider version 4.0, this will be required.
})))
windows_event_log = optional(list(object({
name = string
streams = list(string)
x_path_queries = list(string)
})))
windows_firewall_log = optional(list(object({
name = string
streams = list(string)
})))
})
| `{}` | no | +| [description](#input\_description) | The description of the Data Collection Rule | `string` | `""` | no | +| [destinations](#input\_destinations) | A destinations block as defined below |
object({
azure_monitor_metrics = optional(object({
name = string
}))
event_hub = optional(list(object({
name = string
event_hub_id = string
})))
event_hub_direct = optional(list(object({
name = string
event_hub_id = string
})))
log_analytics = optional(list(object({
name = string
workspace_resource_id = string
})))
monitor_account = optional(list(object({
name = string
monitor_account_id = string
})))
storage_blob = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_blob_direct = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_table_direct = optional(list(object({
name = string
table_name = string
storage_account_id = string
})))
})
| n/a | yes | +| [identity](#input\_identity) | An identity block as defined below. |
object({
type = string
identity_ids = optional(list(string))
})
| `null` | no | +| [kind](#input\_kind) | The kind of the Data Collection Rule. | `string` | `""` | no | +| [location](#input\_location) | The Azure Region where the Data Collection Rule should exist | `string` | n/a | yes | +| [name](#input\_name) | The name which should be used for this Data Collection Rule | `string` | n/a | yes | +| [resource\_group\_name](#input\_resource\_group\_name) | The name of the Resource Group where the Data Collection Rule should exist | `string` | n/a | yes | +| [stream\_declaration](#input\_stream\_declaration) | A stream\_declaration block as defined below. |
object({
stream_name = string
column = list(object({
name = string
type = string
}))
})
| `null` | no | +| [tags](#input\_tags) | A mapping of tags to assign to the resource | `map(string)` | `{}` | no | +| [target\_resource\_id](#input\_target\_resource\_id) | The ID of the Azure Resource which to associate to a Data Collection Rule or a Data Collection Endpoint. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [id](#output\_id) | The ID of the Data Collection Rule. | + diff --git a/modules/monitor-data-collection/main.tf b/modules/monitor-data-collection/main.tf new file mode 100644 index 0000000..a78acb5 --- /dev/null +++ b/modules/monitor-data-collection/main.tf @@ -0,0 +1,254 @@ +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY AN AZURE MONITOR DATA COLLECTION RULE (DCR) +# --------------------------------------------------------------------------------------------------------------------- + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.53" + } + } + required_version = ">= 0.12" + experiments = [module_variable_optional_attrs] +} + +resource "azurerm_monitor_data_collection_rule" "dcr" { + name = var.name + resource_group_name = var.resource_group_name + location = var.location + description = var.description + kind = var.kind + tags = var.tags + + data_sources { #TODO: Fix this + dynamic "data_import" { + for_each = var.data_sources.data_import != null ? [var.data_sources.data_import] : [] + + content { + event_hub_data_source { + name = data_import.value.event_hub_data_source.name + stream = data_import.value.event_hub_data_source.stream + consumer_group = data_import.value.event_hub_data_source.consumer_group + } + } + } + + dynamic "extension" { + for_each = var.data_sources.extension != null ? var.data_sources.extension : [] + + content { + extension { + extension_name = extension.value.extension_name + name = extension.value.name + streams = extension.value.streams + extension_json = extension.value.extension_json + input_data_sources = extension.value.input_data_sources + } + } + } + + dynamic "iis_log" { + for_each = var.data_sources.iis_log != null ? var.data_sources.iis_log : [] + + content { + iis_log { + name = iis_log.value.name + streams = iis_log.value.streams + log_directories = iis_log.value.log_directories + } + } + } + + dynamic "log_file" { + for_each = var.data_sources.log_file != null ? var.data_sources.log_file : [] + + content { + log_file { + name = log_file.value.name + streams = log_file.value.streams + file_patterns = log_file.value.file_patterns + format = log_file.value.format + settings = log_file.value.settings != null ? { + text = { + record_start_timestamp_format = log_file.value.settings.text.record_start_timestamp_format + } + } : null + } + } + } + + dynamic "performance_counter" { + for_each = var.data_sources.performance_counter != null ? var.data_sources.performance_counter : [] + + content { + performance_counter { + name = performance_counter.value.name + counter_specifiers = performance_counter.value.counter_specifiers + sampling_frequency_in_seconds = performance_counter.value.sampling_frequency_in_seconds + streams = performance_counter.value.streams + } + } + } + + dynamic "platform_telemetry" { + for_each = var.data_sources.platform_telemetry != null ? var.data_sources.platform_telemetry : [] + + content { + platform_telemetry { + name = platform_telemetry.value.name + streams = platform_telemetry.value.streams + } + } + } + + dynamic "prometheus_forwarder" { + for_each = var.data_sources.prometheus_forwarder != null ? var.data_sources.prometheus_forwarder : [] + + content { + prometheus_forwarder { + name = prometheus_forwarder.value.name + streams = prometheus_forwarder.value.streams + + label_include_filter = prometheus_forwarder.value.label_include_filter != null ? [ + for label in prometheus_forwarder.value.label_include_filter : { + label = label.label + value = label.value + } + ] : null + } + } + } + + dynamic "syslog" { + for_each = var.data_sources.syslog != null ? var.data_sources.syslog : {} + content { + name = syslog.value.name + log_levels = syslog.value.log_levels + facility_names = syslog.value.facility_names + streams = syslog.value.streams + } + } + + dynamic "windows_event_log" { + for_each = var.data_sources.windows_event_log != null ? var.data_sources.windows_event_log : {} + content { + name = windows_event_log.value.name + streams = windows_event_log.value.streams + x_path_queries = windows_event_log.value.x_path_queries + } + } + + dynamic "windows_firewall_log" { + for_each = var.data_sources.windows_firewall_log != null ? var.data_sources.windows_firewall_log : {} + content { + name = windows_firewall_log.value.name + streams = windows_firewall_log.value.streams + } + } + } + + dynamic "data_flow" { + for_each = var.data_flow + content { + destinations = data_flow.value["destinations"] + streams = data_flow.value["streams"] + built_in_transform = data_flow.value["built_in_transform"] + output_stream = data_flow.value["output_stream"] + transform_kql = data_flow.value["transform_kql"] + } + } + + dynamic "destinations" { + for_each = [var.destinations] + content { + dynamic "azure_monitor_metrics" { + for_each = [destinations.value["azure_monitor_metrics"]] != null ? [destinations.value["azure_monitor_metrics"]] : [] + content { + name = azure_monitor_metrics.value["name"] + } + } + dynamic "event_hub" { + for_each = destinations.value["event_hub"] != null ? destinations.value["event_hub"] : [] + content { + name = event_hub.value["name"] + event_hub_id = event_hub.value["event_hub_id"] + } + } + dynamic "event_hub_direct" { + for_each = destinations.value["event_hub_direct"] != null ? destinations.value["event_hub_direct"] : [] + content { + name = event_hub_direct.value["name"] + event_hub_id = event_hub_direct.value["event_hub_id"] + } + } + dynamic "log_analytics" { + for_each = destinations.value["log_analytics"] != null ? destinations.value["log_analytics"] : [] + content { + name = log_analytics.value["name"] + workspace_resource_id = log_analytics.value["workspace_resource_id"] + } + } + dynamic "monitor_account" { + for_each = destinations.value["monitor_account"] != null ? destinations.value["monitor_account"] : [] + content { + name = monitor_account.value["name"] + monitor_account_id = monitor_account.value["monitor_account_id"] + } + } + dynamic "storage_blob" { + for_each = destinations.value["storage_blob"] != null ? destinations.value["storage_blob"] : [] + content { + name = storage_blob.value["name"] + storage_account_id = storage_blob.value["storage_account_id"] + container_name = storage_blob.value["container_name"] + } + } + dynamic "storage_blob_direct" { + for_each = destinations.value["storage_blob_direct"] != null ? destinations.value["storage_blob_direct"] : [] + content { + name = storage_blob_direct.value["name"] + storage_account_id = storage_blob_direct.value["storage_account_id"] + container_name = storage_blob_direct.value["container_name"] + } + } + dynamic "storage_table_direct" { + for_each = destinations.value["storage_table_direct"] != null ? destinations.value["storage_table_direct"] : [] + content { + name = storage_table_direct.value["name"] + table_name = storage_table_direct.value["table_name"] + storage_account_id = storage_table_direct.value["storage_account_id"] + } + } + } + } + + dynamic "stream_declaration" { + for_each = var.stream_declaration != null ? [var.stream_declaration] : [] + content { + stream_name = stream_declaration.value["stream_name"] + dynamic "column" { + for_each = stream_declaration.value["column"] + content { + name = column.value["name"] + type = column.value["type"] + } + } + } + } + + dynamic "identity" { + for_each = var.identity != null ? [var.identity] : [] + content { + type = identity.value["type"] + identity_ids = identity.value["identity_ids"] + } + } +} + +resource "azurerm_monitor_data_collection_rule_association" "dcr_association" { + name = "${var.name}-association" + description = var.description + target_resource_id = var.target_resource_id + data_collection_rule_id = azurerm_monitor_data_collection_rule.dcr.id +} diff --git a/modules/monitor-data-collection/outputs.tf b/modules/monitor-data-collection/outputs.tf new file mode 100644 index 0000000..f91d937 --- /dev/null +++ b/modules/monitor-data-collection/outputs.tf @@ -0,0 +1,4 @@ +output "id" { + description = "The ID of the Data Collection Rule." + value = azurerm_monitor_data_collection_rule.dcr.id +} diff --git a/modules/monitor-data-collection/variables.tf b/modules/monitor-data-collection/variables.tf new file mode 100644 index 0000000..d709499 --- /dev/null +++ b/modules/monitor-data-collection/variables.tf @@ -0,0 +1,181 @@ +variable "location" { + type = string + description = "The Azure Region where the Data Collection Rule should exist" +} + +variable "resource_group_name" { + type = string + description = "The name of the Resource Group where the Data Collection Rule should exist" +} + +variable "name" { + type = string + description = "The name which should be used for this Data Collection Rule" +} + +variable "target_resource_id" { + type = string + description = "The ID of the Azure Resource which to associate to a Data Collection Rule or a Data Collection Endpoint." +} + +variable "kind" { + type = string + description = "The kind of the Data Collection Rule." + default = "" +} + +variable "description" { + type = string + description = "The description of the Data Collection Rule" + default = "" +} + +variable "tags" { + type = map(string) + description = "A mapping of tags to assign to the resource" + default = {} +} + +variable "data_sources" { + type = object({ + data_import = optional(object({ + event_hub_data_source = object({ + name = string + stream = string + consumer_group = optional(string) + }) + })) + extension = optional(list(object({ + extension_name = string + name = string + streams = list(string) + extension_json = optional(string) + input_data_sources = optional(list(string)) + }))) + iis_log = optional(list(object({ + name = string + streams = list(string) + log_directories = optional(list(string)) + }))) + log_file = optional(list(object({ + name = string + streams = list(string) + file_patterns = list(string) + format = string + settings = optional(object({ + text = object({ + record_start_timestamp_format = string + }) + })) + }))) + performance_counter = optional(list(object({ + name = string + counter_specifiers = list(string) + sampling_frequency_in_seconds = number + streams = list(string) + }))) + platform_telemetry = optional(list(object({ + name = string + streams = list(string) + }))) + prometheus_forwarder = optional(list(object({ + name = string + streams = list(string) + label_include_filter = optional(list(object({ + label = string + value = string + }))) + }))) + syslog = optional(list(object({ + name = string + log_levels = list(string) + facility_names = list(string) + streams = optional(list(string)) # After provider version 4.0, this will be required. + }))) + windows_event_log = optional(list(object({ + name = string + streams = list(string) + x_path_queries = list(string) + }))) + windows_firewall_log = optional(list(object({ + name = string + streams = list(string) + }))) + }) + description = "A data_sources block as defined below." + default = {} + # TODO: add variable validations +} + +variable "data_flow" { + type = list(object({ + destinations = list(string) + streams = list(string) + built_in_transform = optional(string) + output_stream = optional(string) + transform_kql = optional(string) + })) + description = "One or more data_flow blocks as defined below." +} + +variable "destinations" { + type = object({ + azure_monitor_metrics = optional(object({ + name = string + })) + event_hub = optional(list(object({ + name = string + event_hub_id = string + }))) + event_hub_direct = optional(list(object({ + name = string + event_hub_id = string + }))) + log_analytics = optional(list(object({ + name = string + workspace_resource_id = string + }))) + monitor_account = optional(list(object({ + name = string + monitor_account_id = string + }))) + storage_blob = optional(list(object({ + name = string + storage_account_id = string + container_name = string + }))) + storage_blob_direct = optional(list(object({ + name = string + storage_account_id = string + container_name = string + }))) + storage_table_direct = optional(list(object({ + name = string + table_name = string + storage_account_id = string + }))) + }) + description = "A destinations block as defined below" + # TODO: add variable validations +} + +variable "stream_declaration" { + type = object({ + stream_name = string + column = list(object({ + name = string + type = string + })) + }) + default = null + description = "A stream_declaration block as defined below." +} + +variable "identity" { + type = object({ + type = string + identity_ids = optional(list(string)) + }) + default = null + description = "An identity block as defined below." +} From d3576be110fc8d35e5344c506eb685f440b599ef Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Thu, 27 Apr 2023 16:22:10 +0530 Subject: [PATCH 2/9] XDR-4369: update data sources dynamic blocks --- modules/monitor-data-collection/README.md | 4 +- modules/monitor-data-collection/main.tf | 291 ++++++++++------------ 2 files changed, 135 insertions(+), 160 deletions(-) diff --git a/modules/monitor-data-collection/README.md b/modules/monitor-data-collection/README.md index fb3158f..6137754 100644 --- a/modules/monitor-data-collection/README.md +++ b/modules/monitor-data-collection/README.md @@ -31,12 +31,12 @@ No modules. | [data\_sources](#input\_data\_sources) | A data\_sources block as defined below. |
object({
data_import = optional(object({
event_hub_data_source = object({
name = string
stream = string
consumer_group = optional(string)
})
}))
extension = optional(list(object({
extension_name = string
name = string
streams = list(string)
extension_json = optional(string)
input_data_sources = optional(list(string))
})))
iis_log = optional(list(object({
name = string
streams = list(string)
log_directories = optional(list(string))
})))
log_file = optional(list(object({
name = string
streams = list(string)
file_patterns = list(string)
format = string
settings = optional(object({
text = object({
record_start_timestamp_format = string
})
}))
})))
performance_counter = optional(list(object({
name = string
counter_specifiers = list(string)
sampling_frequency_in_seconds = number
streams = list(string)
})))
platform_telemetry = optional(list(object({
name = string
streams = list(string)
})))
prometheus_forwarder = optional(list(object({
name = string
streams = list(string)
label_include_filter = optional(list(object({
label = string
value = string
})))
})))
syslog = optional(list(object({
name = string
log_levels = list(string)
facility_names = list(string)
streams = optional(list(string)) # After provider version 4.0, this will be required.
})))
windows_event_log = optional(list(object({
name = string
streams = list(string)
x_path_queries = list(string)
})))
windows_firewall_log = optional(list(object({
name = string
streams = list(string)
})))
})
| `{}` | no | | [description](#input\_description) | The description of the Data Collection Rule | `string` | `""` | no | | [destinations](#input\_destinations) | A destinations block as defined below |
object({
azure_monitor_metrics = optional(object({
name = string
}))
event_hub = optional(list(object({
name = string
event_hub_id = string
})))
event_hub_direct = optional(list(object({
name = string
event_hub_id = string
})))
log_analytics = optional(list(object({
name = string
workspace_resource_id = string
})))
monitor_account = optional(list(object({
name = string
monitor_account_id = string
})))
storage_blob = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_blob_direct = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_table_direct = optional(list(object({
name = string
table_name = string
storage_account_id = string
})))
})
| n/a | yes | -| [identity](#input\_identity) | An identity block as defined below. |
object({
type = string
identity_ids = optional(list(string))
})
| `null` | no | +| [identity](#input\_identity) | An identity block as defined below. |
object({
type = string
identity_ids = optional(list(string))
})
| `null` | no | | [kind](#input\_kind) | The kind of the Data Collection Rule. | `string` | `""` | no | | [location](#input\_location) | The Azure Region where the Data Collection Rule should exist | `string` | n/a | yes | | [name](#input\_name) | The name which should be used for this Data Collection Rule | `string` | n/a | yes | | [resource\_group\_name](#input\_resource\_group\_name) | The name of the Resource Group where the Data Collection Rule should exist | `string` | n/a | yes | -| [stream\_declaration](#input\_stream\_declaration) | A stream\_declaration block as defined below. |
object({
stream_name = string
column = list(object({
name = string
type = string
}))
})
| `null` | no | +| [stream\_declaration](#input\_stream\_declaration) | A stream\_declaration block as defined below. |
object({
stream_name = string
column = list(object({
name = string
type = string
}))
})
| `null` | no | | [tags](#input\_tags) | A mapping of tags to assign to the resource | `map(string)` | `{}` | no | | [target\_resource\_id](#input\_target\_resource\_id) | The ID of the Azure Resource which to associate to a Data Collection Rule or a Data Collection Endpoint. | `string` | n/a | yes | diff --git a/modules/monitor-data-collection/main.tf b/modules/monitor-data-collection/main.tf index a78acb5..430ba04 100644 --- a/modules/monitor-data-collection/main.tf +++ b/modules/monitor-data-collection/main.tf @@ -21,129 +21,107 @@ resource "azurerm_monitor_data_collection_rule" "dcr" { kind = var.kind tags = var.tags - data_sources { #TODO: Fix this - dynamic "data_import" { - for_each = var.data_sources.data_import != null ? [var.data_sources.data_import] : [] - - content { - event_hub_data_source { - name = data_import.value.event_hub_data_source.name - stream = data_import.value.event_hub_data_source.stream - consumer_group = data_import.value.event_hub_data_source.consumer_group + dynamic "data_sources" { + for_each = var.data_sources != {} ? [var.data_sources] : [] + content { + dynamic "data_import" { + for_each = data_sources.value["data_import"] != null ? [data_sources.value["data_import"]] : [] + content { + event_hub_data_source { + name = data_import.value["event_hub_data_source"]["name"] + stream = data_import.value["event_hub_data_source"]["stream"] + consumer_group = data_import.value["event_hub_data_source"]["consumer_group"] + } } } - } - - dynamic "extension" { - for_each = var.data_sources.extension != null ? var.data_sources.extension : [] - - content { - extension { - extension_name = extension.value.extension_name - name = extension.value.name - streams = extension.value.streams - extension_json = extension.value.extension_json - input_data_sources = extension.value.input_data_sources + dynamic "extension" { + for_each = data_sources.value["extension"] != null ? data_sources.value["extension"] : [] + content { + extension_name = extension.value["extension_name"] + name = extension.value["name"] + streams = extension.value["streams"] + extension_json = extension.value["extension_json"] + input_data_sources = extension.value["input_data_sources"] } } - } - - dynamic "iis_log" { - for_each = var.data_sources.iis_log != null ? var.data_sources.iis_log : [] - - content { - iis_log { - name = iis_log.value.name - streams = iis_log.value.streams - log_directories = iis_log.value.log_directories + dynamic "iis_log" { + for_each = data_sources.value["iis_log"] != null ? data_sources.value["iis_log"] : [] + content { + name = iis_log.value["name"] + streams = iis_log.value["streams"] + log_directories = iis_log.value["log_directories"] } } - } - - dynamic "log_file" { - for_each = var.data_sources.log_file != null ? var.data_sources.log_file : [] - - content { - log_file { - name = log_file.value.name - streams = log_file.value.streams - file_patterns = log_file.value.file_patterns - format = log_file.value.format - settings = log_file.value.settings != null ? { - text = { - record_start_timestamp_format = log_file.value.settings.text.record_start_timestamp_format + dynamic "log_file" { + for_each = data_sources.value["log_file"] != null ? data_sources.value["log_file"] : [] + content { + name = log_file.value["name"] + streams = log_file.value["streams"] + file_patterns = log_file.value["file_patterns"] + format = log_file.value["format"] + dynamic "settings" { + for_each = log_file.value["settings"] != null ? [log_file.value["settings"]] : [] + content { + text { + record_start_timestamp_format = settings.value["text"]["record_start_timestamp_format"] + } } - } : null + } } } - } - - dynamic "performance_counter" { - for_each = var.data_sources.performance_counter != null ? var.data_sources.performance_counter : [] - - content { - performance_counter { - name = performance_counter.value.name - counter_specifiers = performance_counter.value.counter_specifiers - sampling_frequency_in_seconds = performance_counter.value.sampling_frequency_in_seconds - streams = performance_counter.value.streams + dynamic "performance_counter" { + for_each = data_sources.value["performance_counter"] != null ? data_sources.value["performance_counter"] : [] + content { + name = performance_counter.value["name"] + counter_specifiers = performance_counter.value["counter_specifiers"] + sampling_frequency_in_seconds = performance_counter.value["sampling_frequency_in_seconds"] + streams = performance_counter.value["streams"] } } - } - - dynamic "platform_telemetry" { - for_each = var.data_sources.platform_telemetry != null ? var.data_sources.platform_telemetry : [] - - content { - platform_telemetry { - name = platform_telemetry.value.name - streams = platform_telemetry.value.streams + dynamic "platform_telemetry" { + for_each = data_sources.value["platform_telemetry"] != null ? data_sources.value["platform_telemetry"] : [] + content { + name = platform_telemetry.value["name"] + streams = platform_telemetry.value["streams"] } } - } - - dynamic "prometheus_forwarder" { - for_each = var.data_sources.prometheus_forwarder != null ? var.data_sources.prometheus_forwarder : [] - - content { - prometheus_forwarder { - name = prometheus_forwarder.value.name - streams = prometheus_forwarder.value.streams - - label_include_filter = prometheus_forwarder.value.label_include_filter != null ? [ - for label in prometheus_forwarder.value.label_include_filter : { - label = label.label - value = label.value + dynamic "prometheus_forwarder" { + for_each = data_sources.value["prometheus_forwarder"] != null ? data_sources.value["prometheus_forwarder"] : [] + content { + name = prometheus_forwarder.value["name"] + streams = prometheus_forwarder.value["streams"] + dynamic "label_include_filter" { + for_each = prometheus_forwarder.value["label_include_filter"] != null ? prometheus_forwarder.value["label_include_filter"] : [] + content { + label = label_include_filter.value["label"] + value = label_include_filter.value["value"] } - ] : null + } } } - } - - dynamic "syslog" { - for_each = var.data_sources.syslog != null ? var.data_sources.syslog : {} - content { - name = syslog.value.name - log_levels = syslog.value.log_levels - facility_names = syslog.value.facility_names - streams = syslog.value.streams + dynamic "syslog" { + for_each = data_sources.value["syslog"] != null ? data_sources.value["syslog"] : [] + content { + name = syslog.value["name"] + log_levels = syslog.value["log_levels"] + facility_names = syslog.value["facility_names"] + streams = syslog.value["streams"] + } } - } - - dynamic "windows_event_log" { - for_each = var.data_sources.windows_event_log != null ? var.data_sources.windows_event_log : {} - content { - name = windows_event_log.value.name - streams = windows_event_log.value.streams - x_path_queries = windows_event_log.value.x_path_queries + dynamic "windows_event_log" { + for_each = data_sources.value["windows_event_log"] != null ? data_sources.value["windows_event_log"] : [] + content { + name = windows_event_log.value["name"] + streams = windows_event_log.value["streams"] + x_path_queries = windows_event_log.value["x_path_queries"] + } } - } - - dynamic "windows_firewall_log" { - for_each = var.data_sources.windows_firewall_log != null ? var.data_sources.windows_firewall_log : {} - content { - name = windows_firewall_log.value.name - streams = windows_firewall_log.value.streams + dynamic "windows_firewall_log" { + for_each = data_sources.value["windows_firewall_log"] != null ? data_sources.value["windows_firewall_log"] : [] + content { + name = windows_firewall_log.value["name"] + streams = windows_firewall_log.value["streams"] + } } } } @@ -159,66 +137,63 @@ resource "azurerm_monitor_data_collection_rule" "dcr" { } } - dynamic "destinations" { - for_each = [var.destinations] - content { - dynamic "azure_monitor_metrics" { - for_each = [destinations.value["azure_monitor_metrics"]] != null ? [destinations.value["azure_monitor_metrics"]] : [] - content { - name = azure_monitor_metrics.value["name"] - } + destinations { + dynamic "azure_monitor_metrics" { + for_each = [var.destinations["azure_monitor_metrics"]] != null ? [var.destinations["azure_monitor_metrics"]] : [] + content { + name = azure_monitor_metrics.value["name"] } - dynamic "event_hub" { - for_each = destinations.value["event_hub"] != null ? destinations.value["event_hub"] : [] - content { - name = event_hub.value["name"] - event_hub_id = event_hub.value["event_hub_id"] - } + } + dynamic "event_hub" { + for_each = var.destinations["event_hub"] != null ? var.destinations["event_hub"] : [] + content { + name = event_hub.value["name"] + event_hub_id = event_hub.value["event_hub_id"] } - dynamic "event_hub_direct" { - for_each = destinations.value["event_hub_direct"] != null ? destinations.value["event_hub_direct"] : [] - content { - name = event_hub_direct.value["name"] - event_hub_id = event_hub_direct.value["event_hub_id"] - } + } + dynamic "event_hub_direct" { + for_each = var.destinations["event_hub_direct"] != null ? var.destinations["event_hub_direct"] : [] + content { + name = event_hub_direct.value["name"] + event_hub_id = event_hub_direct.value["event_hub_id"] } - dynamic "log_analytics" { - for_each = destinations.value["log_analytics"] != null ? destinations.value["log_analytics"] : [] - content { - name = log_analytics.value["name"] - workspace_resource_id = log_analytics.value["workspace_resource_id"] - } + } + dynamic "log_analytics" { + for_each = var.destinations["log_analytics"] != null ? var.destinations["log_analytics"] : [] + content { + name = log_analytics.value["name"] + workspace_resource_id = log_analytics.value["workspace_resource_id"] } - dynamic "monitor_account" { - for_each = destinations.value["monitor_account"] != null ? destinations.value["monitor_account"] : [] - content { - name = monitor_account.value["name"] - monitor_account_id = monitor_account.value["monitor_account_id"] - } + } + dynamic "monitor_account" { + for_each = var.destinations["monitor_account"] != null ? var.destinations["monitor_account"] : [] + content { + name = monitor_account.value["name"] + monitor_account_id = monitor_account.value["monitor_account_id"] } - dynamic "storage_blob" { - for_each = destinations.value["storage_blob"] != null ? destinations.value["storage_blob"] : [] - content { - name = storage_blob.value["name"] - storage_account_id = storage_blob.value["storage_account_id"] - container_name = storage_blob.value["container_name"] - } + } + dynamic "storage_blob" { + for_each = var.destinations["storage_blob"] != null ? var.destinations["storage_blob"] : [] + content { + name = storage_blob.value["name"] + storage_account_id = storage_blob.value["storage_account_id"] + container_name = storage_blob.value["container_name"] } - dynamic "storage_blob_direct" { - for_each = destinations.value["storage_blob_direct"] != null ? destinations.value["storage_blob_direct"] : [] - content { - name = storage_blob_direct.value["name"] - storage_account_id = storage_blob_direct.value["storage_account_id"] - container_name = storage_blob_direct.value["container_name"] - } + } + dynamic "storage_blob_direct" { + for_each = var.destinations["storage_blob_direct"] != null ? var.destinations["storage_blob_direct"] : [] + content { + name = storage_blob_direct.value["name"] + storage_account_id = storage_blob_direct.value["storage_account_id"] + container_name = storage_blob_direct.value["container_name"] } - dynamic "storage_table_direct" { - for_each = destinations.value["storage_table_direct"] != null ? destinations.value["storage_table_direct"] : [] - content { - name = storage_table_direct.value["name"] - table_name = storage_table_direct.value["table_name"] - storage_account_id = storage_table_direct.value["storage_account_id"] - } + } + dynamic "storage_table_direct" { + for_each = var.destinations["storage_table_direct"] != null ? var.destinations["storage_table_direct"] : [] + content { + name = storage_table_direct.value["name"] + table_name = storage_table_direct.value["table_name"] + storage_account_id = storage_table_direct.value["storage_account_id"] } } } From c29b8cd05f0ed01eab55d2ce81765544be1e7146 Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 11:41:37 +0530 Subject: [PATCH 3/9] XDR-4369: rename module --- .../README.md | 0 .../main.tf | 0 .../outputs.tf | 0 .../variables.tf | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename modules/{monitor-data-collection => monitor-data-collection-rule}/README.md (100%) rename modules/{monitor-data-collection => monitor-data-collection-rule}/main.tf (100%) rename modules/{monitor-data-collection => monitor-data-collection-rule}/outputs.tf (100%) rename modules/{monitor-data-collection => monitor-data-collection-rule}/variables.tf (100%) diff --git a/modules/monitor-data-collection/README.md b/modules/monitor-data-collection-rule/README.md similarity index 100% rename from modules/monitor-data-collection/README.md rename to modules/monitor-data-collection-rule/README.md diff --git a/modules/monitor-data-collection/main.tf b/modules/monitor-data-collection-rule/main.tf similarity index 100% rename from modules/monitor-data-collection/main.tf rename to modules/monitor-data-collection-rule/main.tf diff --git a/modules/monitor-data-collection/outputs.tf b/modules/monitor-data-collection-rule/outputs.tf similarity index 100% rename from modules/monitor-data-collection/outputs.tf rename to modules/monitor-data-collection-rule/outputs.tf diff --git a/modules/monitor-data-collection/variables.tf b/modules/monitor-data-collection-rule/variables.tf similarity index 100% rename from modules/monitor-data-collection/variables.tf rename to modules/monitor-data-collection-rule/variables.tf From 0bed1754700ac4900658ff08cee1a1fb9306ffaa Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 12:27:59 +0530 Subject: [PATCH 4/9] XDR-4369: update variables description --- modules/monitor-data-collection-rule/variables.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/monitor-data-collection-rule/variables.tf b/modules/monitor-data-collection-rule/variables.tf index d709499..d5fbe3a 100644 --- a/modules/monitor-data-collection-rule/variables.tf +++ b/modules/monitor-data-collection-rule/variables.tf @@ -102,7 +102,7 @@ variable "data_sources" { streams = list(string) }))) }) - description = "A data_sources block as defined below." + description = "A data_sources block." default = {} # TODO: add variable validations } @@ -115,7 +115,7 @@ variable "data_flow" { output_stream = optional(string) transform_kql = optional(string) })) - description = "One or more data_flow blocks as defined below." + description = "One or more data_flow blocks." } variable "destinations" { @@ -155,7 +155,7 @@ variable "destinations" { storage_account_id = string }))) }) - description = "A destinations block as defined below" + description = "A destinations block." # TODO: add variable validations } @@ -168,7 +168,7 @@ variable "stream_declaration" { })) }) default = null - description = "A stream_declaration block as defined below." + description = "A stream_declaration block." } variable "identity" { @@ -177,5 +177,5 @@ variable "identity" { identity_ids = optional(list(string)) }) default = null - description = "An identity block as defined below." + description = "An identity block." } From d3811ffbefbcd929eda5fe2241abd66208ac462f Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 12:56:49 +0530 Subject: [PATCH 5/9] XDR-4369: update azure_monitor_metrics dynamic loop --- modules/monitor-data-collection-rule/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/monitor-data-collection-rule/main.tf b/modules/monitor-data-collection-rule/main.tf index 430ba04..cf049bc 100644 --- a/modules/monitor-data-collection-rule/main.tf +++ b/modules/monitor-data-collection-rule/main.tf @@ -139,7 +139,7 @@ resource "azurerm_monitor_data_collection_rule" "dcr" { destinations { dynamic "azure_monitor_metrics" { - for_each = [var.destinations["azure_monitor_metrics"]] != null ? [var.destinations["azure_monitor_metrics"]] : [] + for_each = var.destinations["azure_monitor_metrics"] != null ? [var.destinations["azure_monitor_metrics"]] : [] content { name = azure_monitor_metrics.value["name"] } From 9ed3953b395418455cb01c302c2170a029bf83b1 Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 13:04:26 +0530 Subject: [PATCH 6/9] XDR-4369: update variable kind default value --- modules/monitor-data-collection-rule/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/monitor-data-collection-rule/variables.tf b/modules/monitor-data-collection-rule/variables.tf index d5fbe3a..f0a09a2 100644 --- a/modules/monitor-data-collection-rule/variables.tf +++ b/modules/monitor-data-collection-rule/variables.tf @@ -21,7 +21,7 @@ variable "target_resource_id" { variable "kind" { type = string description = "The kind of the Data Collection Rule." - default = "" + default = null } variable "description" { From 46fa4db41201e97ed0acced5553cdcf6737c12bd Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 13:04:53 +0530 Subject: [PATCH 7/9] XDR-4369: update README.md --- modules/monitor-data-collection-rule/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/monitor-data-collection-rule/README.md b/modules/monitor-data-collection-rule/README.md index 6137754..97239ba 100644 --- a/modules/monitor-data-collection-rule/README.md +++ b/modules/monitor-data-collection-rule/README.md @@ -27,16 +27,16 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [data\_flow](#input\_data\_flow) | One or more data\_flow blocks as defined below. |
list(object({
destinations = list(string)
streams = list(string)
built_in_transform = optional(string)
output_stream = optional(string)
transform_kql = optional(string)
}))
| n/a | yes | -| [data\_sources](#input\_data\_sources) | A data\_sources block as defined below. |
object({
data_import = optional(object({
event_hub_data_source = object({
name = string
stream = string
consumer_group = optional(string)
})
}))
extension = optional(list(object({
extension_name = string
name = string
streams = list(string)
extension_json = optional(string)
input_data_sources = optional(list(string))
})))
iis_log = optional(list(object({
name = string
streams = list(string)
log_directories = optional(list(string))
})))
log_file = optional(list(object({
name = string
streams = list(string)
file_patterns = list(string)
format = string
settings = optional(object({
text = object({
record_start_timestamp_format = string
})
}))
})))
performance_counter = optional(list(object({
name = string
counter_specifiers = list(string)
sampling_frequency_in_seconds = number
streams = list(string)
})))
platform_telemetry = optional(list(object({
name = string
streams = list(string)
})))
prometheus_forwarder = optional(list(object({
name = string
streams = list(string)
label_include_filter = optional(list(object({
label = string
value = string
})))
})))
syslog = optional(list(object({
name = string
log_levels = list(string)
facility_names = list(string)
streams = optional(list(string)) # After provider version 4.0, this will be required.
})))
windows_event_log = optional(list(object({
name = string
streams = list(string)
x_path_queries = list(string)
})))
windows_firewall_log = optional(list(object({
name = string
streams = list(string)
})))
})
| `{}` | no | +| [data\_flow](#input\_data\_flow) | One or more data\_flow blocks. |
list(object({
destinations = list(string)
streams = list(string)
built_in_transform = optional(string)
output_stream = optional(string)
transform_kql = optional(string)
}))
| n/a | yes | +| [data\_sources](#input\_data\_sources) | A data\_sources block. |
object({
data_import = optional(object({
event_hub_data_source = object({
name = string
stream = string
consumer_group = optional(string)
})
}))
extension = optional(list(object({
extension_name = string
name = string
streams = list(string)
extension_json = optional(string)
input_data_sources = optional(list(string))
})))
iis_log = optional(list(object({
name = string
streams = list(string)
log_directories = optional(list(string))
})))
log_file = optional(list(object({
name = string
streams = list(string)
file_patterns = list(string)
format = string
settings = optional(object({
text = object({
record_start_timestamp_format = string
})
}))
})))
performance_counter = optional(list(object({
name = string
counter_specifiers = list(string)
sampling_frequency_in_seconds = number
streams = list(string)
})))
platform_telemetry = optional(list(object({
name = string
streams = list(string)
})))
prometheus_forwarder = optional(list(object({
name = string
streams = list(string)
label_include_filter = optional(list(object({
label = string
value = string
})))
})))
syslog = optional(list(object({
name = string
log_levels = list(string)
facility_names = list(string)
streams = optional(list(string)) # After provider version 4.0, this will be required.
})))
windows_event_log = optional(list(object({
name = string
streams = list(string)
x_path_queries = list(string)
})))
windows_firewall_log = optional(list(object({
name = string
streams = list(string)
})))
})
| `{}` | no | | [description](#input\_description) | The description of the Data Collection Rule | `string` | `""` | no | -| [destinations](#input\_destinations) | A destinations block as defined below |
object({
azure_monitor_metrics = optional(object({
name = string
}))
event_hub = optional(list(object({
name = string
event_hub_id = string
})))
event_hub_direct = optional(list(object({
name = string
event_hub_id = string
})))
log_analytics = optional(list(object({
name = string
workspace_resource_id = string
})))
monitor_account = optional(list(object({
name = string
monitor_account_id = string
})))
storage_blob = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_blob_direct = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_table_direct = optional(list(object({
name = string
table_name = string
storage_account_id = string
})))
})
| n/a | yes | -| [identity](#input\_identity) | An identity block as defined below. |
object({
type = string
identity_ids = optional(list(string))
})
| `null` | no | -| [kind](#input\_kind) | The kind of the Data Collection Rule. | `string` | `""` | no | +| [destinations](#input\_destinations) | A destinations block. |
object({
azure_monitor_metrics = optional(object({
name = string
}))
event_hub = optional(list(object({
name = string
event_hub_id = string
})))
event_hub_direct = optional(list(object({
name = string
event_hub_id = string
})))
log_analytics = optional(list(object({
name = string
workspace_resource_id = string
})))
monitor_account = optional(list(object({
name = string
monitor_account_id = string
})))
storage_blob = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_blob_direct = optional(list(object({
name = string
storage_account_id = string
container_name = string
})))
storage_table_direct = optional(list(object({
name = string
table_name = string
storage_account_id = string
})))
})
| n/a | yes | +| [identity](#input\_identity) | An identity block. |
object({
type = string
identity_ids = optional(list(string))
})
| `null` | no | +| [kind](#input\_kind) | The kind of the Data Collection Rule. | `string` | `null` | no | | [location](#input\_location) | The Azure Region where the Data Collection Rule should exist | `string` | n/a | yes | | [name](#input\_name) | The name which should be used for this Data Collection Rule | `string` | n/a | yes | | [resource\_group\_name](#input\_resource\_group\_name) | The name of the Resource Group where the Data Collection Rule should exist | `string` | n/a | yes | -| [stream\_declaration](#input\_stream\_declaration) | A stream\_declaration block as defined below. |
object({
stream_name = string
column = list(object({
name = string
type = string
}))
})
| `null` | no | +| [stream\_declaration](#input\_stream\_declaration) | A stream\_declaration block. |
object({
stream_name = string
column = list(object({
name = string
type = string
}))
})
| `null` | no | | [tags](#input\_tags) | A mapping of tags to assign to the resource | `map(string)` | `{}` | no | | [target\_resource\_id](#input\_target\_resource\_id) | The ID of the Azure Resource which to associate to a Data Collection Rule or a Data Collection Endpoint. | `string` | n/a | yes | From 6765d686ea4562c65ba4b1c7a2d50c7c13bbfb7e Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Fri, 28 Apr 2023 13:28:34 +0530 Subject: [PATCH 8/9] XDR-4369: replace association with AZ CLI --- .../monitor-data-collection-rule/README.md | 3 +- modules/monitor-data-collection-rule/main.tf | 28 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/modules/monitor-data-collection-rule/README.md b/modules/monitor-data-collection-rule/README.md index 97239ba..e47c88b 100644 --- a/modules/monitor-data-collection-rule/README.md +++ b/modules/monitor-data-collection-rule/README.md @@ -11,6 +11,7 @@ | Name | Version | |------|---------| | [azurerm](#provider\_azurerm) | 3.53.0 | +| [null](#provider\_null) | n/a | ## Modules @@ -21,7 +22,7 @@ No modules. | Name | Type | |------|------| | [azurerm_monitor_data_collection_rule.dcr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_data_collection_rule) | resource | -| [azurerm_monitor_data_collection_rule_association.dcr_association](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_data_collection_rule_association) | resource | +| [null_resource.dcr_association](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | ## Inputs diff --git a/modules/monitor-data-collection-rule/main.tf b/modules/monitor-data-collection-rule/main.tf index cf049bc..f787e2d 100644 --- a/modules/monitor-data-collection-rule/main.tf +++ b/modules/monitor-data-collection-rule/main.tf @@ -8,6 +8,10 @@ terraform { source = "hashicorp/azurerm" version = "~> 3.53" } + null = { + source = "hashicorp/null" + version = "~> 3.2" + } } required_version = ">= 0.12" experiments = [module_variable_optional_attrs] @@ -221,9 +225,23 @@ resource "azurerm_monitor_data_collection_rule" "dcr" { } } -resource "azurerm_monitor_data_collection_rule_association" "dcr_association" { - name = "${var.name}-association" - description = var.description - target_resource_id = var.target_resource_id - data_collection_rule_id = azurerm_monitor_data_collection_rule.dcr.id +# azurerm_monitor_data_collection_rule_association doesn't support resource of type +# 'Microsoft.OperationalInsights/workspaces' +# https://github.com/hashicorp/terraform-provider-azurerm/issues/20637 + +#resource "azurerm_monitor_data_collection_rule_association" "dcr_association" { +# name = "${var.name}-association" +# description = var.description +# target_resource_id = var.target_resource_id +# data_collection_rule_id = azurerm_monitor_data_collection_rule.dcr.id +#} + +# TODO: add validations for resource_group_name and target_resource_id to prevent potential CLI injection. +resource "null_resource" "dcr_association" { + provisioner "local-exec" { + command = "az monitor log-analytics workspace update --resource-group ${var.resource_group_name} --workspace-name ${var.target_resource_id} --data-collection-rule \"${azurerm_monitor_data_collection_rule.dcr.id}\"" + } + depends_on = [ + azurerm_monitor_data_collection_rule.dcr + ] } From c27e97b6583ac95b7dbcaa42ff7e963255ccdcc7 Mon Sep 17 00:00:00 2001 From: Chris Jaimon Date: Wed, 3 May 2023 00:57:27 +0530 Subject: [PATCH 9/9] XDR-4369: add description to null_resource --- modules/monitor-data-collection-rule/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/monitor-data-collection-rule/main.tf b/modules/monitor-data-collection-rule/main.tf index f787e2d..71912fe 100644 --- a/modules/monitor-data-collection-rule/main.tf +++ b/modules/monitor-data-collection-rule/main.tf @@ -237,6 +237,8 @@ resource "azurerm_monitor_data_collection_rule" "dcr" { #} # TODO: add validations for resource_group_name and target_resource_id to prevent potential CLI injection. +# DCR needs to be linked to a Log Analytics workspace +# https://learn.microsoft.com/en-gb/azure/azure-monitor/logs/tutorial-workspace-transformations-api#link-workspace-to-dcr resource "null_resource" "dcr_association" { provisioner "local-exec" { command = "az monitor log-analytics workspace update --resource-group ${var.resource_group_name} --workspace-name ${var.target_resource_id} --data-collection-rule \"${azurerm_monitor_data_collection_rule.dcr.id}\""