JamesDLD
commented
3 weeks ago Hi @matt-FFFFFF thank you for this initiative! If it can help I am sharing here a code I used that can help to create the 3 Data Collection Rule:
We could have something link this in the file resources.management.tf
resource "azurerm_monitor_data_collection_rule" "management" {
for_each = local.azurerm_monitor_data_collection_rule_management
name = each.value.user_given_dcr_name
location = each.value.workspace_location
resource_group_name = each.value.resource_group_name
description = each.value.description
dynamic "data_sources" {
for_each = lookup(each.value, "data_sources", [])
content {
dynamic "data_import" {
for_each = lookup(data_sources.value, "data_import", [])
content {
dynamic "event_hub_data_source" {
for_each = lookup(data_import.value, "event_hub_data_source", [])
content {
name = event_hub_data_source.value.name
stream = event_hub_data_source.value.stream
consumer_group = lookup(event_hub_data_source.value, "consumer_group", null)
}
}
}
}
dynamic "extension" {
for_each = lookup(data_sources.value, "extension", [])
content {
extension_name = extension.value.extension_name
name = extension.value.name
streams = extension.value.streams #(Required) Specifies a list of streams that this data source will be sent to. A stream indicates what schema will be used for this data and usually what table in Log Analytics the data will be sent to. Possible values include but not limited to Microsoft-Event, Microsoft-InsightsMetrics, Microsoft-Perf, Microsoft-Syslog, Microsoft-WindowsEvent.
extension_json = lookup(extension.value, "extension_json", null) #(Optional) A JSON String which specifies the extension setting.
input_data_sources = lookup(extension.value, "input_data_sources", null) #(Optional) Specifies a list of data sources this extension needs data from. An item should be a name of a supported data source which produces only one stream. Supported data sources type: performance_counter, windows_event_log,and syslog.
}
}
dynamic "iis_log" {
for_each = lookup(data_sources.value, "iis_log", [])
content {
name = iis_log.value.name
streams = iis_log.value.streams
log_directories = lookup(iis_log.value, "log_directories", null)
}
}
dynamic "log_file" {
for_each = lookup(data_sources.value, "log_file", [])
content {
name = log_file.value.name
streams = log_file.value.streams
file_patterns = log_file.value.file_patterns
format = log_file.value.format
dynamic "settings" {
for_each = lookup(log_file.value, "settings", [])
content {
dynamic "text" {
for_each = lookup(settings.value, "text", [])
content {
record_start_timestamp_format = text.value.record_start_timestamp_format #(Required) The timestamp format of the text log files. Possible values are ISO 8601, YYYY-MM-DD HH:MM:SS, M/D/YYYY HH:MM:SS AM/PM, Mon DD, YYYY HH:MM:SS, yyMMdd HH:mm:ss, ddMMyy HH:mm:ss, MMM d hh:mm:ss, dd/MMM/yyyy:HH:mm:ss zzz,and yyyy-MM-ddTHH:mm:ssK.
}
}
}
}
}
}
dynamic "performance_counter" {
for_each = lookup(data_sources.value, "performance_counter", [])
content {
counter_specifiers = performance_counter.value.counter_specifiers
name = performance_counter.value.name
sampling_frequency_in_seconds = performance_counter.value.sampling_frequency_in_seconds
streams = performance_counter.value.streams
}
}
dynamic "platform_telemetry" {
for_each = lookup(data_sources.value, "platform_telemetry", [])
content {
name = platform_telemetry.value.name
streams = platform_telemetry.value.streams
}
}
dynamic "prometheus_forwarder" {
for_each = lookup(data_sources.value, "prometheus_forwarder", [])
content {
name = prometheus_forwarder.value.name
streams = prometheus_forwarder.value.streams
dynamic "label_include_filter" {
for_each = lookup(prometheus_forwarder.value, "label_include_filter", [])
content {
label = label_include_filter.value.label
value = label_include_filter.value.value
}
}
}
}
dynamic "syslog" {
for_each = lookup(data_sources.value, "syslog", [])
content {
facility_names = syslog.value.facility_names
log_levels = syslog.value.log_levels
name = syslog.value.name
streams = lookup(syslog.value, "streams", null)
}
}
dynamic "windows_event_log" {
for_each = lookup(data_sources.value, "windows_event_log", [])
content {
name = windows_event_log.value.name
streams = windows_event_log.value.streams
x_path_queries = windows_event_log.value.x_path_queries
}
}
dynamic "windows_firewall_log" {
for_each = lookup(data_sources.value, "windows_firewall_log", [])
content {
name = windows_firewall_log.value.name
streams = windows_firewall_log.value.streams
}
}
}
}
dynamic "destinations" {
for_each = lookup(each.value, "destinations", [])
content {
dynamic "azure_monitor_metrics" {
for_each = lookup(destinations.value, "azure_monitor_metrics", [])
content {
name = azure_monitor_metrics.value.name
}
}
dynamic "event_hub" {
for_each = lookup(destinations.value, "event_hub", [])
content {
event_hub_id = event_hub.value.event_hub_id #(Required) The resource ID of the Event Hub.
name = event_hub.value.name #(Required) The name which should be used for this destination. This name should be unique across all destinations regardless of type within the Data Collection Rule.
}
}
dynamic "event_hub_direct" {
for_each = lookup(destinations.value, "event_hub_direct", [])
content {
event_hub_id = event_hub_direct.value.event_hub_id
name = event_hub_direct.value.name
}
}
dynamic "log_analytics" {
for_each = lookup(destinations.value, "log_analytics", [])
content {
workspace_resource_id = log_analytics.value.workspace_resource_id
name = log_analytics.value.name
}
}
dynamic "monitor_account" {
for_each = lookup(destinations.value, "monitor_account", [])
content {
monitor_account_id = monitor_account.value.monitor_account_id
name = monitor_account.value.name
}
}
dynamic "storage_blob" {
for_each = lookup(destinations.value, "storage_blob", [])
content {
container_name = storage_blob.value.container_name
name = storage_blob.value.name
storage_account_id = storage_blob.value.storage_account_id
}
}
dynamic "storage_blob_direct" {
for_each = lookup(destinations.value, "storage_blob_direct", [])
content {
container_name = storage_blob_direct.value.container_name
name = storage_blob_direct.value.name
storage_account_id = storage_blob_direct.value.storage_account_id
}
}
dynamic "storage_table_direct" {
for_each = lookup(destinations.value, "storage_table_direct", [])
content {
table_name = storage_table_direct.value.table_name
name = storage_table_direct.value.name
storage_account_id = storage_table_direct.value.storage_account_id
}
}
}
}
dynamic "data_flow" {
for_each = lookup(each.value, "data_flow", [])
content {
streams = data_flow.value.streams #(Required) Specifies a list of streams. Possible values include but not limited to Microsoft-Event, Microsoft-InsightsMetrics, Microsoft-Perf, Microsoft-Syslog,and Microsoft-WindowsEvent.
destinations = data_flow.value.destinations #(Required) Specifies a list of destination names. A azure_monitor_metrics data source only allows for stream of kind Microsoft-InsightsMetrics.
built_in_transform = lookup(data_flow.value, "built_in_transform", null) #(Optional) The built-in transform to transform stream data.
output_stream = lookup(data_flow.value, "output_stream", null) #(Optional) The output stream of the transform. Only required if the data flow changes data to a different stream.
transform_kql = lookup(data_flow.value, "transform_kql", null) #(Optional) The KQL query to transform stream data.
}
}
dynamic "identity" {
for_each = lookup(each.value, "identity", [])
content {
type = lookup(identity.value, "type", null)
identity_ids = lookup(identity.value, "identity_id_key", null) == null ? null : [azurerm_user_assigned_identity.id[identity.value.identity_id_key].id]
}
}
dynamic "stream_declaration" {
for_each = lookup(each.value, "stream_declaration", [])
content {
stream_name = stream_declaration.value.stream_name
dynamic "column" {
for_each = lookup(stream_declaration.value, "column", [])
content {
name = column.value.name
type = column.value.type
}
}
}
}
kind = lookup(each.value, "kind", null) #(Optional) The kind of the Data Collection Rule. Possible values are Linux, Windows,and AgentDirectToStore. A rule of kind Linux does not allow for windows_event_log data sources. And a rule of kind Windows does not allow for syslog data sources. If kind is not specified, all kinds of data sources are allowed.
tags = lookup(each.value, "tags", null)
}And something link this in the file locals.management.tf
# The following locals are used to build the map of Data
# Collection Rules to deploy.
locals {
management_workspace_resource_id = "/subscriptions/${var.subscription_id_management}/resourceGroups/${local.archetypes.configure_management_resources.advanced.custom_settings_by_resource_type.azurerm_resource_group.management.name}/providers/Microsoft.OperationalInsights/workspaces/${local.archetypes.configure_management_resources.advanced.custom_settings_by_resource_type.azurerm_log_analytics_workspace.management.name}"
azurerm_monitor_data_collection_rule_management = {
VmInsights = {
user_given_dcr_name = "dcr-vminsights-001"
workspace_location = "francecentral"
resource_group_name = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].resource_group_name
description = "Data collection rule for VM Insights."
tags = local.archetypes.default_tags
data_sources = [
{
performance_counter = [
{
streams = ["Microsoft-InsightsMetrics"]
sampling_frequency_in_seconds = 60
counter_specifiers = [
"\\VmInsights\\DetailedMetrics"
]
name = "VMInsightsPerfCounters"
}
]
extension = [
{
streams = ["Microsoft-ServiceMap"]
extension_name = "DependencyAgent"
name = "DependencyAgentDataSource"
}
]
}
]
destinations = [
{
log_analytics = [
{
workspace_resource_id = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].id
name = "VMInsightsPerf-Logs-Dest"
}
]
}
]
data_flow = [
{
streams = ["Microsoft-InsightsMetrics"]
destinations = ["VMInsightsPerf-Logs-Dest"]
},
{
streams = ["Microsoft-ServiceMap"]
destinations = ["VMInsightsPerf-Logs-Dest"]
}
]
}
ChangeTracking = {
user_given_dcr_name = "dcr-changetracking-001"
workspace_location = "francecentral"
resource_group_name = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].resource_group_name
description = "Data collection rule for CT."
tags = local.archetypes.default_tags
data_sources = [
{
extension = [
{
streams = [
"Microsoft-ConfigurationChange",
"Microsoft-ConfigurationChangeV2",
"Microsoft-ConfigurationData"
]
extension_name = "ChangeTracking-Windows"
extension_json = jsonencode({
enableFiles = true,
enableSoftware = true,
enableRegistry = false,
enableServices = true,
enableInventory = true,
fileSettings = {
fileCollectionFrequency = 900,
fileInfo = [
{
name = "ChangeTrackingLinuxPath_default",
enabled = true,
destinationPath = "/etc/.*.conf",
useSudo = true,
recurse = true,
maxContentsReturnable = 5000000,
pathType = "File",
type = "File",
links = "Follow",
maxOutputSize = 500000,
groupTag = "Recommended"
}
]
},
softwareSettings = {
softwareCollectionFrequency = 300
},
inventorySettings = {
inventoryCollectionFrequency = 36000
},
servicesSettings = {
serviceCollectionFrequency = 300
}
})
name = "CTDataSource-Windows"
},
{
streams = [
"Microsoft-ConfigurationChange",
"Microsoft-ConfigurationChangeV2",
"Microsoft-ConfigurationData"
]
extension_name = "ChangeTracking-Linux"
extension_json = jsonencode({
enableFiles = true,
enableSoftware = true,
enableRegistry = false,
enableServices = true,
enableInventory = true,
fileSettings = {
fileCollectionFrequency = 900,
fileInfo = [
{
name = "ChangeTrackingLinuxPath_default",
enabled = true,
destinationPath = "/etc/.*.conf",
useSudo = true,
recurse = true,
maxContentsReturnable = 5000000,
pathType = "File",
type = "File",
links = "Follow",
maxOutputSize = 500000,
groupTag = "Recommended"
}
]
},
softwareSettings = {
softwareCollectionFrequency = 300
},
inventorySettings = {
inventoryCollectionFrequency = 36000
},
servicesSettings = {
serviceCollectionFrequency = 300
}
})
name = "CTDataSource-Linux"
}
]
}
]
destinations = [
{
log_analytics = [
{
workspace_resource_id = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].id
name = "Microsoft-CT-Dest"
}
]
}
]
data_flow = [
{
streams = [
"Microsoft-ConfigurationChange",
"Microsoft-ConfigurationChangeV2",
"Microsoft-ConfigurationData"
]
destinations = ["Microsoft-CT-Dest"]
}
]
}
DefenderSQL = {
user_given_dcr_name = "dcr-defendersql-001"
workspace_location = "francecentral"
resource_group_name = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].resource_group_name
description = "Data collection rule for Defender for SQL."
tags = local.archetypes.default_tags
data_sources = [
{
extension = [
{
streams = [
"Microsoft-DefenderForSqlAlerts",
"Microsoft-DefenderForSqlLogins",
"Microsoft-DefenderForSqlTelemetry",
"Microsoft-DefenderForSqlScanEvents",
"Microsoft-DefenderForSqlScanResults"
]
extension_name = "MicrosoftDefenderForSQL"
extension_json = jsonencode({
enableCollectionOfSqlQueriesForSecurityResearch = false
})
name = "MicrosoftDefenderForSQL"
}
]
}
]
destinations = [
{
log_analytics = [
{
workspace_resource_id = module.enterprise_scale[0].azurerm_log_analytics_workspace.management[local.management_workspace_resource_id].id
name = "LogAnalyticsDest"
}
]
}
]
data_flow = [
{
streams = [
"Microsoft-DefenderForSqlAlerts",
"Microsoft-DefenderForSqlLogins",
"Microsoft-DefenderForSqlTelemetry",
"Microsoft-DefenderForSqlScanEvents",
"Microsoft-DefenderForSqlScanResults"
]
destinations = ["LogAnalyticsDest"]
}
]
}
}
}
matt-FFFFFF
azure-pipelines[bot]
Overview/Summary
This will be in the next major release, following the update of Azure Landing Zones with it's major policy refresh and move to Azure Monitoring Agent from Microsoft Monitoring Agent.
Incorporates the following changes from upstream
Changes from our awesome community
918 (thanks @chrsundermann!)
925 (thanks @nyanhp!)
952 (thanks @Keetika-Yogendra!)
‼️ Breaking Changes
3.107.01.13.11.7.0var.configure_management_resourcesschema change, removing legacy components and adding support for AMA resourcesAcknowledgements
Thanks to:
Thanks to:
Testing Evidence
Please provide any testing evidence to show that your Pull Request works/fixes as described and planned (include screenshots, if appropriate).
As part of this Pull Request I have
mainbranch