duplicate key on private dns zones when upgrading Bug Report

davidkarlsen commented 1 year ago

Community Note

Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request
If you are interested in working on this issue or have submitted a pull request, please leave a comment

Versions

terraform:

azure provider: 3.37

module: 3.1.1

Description

fails on "produces duplicate key"

Describe the bug

config:

module "caf-enterprise-scale" {
  source  = "Azure/caf-enterprise-scale/azurerm"
  version = "2.4.1"
  providers = {
    azurerm              = azurerm
    azurerm.connectivity = azurerm
    azurerm.management   = azurerm
  }

  root_parent_id                = data.azurerm_client_config.core.tenant_id
  subscription_id_management    = var.subscription_id_management
  subscription_id_connectivity  = var.subscription_id_connectivity
  deploy_core_landing_zones     = false
  deploy_connectivity_resources = true
  default_tags                  = var.default_tags
  default_location              = var.default_location
  configure_connectivity_resources = {
    advanced = {
      custom_settings_by_resource_type = {
        azurerm_firewall = {
          connectivity = {
            (var.default_location) = {
              firewall_policy_id = azurerm_firewall_policy.firewall_policy.id
            }
          }
        }
        azurerm_virtual_network_gateway = {
          connectivity_vpn = {
            (var.default_location) = {
              ip_configuration = [
                {
                  name                          = "vnetGatewayConfig1"
                  public_ip_address_id          = azurerm_public_ip.vpn_gw[0].id
                  private_ip_address_allocation = "Dynamic"
                  subnet_id                     = data.azurerm_subnet.gateway_subnet.id
                },
                {
                  name                          = "vnetGatewayConfig2"
                  public_ip_address_id          = azurerm_public_ip.vpn_gw[1].id
                  private_ip_address_allocation = "Dynamic"
                  subnet_id                     = data.azurerm_subnet.gateway_subnet.id
                },
                {
                  name                          = "vnetGatewayConfig3"
                  public_ip_address_id          = azurerm_public_ip.vpn_gw[2].id
                  private_ip_address_allocation = "Dynamic"
                  subnet_id                     = data.azurerm_subnet.gateway_subnet.id
                }
              ]
            }
          }
        }
        azurerm_subnet = {
          connectivity = {
            (var.default_location) = {
              "ApplicationGatewaySubnet" = {
                service_endpoints = ["Microsoft.Storage"]
              }
            }
          }
        }
      }
    }
    settings = {
      hub_networks = [
        {
          enabled = true
          config = {
            address_space                   = var.hub_network_address_space
            location                        = var.default_location
            link_to_ddos_protection_plan    = false
            dns_servers                     = []
            bgp_community                   = ""
            enable_hub_network_mesh_peering = false
            subnets = [
              {
                name                      = "AzureBastionSubnet"
                address_prefixes          = ["10.170.64.128/26"]
                network_security_group_id = null
                route_table_id            = null
              },
              {
                name                      = "ApplicationGatewaySubnet"
                address_prefixes          = ["10.170.65.0/24"]
                network_security_group_id = null
                route_table_id            = null
              },
              {
                name                      = "APIManagementSubnet"
                address_prefixes          = [var.api_management_subnet]
                network_security_group_id = null
                route_table_id            = null

              }
            ]
            virtual_network_gateway = {
              enabled = true
              config = {
                address_prefix           = var.virtual_network_gateway_range
                gateway_sku_expressroute = ""
                gateway_sku_vpn          = var.vpn_gw_sku
                advanced_vpn_settings = {
                  enable_bgp                       = true
                  active_active                    = true
                  private_ip_address_allocation    = ""
                  default_local_network_gateway_id = ""
                  bgp_settings = [{
                    asn         = var.az_bgp_asn
                    peer_weight = 32768
                    peering_addresses = [
                      { ip_configuration_name = "vnetGatewayConfig1", apipa_addresses = null },
                      { ip_configuration_name = "vnetGatewayConfig2", apipa_addresses = null }
                    ]
                  }]
                  vpn_client_configuration = [{
                    address_space         = [var.client_vpn_ipaddress_range]
                    vpn_auth_types        = ["AAD"]
                    vpn_client_protocols  = ["OpenVPN"]
                    aad_issuer            = "https://sts.windows.net/${data.azurerm_client_config.core.tenant_id}/"
                    aad_audience          = var.client_vpn_audience
                    aad_tenant            = "https://login.microsoftonline.com/${data.azurerm_client_config.core.tenant_id}"
                    radius_server_address = null
                    radius_server_secret  = null
                    revoked_certificate   = []
                    root_certificate      = []
                  }]
                  custom_route = []
                }
              }
            }
            azure_firewall = {
              enabled = true
              config = {
                address_prefix                = var.azure_firewall_range
                sku_tier                      = "Standard"
                private_ip_ranges             = []
                threat_intelligence_mode      = "Deny"
                threat_intelligence_allowlist = []
                base_policy_id                = azurerm_firewall_policy.firewall_policy.id
                dns_servers                   = []
                enable_dns_proxy              = true
                availability_zones = {
                  zone_1 = true
                  zone_2 = true
                  zone_3 = true
                }
              }
            }
            spoke_virtual_network_resource_ids = [
              "/subscriptions/${var.lz1.subscription_id}/resourceGroups/${var.lz1.spoke_vnet_resouce_group}/providers/Microsoft.Network/virtualNetworks/${var.lz1.spoke_vnet_name}",
              "/subscriptions/${var.lz2.subscription_id}/resourceGroups/${var.lz2.spoke_vnet_resouce_group}/providers/Microsoft.Network/virtualNetworks/${var.lz2.spoke_vnet_name}"
            ]
            enable_outbound_virtual_network_peering = true
          }
        }
      ]
      vwan_hub_networks = []
      ddos_protection_plan = {
        enabled = false
        config = {
          location = var.default_location
        }
      }
      dns = {
        enabled = true
        config = {
          location = null
          enable_private_link_by_service = {
            azure_automation_webhook             = true
            azure_automation_dscandhybridworker  = true
            azure_sql_database_sqlserver         = true
            azure_synapse_analytics_sqlserver    = true
            azure_synapse_analytics_sql          = true
            storage_account_blob                 = true
            storage_account_table                = true
            storage_account_queue                = true
            storage_account_file                 = true
            storage_account_web                  = true
            azure_data_lake_file_system_gen2     = true
            azure_cosmos_db_sql                  = true
            azure_cosmos_db_mongodb              = true
            azure_cosmos_db_cassandra            = true
            azure_cosmos_db_gremlin              = true
            azure_cosmos_db_table                = true
            azure_database_for_postgresql_server = true
            azure_database_for_mysql_server      = true
            azure_database_for_mariadb_server    = true
            azure_key_vault                      = true
            azure_kubernetes_service_management  = true
            azure_search_service                 = true
            azure_container_registry             = true
            azure_app_configuration_stores       = true
            azure_backup                         = true
            azure_site_recovery                  = true
            azure_event_hubs_namespace           = true
            azure_service_bus_namespace          = true
            azure_iot_hub                        = true
            azure_relay_namespace                = true
            azure_event_grid_topic               = true
            azure_event_grid_domain              = true
            azure_web_apps_sites                 = true
            azure_machine_learning_workspace     = true
            signalr                              = true
            azure_monitor                        = true
            cognitive_services_account           = true
            azure_file_sync                      = true
            azure_data_factory                   = true
            azure_data_factory_portal            = true
            azure_cache_for_redis                = true
          }
          private_link_locations                                 = []
          public_dns_zones                                       = ["dev.az.finods.com"]
          private_dns_zones                                      = ["appserviceenvironment.net", "privatelink.norwayeast.kusto.windows.net"]
          enable_private_dns_zone_virtual_network_link_on_hubs   = true
          enable_private_dns_zone_virtual_network_link_on_spokes = true
        }
      }
    }

    location = var.default_location
    tags     = var.default_tags
  }
}

2022-12-29T18:55:08.2205265Z Error: Duplicate object key
2022-12-29T18:55:08.2206015Z 
2022-12-29T18:55:08.2206715Z   on .terraform/modules/caf-enterprise-scale/locals.connectivity.tf line 89, in locals:
2022-12-29T18:55:08.2207411Z   87:   azurerm_private_dns_zone_connectivity = {
2022-12-29T18:55:08.2208121Z   88:     for resource in module.connectivity_resources.configuration.azurerm_private_dns_zone :
2022-12-29T18:55:08.2208756Z   89:     resource.resource_id => resource
2022-12-29T18:55:08.2209227Z   90:     if resource.managed_by_module
2022-12-29T18:55:08.2209747Z   91:   }
2022-12-29T18:55:08.2210651Z     ├────────────────
2022-12-29T18:55:08.2211867Z     │ resource.resource_id is "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net"
2022-12-29T18:55:08.2212647Z 
2022-12-29T18:55:08.2212932Z Two different items produced the key
2022-12-29T18:55:08.2214007Z "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net"
2022-12-29T18:55:08.2214963Z in this 'for' expression. If duplicates are expected, use the ellipsis (...)
2022-12-29T18:55:08.2215551Z after the value expression to enable grouping by key.
2022-12-29T18:55:08.2230049Z 
2022-12-29T18:55:08.2230369Z Error: Duplicate object key
2022-12-29T18:55:08.2230640Z 
2022-12-29T18:55:08.2231319Z   on .terraform/modules/caf-enterprise-scale/locals.connectivity.tf line 109, in locals:
2022-12-29T18:55:08.2232268Z  107:   azurerm_private_dns_zone_virtual_network_link_connectivity = {
2022-12-29T18:55:08.2232989Z  108:     for resource in module.connectivity_resources.configuration.azurerm_private_dns_zone_virtual_network_link :
2022-12-29T18:55:08.2233893Z  109:     resource.resource_id => resource
2022-12-29T18:55:08.2234538Z  110:     if resource.managed_by_module
2022-12-29T18:55:08.2235104Z  111:   }
2022-12-29T18:55:08.2235660Z     ├────────────────
2022-12-29T18:55:08.2237117Z     │ resource.resource_id is "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/***-07feea57-e490-5a21-95e3-0e86d33dd7b8"
2022-12-29T18:55:08.2238039Z 
2022-12-29T18:55:08.2238354Z Two different items produced the key
2022-12-29T18:55:08.2239620Z "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/***-07feea57-e490-5a21-95e3-0e86d33dd7b8"
2022-12-29T18:55:08.2240563Z in this 'for' expression. If duplicates are expected, use the ellipsis (...)
2022-12-29T18:55:08.2241173Z after the value expression to enable grouping by key.
2022-12-29T18:55:08.2259264Z 
2022-12-29T18:55:08.2259665Z Error: Duplicate object key
2022-12-29T18:55:08.2259950Z 
2022-12-29T18:55:08.2260519Z   on .terraform/modules/caf-enterprise-scale/locals.connectivity.tf line 109, in locals:
2022-12-29T18:55:08.2261278Z  107:   azurerm_private_dns_zone_virtual_network_link_connectivity = {
2022-12-29T18:55:08.2261967Z  108:     for resource in module.connectivity_resources.configuration.azurerm_private_dns_zone_virtual_network_link :
2022-12-29T18:55:08.2263346Z  109:     resource.resource_id => resource
2022-12-29T18:55:08.2263933Z  110:     if resource.managed_by_module
2022-12-29T18:55:08.2264333Z  111:   }
2022-12-29T18:55:08.2264819Z     ├────────────────
2022-12-29T18:55:08.2266188Z     │ resource.resource_id is "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/b302aea5-2b20-4b9a-a223-314c4b2bafab-e8214c97-d31a-54c9-893f-cd903b75061d"
2022-12-29T18:55:08.2267081Z 
2022-12-29T18:55:08.2267283Z Two different items produced the key
2022-12-29T18:55:08.2268566Z "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/b302aea5-2b20-4b9a-a223-314c4b2bafab-e8214c97-d31a-54c9-893f-cd903b75061d"
2022-12-29T18:55:08.2272610Z in this 'for' expression. If duplicates are expected, use the ellipsis (...)
2022-12-29T18:55:08.2273553Z after the value expression to enable grouping by key.
2022-12-29T18:55:08.2273892Z 
2022-12-29T18:55:08.2274073Z Error: Duplicate object key
2022-12-29T18:55:08.2274323Z 
2022-12-29T18:55:08.2274778Z   on .terraform/modules/caf-enterprise-scale/locals.connectivity.tf line 109, in locals:
2022-12-29T18:55:08.2275412Z  107:   azurerm_private_dns_zone_virtual_network_link_connectivity = {
2022-12-29T18:55:08.2276143Z  108:     for resource in module.connectivity_resources.configuration.azurerm_private_dns_zone_virtual_network_link :
2022-12-29T18:55:08.2276816Z  109:     resource.resource_id => resource
2022-12-29T18:55:08.2277471Z  110:     if resource.managed_by_module
2022-12-29T18:55:08.2277918Z  111:   }
2022-12-29T18:55:08.2278605Z     ├────────────────
2022-12-29T18:55:08.2279971Z     │ resource.resource_id is "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/3a6047b6-ce51-41ec-b86b-f42ed9ef2016-bc77e959-fcf1-5ade-a62c-fefe59b99967"
2022-12-29T18:55:08.2280762Z 
2022-12-29T18:55:08.2280966Z Two different items produced the key
2022-12-29T18:55:08.2282292Z "/subscriptions/***/resourceGroups/es-dns/providers/Microsoft.Network/privateDnsZones/privatelink.norwayeast.kusto.windows.net/virtualNetworkLinks/3a6047b6-ce51-41ec-b86b-f42ed9ef2016-bc77e959-fcf1-5ade-a62c-fefe59b99967"
2022-12-29T18:55:08.2283329Z in this 'for' expression. If duplicates are expected, use the ellipsis (...)
2022-12-29T18:55:08.2283954Z after the value expression to enable grouping by key.
2022-12-29T18:55:08.2332518Z 
2022-12-29T18:55:08.2332946Z Error: Not enough list items
2022-12-29T18:55:08.2333234Z 
2022-12-29T18:55:08.2334356Z   with module.caf-enterprise-scale.azurerm_virtual_network_gateway.connectivity["/subscriptions/***/resourceGroups/es-connectivity-norwayeast/providers/Microsoft.Network/virtualNetworkGateways/es-vpngw-norwayeast"],
2022-12-29T18:55:08.2335662Z   on .terraform/modules/caf-enterprise-scale/resources.connectivity.tf line 138, in resource "azurerm_virtual_network_gateway" "connectivity":
2022-12-29T18:55:08.2336399Z  138: resource "azurerm_virtual_network_gateway" "connectivity" {
2022-12-29T18:55:08.2336760Z 
2022-12-29T18:55:08.2337070Z Attribute bgp_settings.0.peering_addresses.0.apipa_addresses requires 1 item
2022-12-29T18:55:08.2337613Z minimum, but config has only 0 declared.
2022-12-29T18:55:08.2390647Z 
2022-12-29T18:55:08.2391070Z Error: Not enough list items
2022-12-29T18:55:08.2391440Z 
2022-12-29T18:55:08.2392659Z   with module.caf-enterprise-scale.azurerm_virtual_network_gateway.connectivity["/subscriptions/***/resourceGroups/es-connectivity-norwayeast/providers/Microsoft.Network/virtualNetworkGateways/es-vpngw-norwayeast"],
2022-12-29T18:55:08.2393909Z   on .terraform/modules/caf-enterprise-scale/resources.connectivity.tf line 138, in resource "azurerm_virtual_network_gateway" "connectivity":
2022-12-29T18:55:08.2394629Z  138: resource "azurerm_virtual_network_gateway" "connectivity" {
2022-12-29T18:55:08.2395012Z 
2022-12-29T18:55:08.2395359Z Attribute bgp_settings.0.peering_addresses.1.apipa_addresses requires 1 item
2022-12-29T18:55:08.2395948Z minimum, but config has only 0 declared.
2022-12-29T18:55:08.6406269Z Releasing state lock. This may take a few moments...
2022-12-29T18:55:08.6934611Z ##[error]Terraform exited with code 1.
2022-12-29T18:55:08.6951689Z ##[error]Process completed with exit code 1.

Steps to Reproduce

Upgrade from 2.4.1 to 3.1.1

Screenshots

See bug-report.

Additional context

krowlandson commented 1 year ago

Thank you for reporting this @davidkarlsen... we didn't observe this during testing but will see whether we can reproduce.

As part of the 3.x releases we have made a number of updates to the DNS zones managed by the module. I notice you are specifying the duplicate key privatelink.norwayeast.kusto.windows.net in private_dns_zones.

Does this error still occur if you remove this from your config?

We automatically de-duplicate the managed zones, but not user-specified ones. We can certainly consider adding this capability.

davidkarlsen commented 1 year ago

@krowlandson It's in the list because we want the private zone for this service too, I guess it will be dropped if I remove it. I tried that, but now it fails on:

2022-12-29T22:19:51.3784137Z Error: Not enough list items
2022-12-29T22:19:51.3784329Z 
2022-12-29T22:19:51.3785843Z   with module.caf-enterprise-scale.azurerm_virtual_network_gateway.connectivity["/subscriptions/***/resourceGroups/es-connectivity-norwayeast/providers/Microsoft.Network/virtualNetworkGateways/es-vpngw-norwayeast"],
2022-12-29T22:19:51.3786796Z   on .terraform/modules/caf-enterprise-scale/resources.connectivity.tf line 138, in resource "azurerm_virtual_network_gateway" "connectivity":
2022-12-29T22:19:51.3787298Z  138: resource "azurerm_virtual_network_gateway" "connectivity" {
2022-12-29T22:19:51.3787508Z 
2022-12-29T22:19:51.3787726Z Attribute bgp_settings.0.peering_addresses.0.apipa_addresses requires 1 item
2022-12-29T22:19:51.3788092Z minimum, but config has only 0 declared.
2022-12-29T22:19:51.3837137Z 
2022-12-29T22:19:51.3837695Z Error: Not enough list items
2022-12-29T22:19:51.3838112Z 
2022-12-29T22:19:51.3840354Z   with module.caf-enterprise-scale.azurerm_virtual_network_gateway.connectivity["/subscriptions/***/resourceGroups/es-connectivity-norwayeast/providers/Microsoft.Network/virtualNetworkGateways/es-vpngw-norwayeast"],
2022-12-29T22:19:51.3844029Z   on .terraform/modules/caf-enterprise-scale/resources.connectivity.tf line 138, in resource "azurerm_virtual_network_gateway" "connectivity":
2022-12-29T22:19:51.3845023Z  138: resource "azurerm_virtual_network_gateway" "connectivity" {
2022-12-29T22:19:51.3845274Z 
2022-12-29T22:19:51.3845504Z Attribute bgp_settings.0.peering_addresses.1.apipa_addresses requires 1 item
2022-12-29T22:19:51.3845879Z minimum, but config has only 0 declared.
2022-12-29T22:19:51.7841852Z Releasing state lock. This may take a few moments...
2022-12-29T22:19:51.9473425Z ##[error]Terraform exited with code 1.
2022-12-29T22:19:51.9486131Z ##[error]Process completed with exit code 1.

but we don't use apipa addresses.

krowlandson commented 1 year ago

Thank you for providing the above @davidkarlsen.

The regional Kusto DNS zone is now created by the module (by default) which is why you no longer need that entry. It shouldn't effect your configuration, but it might also be worth reviewing the other zones we have added.

Regarding the new error above, I will look into this as this is likely caused by the new optional() defaults. Let me see if I can find the cause.

krowlandson commented 1 year ago

Hi @davidkarlsen, please can you test with release v3.1.2 and let me know if you run into any further issues?

davidkarlsen commented 1 year ago

The regional Kusto DNS zone is now created by the module (by default) which is why you no longer need that entry. It shouldn't effect your configuration, but it might also be worth reviewing the other zones we have added.

aha, I was under the impression that it was opt-in, and you had to enable via the enable_private_link_by_service element.

Regarding the new error above, I will look into this as this is likely caused by the new optional() defaults. Let me see if I can find the cause.

I'll see what happens if I drop them, not having to define the elements at all is the most clean seen from a user-perspective.

davidkarlsen commented 1 year ago

So I ended up with this diff which worked fine:

diff --git a/main.tf b/main.tf
index f39e814..78a8570 100644
--- a/main.tf
+++ b/main.tf
@@ -8,7 +8,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "~> 3.7, < 3.19"
+      version = "~> 3.37"
     }
     cloudflare = {
       source  = "cloudflare/cloudflare"
@@ -55,7 +55,7 @@ data "azurerm_client_config" "core" {}

 module "caf-enterprise-scale" {
   source  = "Azure/caf-enterprise-scale/azurerm"
-  version = "2.4.1"
+  version = "3.1.2"
   providers = {
     azurerm              = azurerm
     azurerm.connectivity = azurerm
@@ -163,8 +163,8 @@ module "caf-enterprise-scale" {
                     asn         = var.az_bgp_asn
                     peer_weight = 32768
                     peering_addresses = [
-                      { ip_configuration_name = "vnetGatewayConfig1", apipa_addresses = null },
-                      { ip_configuration_name = "vnetGatewayConfig2", apipa_addresses = null }
+                      { ip_configuration_name = "vnetGatewayConfig1" },
+                      { ip_configuration_name = "vnetGatewayConfig2" }
                     ]
                   }]
                   vpn_client_configuration = [{
@@ -265,7 +265,7 @@ module "caf-enterprise-scale" {
           }
           private_link_locations                                 = []
           public_dns_zones                                       = ["dev.az.finods.com"]
-          private_dns_zones                                      = ["appserviceenvironment.net", "privatelink.norwayeast.kusto.windows.net"]
+          private_dns_zones                                      = ["appserviceenvironment.net"]
           enable_private_dns_zone_virtual_network_link_on_hubs   = true
           enable_private_dns_zone_virtual_network_link_on_spokes = true
         }

Thanks a lot for your help and have a happy new year!

krowlandson commented 1 year ago

That's great, thank you for confirming @davidkarlsen 👍🏻

Azure / terraform-azurerm-caf-enterprise-scale