search

hashicorp / terraform-provider-azurerm

Terraform provider for Azure Resource Manager
https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs
Mozilla Public License 2.0
4.46k stars 4.54k forks source link

Wrong order of destroy on Associated azurerm_virtual_network_gateway_nat_rule on azurerm_virtual_network_gateway_connection #25231

Open Nurtic-Vibe opened 3 months ago

Nurtic-Vibe commented 3 months ago

Is there an existing issue for this?

Community Note

Terraform Version

1.7.4

AzureRM Provider Version

3.95.0

Affected Resource(s)/Data Source(s)

azurerm_virtual_network_gateway_nat_rule, azurerm_virtual_network_gateway_connection

Terraform Configuration Files

variable "vpn_nat" {
  type        = bool
  description = "Enable NAT Rules on VPN Gateway"
  default     = false
}

variable "vpn_nat_src_ip_space" {
  type        = string
  description = "Non-routable IP spaces used for NAT Rules"
  default     = "203.0.133.0/24"
  validation {
    condition     = can(cidrnetmask(var.vpn_nat_src_ip_space))
    error_message = "Must be valid IPv4 CIDR block addresses."
  }
}

variable "vpn_nat_dest_ip_spaces" {
  type        = list(string)
  description = "List of routable IP spaces used for NAT Rules"
  default     = ["192.0.2.0/25", "192.0.2.128/25"]
  validation {
    condition = length(var.vpn_nat_dest_ip_spaces) == 2 && alltrue([
      for val in var.vpn_nat_dest_ip_spaces : can(cidrnetmask(val))
    ])
    error_message = "Must contain two valid IPv4 CIDR block addresses."
  }
}

...

resource "azurerm_virtual_network_gateway_connection" "conn-pri" {
  name                = "conn-${lower(var.bh_name)}-${var.vpn_onprem_gws[0].name}"
  location            = var.beachhead_region
  resource_group_name = data.azurerm_resource_group.rg-beachhead.name

  type                       = "IPsec"
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  local_network_gateway_id   = azurerm_local_network_gateway.gw-pri.id

  enable_bgp          = true
  shared_key          = var.vpn_psk
  dpd_timeout_seconds = 45

  ipsec_policy {
    ike_encryption   = "GCMAES256"
    ike_integrity    = "SHA384"
    dh_group         = "ECP384"
    ipsec_encryption = "GCMAES256"
    ipsec_integrity  = "GCMAES256"
    pfs_group        = "ECP384"
  }

  egress_nat_rule_ids = var.vpn_nat == true ? [azurerm_virtual_network_gateway_nat_rule.nat-pri-pri[0].id, azurerm_virtual_network_gateway_nat_rule.nat-pri-sec[0].id] : null
}

resource "azurerm_virtual_network_gateway_connection" "conn-sec" {
  name                = "conn-${lower(var.bh_name)}-${var.vpn_onprem_gws[1].name}"
  location            = var.beachhead_region
  resource_group_name = data.azurerm_resource_group.rg-beachhead.name

  type                       = "IPsec"
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  local_network_gateway_id   = azurerm_local_network_gateway.gw-sec.id

  enable_bgp          = true
  shared_key          = var.vpn_psk
  dpd_timeout_seconds = 45

  ipsec_policy {
    ike_encryption   = "GCMAES256"
    ike_integrity    = "SHA384"
    dh_group         = "ECP384"
    ipsec_encryption = "GCMAES256"
    ipsec_integrity  = "GCMAES256"
    pfs_group        = "ECP384"
  }

  egress_nat_rule_ids = var.vpn_nat == true ? [azurerm_virtual_network_gateway_nat_rule.nat-sec-pri[0].id, azurerm_virtual_network_gateway_nat_rule.nat-sec-sec[0].id] : null
}

data "azurerm_virtual_network_gateway" "vnetgw-beachhead" {
  name                = azurerm_virtual_network_gateway.vnetgw-beachhead.name
  resource_group_name = data.azurerm_resource_group.rg-beachhead.name
}

resource "azurerm_virtual_network_gateway_nat_rule" "nat-pri-pri" {
  count                      = var.vpn_nat == true ? 1 : 0
  name                       = "nat-${lower(var.bh_name)}-${var.vpn_onprem_gws[0].name}-pri"
  resource_group_name        = data.azurerm_resource_group.rg-beachhead.name
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  mode                       = "EgressSnat"
  type                       = "Dynamic"
  ip_configuration_id        = data.azurerm_virtual_network_gateway.vnetgw-beachhead.ip_configuration[0].id

  internal_mapping {
    address_space = var.vpn_nat_src_ip_space
  }
  external_mapping {
    address_space = var.vpn_nat_dest_ip_spaces[0]
  }
}

resource "azurerm_virtual_network_gateway_nat_rule" "nat-pri-sec" {
  count                      = var.vpn_nat == true ? 1 : 0
  name                       = "nat-${lower(var.bh_name)}-${var.vpn_onprem_gws[0].name}-sec"
  resource_group_name        = data.azurerm_resource_group.rg-beachhead.name
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  mode                       = "EgressSnat"
  type                       = "Dynamic"
  ip_configuration_id        = data.azurerm_virtual_network_gateway.vnetgw-beachhead.ip_configuration[1].id

  internal_mapping {
    address_space = var.vpn_nat_src_ip_space
  }
  external_mapping {
    address_space = var.vpn_nat_dest_ip_spaces[1]
  }
}

resource "azurerm_virtual_network_gateway_nat_rule" "nat-sec-pri" {
  count                      = var.vpn_nat == true ? 1 : 0
  name                       = "nat-${lower(var.bh_name)}-${var.vpn_onprem_gws[1].name}-pri"
  resource_group_name        = data.azurerm_resource_group.rg-beachhead.name
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  mode                       = "EgressSnat"
  type                       = "Dynamic"
  ip_configuration_id        = data.azurerm_virtual_network_gateway.vnetgw-beachhead.ip_configuration[0].id

  internal_mapping {
    address_space = var.vpn_nat_src_ip_space
  }
  external_mapping {
    address_space = var.vpn_nat_dest_ip_spaces[0]
  }
}

resource "azurerm_virtual_network_gateway_nat_rule" "nat-sec-sec" {
  count                      = var.vpn_nat == true ? 1 : 0
  name                       = "nat-${lower(var.bh_name)}-${var.vpn_onprem_gws[1].name}-sec"
  resource_group_name        = data.azurerm_resource_group.rg-beachhead.name
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  mode                       = "EgressSnat"
  type                       = "Dynamic"
  ip_configuration_id        = data.azurerm_virtual_network_gateway.vnetgw-beachhead.ip_configuration[1].id

  internal_mapping {
    address_space = var.vpn_nat_src_ip_space
  }
  external_mapping {
    address_space = var.vpn_nat_dest_ip_spaces[1]
  }
}
...

### Debug Output/Panic Output

```shell
╷
│ Error: deleting Virtual Network Gateway Nat Rule: (Nat Rule Name "nat-aztst3-router-02-sec" / Virtual Network Gateway Name "vnetgw-beachhead-aztst3" / Resource Group "rg-herbert"): network.VirtualNetworkGatewayNatRulesClient#Delete: Failure sending request: StatusCode=400 -- Original Error: Code="GatewayNatRuleIsStillInUse" Message="The NAT rule /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-sec is still in use and cannot be deleted. Resources using NAT rule: /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-02." Details=[]
│
│
╵
╷
│ Error: deleting Virtual Network Gateway Nat Rule: (Nat Rule Name "nat-aztst3-router-01-sec" / Virtual Network Gateway Name "vnetgw-beachhead-aztst3" / Resource Group "rg-herbert"): network.VirtualNetworkGatewayNatRulesClient#Delete: Failure sending request: StatusCode=400 -- Original Error: Code="GatewayNatRuleIsStillInUse" Message="The NAT rule /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-sec is still in use and cannot be deleted. Resources using NAT rule: /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-01." Details=[]
│
│
╵
╷
│ Error: deleting Virtual Network Gateway Nat Rule: (Nat Rule Name "nat-aztst3-router-02-pri" / Virtual Network Gateway Name "vnetgw-beachhead-aztst3" / Resource Group "rg-herbert"): network.VirtualNetworkGatewayNatRulesClient#Delete: Failure sending request: StatusCode=400 -- Original Error: Code="GatewayNatRuleIsStillInUse" Message="The NAT rule /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-pri is still in use and cannot be deleted. Resources using NAT rule: /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-02." Details=[]
│
│
╵
╷
│ Error: deleting Virtual Network Gateway Nat Rule: (Nat Rule Name "nat-aztst3-router-01-pri" / Virtual Network Gateway Name "vnetgw-beachhead-aztst3" / Resource Group "rg-herbert"): network.VirtualNetworkGatewayNatRulesClient#Delete: Failure sending request: StatusCode=400 -- Original Error: Code="GatewayNatRuleIsStillInUse" Message="The NAT rule /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-pri is still in use and cannot be deleted. Resources using NAT rule: /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-01." Details=[]

Expected Behaviour

  1. Disassociation of Egress NAT Rules from VPN Connections
  2. Deletion of NAT Rules

Actual Behaviour

  1. Deletion of NAT Rules (fails)
  2. Disassociation of Egress NAT Rules from VPN Connections

Steps to Reproduce

  1. Set "var.vpn_nat" to true
  2. run terraform apply
  3. Set "var.vpn_nat" to false
  4. run terraform apply

Important Factoids

No response

References

No response

neil-yechenwei commented 3 months ago

Thanks for raising this issue. Please try to add "create_before_destroy" on azurerm_virtual_network_gateway_nat_rule and only apply it. Once it's applied successfully, then the order is changed for above situation.

Nurtic-Vibe commented 3 months ago

Hello Neil,

I added the below section to all nat rules as requested, now receiving a different error:

...
resource "azurerm_virtual_network_gateway_nat_rule" "nat-sec-sec" {
  count                      = var.vpn_nat == true ? 1 : 0
  name                       = "nat-${lower(var.bh_name)}-${var.vpn_onprem_gws[1].name}-sec"
  resource_group_name        = data.azurerm_resource_group.rg-beachhead.name
  virtual_network_gateway_id = azurerm_virtual_network_gateway.vnetgw-beachhead.id
  mode                       = "EgressSnat"
  type                       = "Dynamic"
  ip_configuration_id        = data.azurerm_virtual_network_gateway.vnetgw-beachhead.ip_configuration[1].id

  internal_mapping {
    address_space = var.vpn_nat_src_ip_space
  }
  external_mapping {
    address_space = var.vpn_nat_dest_ip_spaces[1]
  }

  lifecycle {
    create_before_destroy = true
  }
}
...

terraform apply output as per below

Terraform will perform the following actions:

  # data.azurerm_virtual_network_gateway.vnetgw-beachhead will be read during apply
  # (depends on a resource or a module with changes pending)
 <= data "azurerm_virtual_network_gateway" "vnetgw-beachhead" {
      + active_active                    = (known after apply)
      + bgp_settings                     = (known after apply)
      + custom_route                     = (known after apply)
      + default_local_network_gateway_id = (known after apply)
      + enable_bgp                       = (known after apply)
      + generation                       = (known after apply)
      + id                               = (known after apply)
      + ip_configuration                 = (known after apply)
      + location                         = (known after apply)
      + name                             = "vnetgw-beachhead-aztst3"
      + private_ip_address_enabled       = (known after apply)
      + resource_group_name              = "rg-herbert"
      + sku                              = (known after apply)
      + tags                             = (known after apply)
      + type                             = (known after apply)
      + vpn_client_configuration         = (known after apply)
      + vpn_type                         = (known after apply)
    }

  # azurerm_virtual_network_gateway.vnetgw-beachhead will be updated in-place
  ~ resource "azurerm_virtual_network_gateway" "vnetgw-beachhead" {
      ~ bgp_route_translation_for_nat_enabled = true -> false
        id                                    = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3"
        name                                  = "vnetgw-beachhead-aztst3"
        tags                                  = {}
        # (13 unchanged attributes hidden)

        # (3 unchanged blocks hidden)
    }

  # azurerm_virtual_network_gateway_connection.conn-pri will be updated in-place
  ~ resource "azurerm_virtual_network_gateway_connection" "conn-pri" {
      ~ egress_nat_rule_ids                = [
          - "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-pri",
          - "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-sec",
        ]
        id                                 = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-01"
        name                               = "conn-aztst3-router-01"
        tags                               = {}
        # (15 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

  # azurerm_virtual_network_gateway_connection.conn-sec will be updated in-place
  ~ resource "azurerm_virtual_network_gateway_connection" "conn-sec" {
      ~ egress_nat_rule_ids                = [
          - "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-pri",
          - "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-sec",
        ]
        id                                 = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-02"
        name                               = "conn-aztst3-router-02"
        tags                               = {}
        # (15 unchanged attributes hidden)

        # (1 unchanged block hidden)
    }

  # azurerm_virtual_network_gateway_nat_rule.nat-pri-pri[0] will be destroyed
  # (because index [0] is out of range for count)
  - resource "azurerm_virtual_network_gateway_nat_rule" "nat-pri-pri" {
      - id                         = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-pri" -> null
      - ip_configuration_id        = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/ipConfigurations/vnetGatewayConfig1" -> null
      - mode                       = "EgressSnat" -> null
      - name                       = "nat-aztst3-router-01-pri" -> null
      - resource_group_name        = "rg-herbert" -> null
      - type                       = "Dynamic" -> null
      - virtual_network_gateway_id = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3" -> null

      - external_mapping {
          - address_space = "10.238.109.0/28" -> null
        }

      - internal_mapping {
          - address_space = "10.180.0.0/24" -> null
        }
    }

  # azurerm_virtual_network_gateway_nat_rule.nat-pri-sec[0] will be destroyed
  # (because index [0] is out of range for count)
  - resource "azurerm_virtual_network_gateway_nat_rule" "nat-pri-sec" {
      - id                         = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-01-sec" -> null
      - ip_configuration_id        = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/ipConfigurations/vnetGatewayConfig2" -> null
      - mode                       = "EgressSnat" -> null
      - name                       = "nat-aztst3-router-01-sec" -> null
      - resource_group_name        = "rg-herbert" -> null
      - type                       = "Dynamic" -> null
      - virtual_network_gateway_id = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3" -> null

      - external_mapping {
          - address_space = "10.238.109.32/28" -> null
        }

      - internal_mapping {
          - address_space = "10.180.0.0/24" -> null
        }
    }

  # azurerm_virtual_network_gateway_nat_rule.nat-sec-pri[0] will be destroyed
  # (because index [0] is out of range for count)
  - resource "azurerm_virtual_network_gateway_nat_rule" "nat-sec-pri" {
      - id                         = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-pri" -> null
      - ip_configuration_id        = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/ipConfigurations/vnetGatewayConfig1" -> null
      - mode                       = "EgressSnat" -> null
      - name                       = "nat-aztst3-router-02-pri" -> null
      - resource_group_name        = "rg-herbert" -> null
      - type                       = "Dynamic" -> null
      - virtual_network_gateway_id = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3" -> null

      - external_mapping {
          - address_space = "10.238.109.0/28" -> null
        }

      - internal_mapping {
          - address_space = "10.180.0.0/24" -> null
        }
    }

  # azurerm_virtual_network_gateway_nat_rule.nat-sec-sec[0] will be destroyed
  # (because index [0] is out of range for count)
  - resource "azurerm_virtual_network_gateway_nat_rule" "nat-sec-sec" {
      - id                         = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-sec" -> null
      - ip_configuration_id        = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/ipConfigurations/vnetGatewayConfig2" -> null
      - mode                       = "EgressSnat" -> null
      - name                       = "nat-aztst3-router-02-sec" -> null
      - resource_group_name        = "rg-herbert" -> null
      - type                       = "Dynamic" -> null
      - virtual_network_gateway_id = "/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3" -> null

      - external_mapping {
          - address_space = "10.238.109.32/28" -> null
        }

      - internal_mapping {
          - address_space = "10.180.0.0/24" -> null
        }
    }

Plan: 0 to add, 3 to change, 4 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

azurerm_virtual_network_gateway.vnetgw-beachhead: Modifying... [id=/subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3]
╷
│ Error: Creating/Updating Virtual Network Gateway: (Name "vnetgw-beachhead-aztst3" / Resource Group "rg-herbert"): network.VirtualNetworkGatewaysClient#CreateOrUpdate: Failure sending request: StatusCode=400 -- Original Error: Code="GatewayNatRuleIsStillInUse" Message="The NAT rule /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/virtualNetworkGateways/vnetgw-beachhead-aztst3/natRules/nat-aztst3-router-02-pri is still in use and cannot be deleted. Resources using NAT rule: /subscriptions/47b6f347-bad9-4c18-8601-ebfdbf4a8b47/resourceGroups/rg-herbert/providers/Microsoft.Network/connections/conn-aztst3-router-02." Details=[]
│
│   with azurerm_virtual_network_gateway.vnetgw-beachhead,
│   on main.tf line 65, in resource "azurerm_virtual_network_gateway" "vnetgw-beachhead":
│   65: resource "azurerm_virtual_network_gateway" "vnetgw-beachhead" {
│
╵
Terraform v1.7.4
on linux_amd64
+ provider registry.terraform.io/hashicorp/azurerm v3.95.0