Issue retrieving dynamically generated ip address from private endpoint resource

[sfc-gh-raram]

What happened?

Hi team. I'm working on creating a private endpoint, private dns zone, and A records that use the ip address from the private endpoint using the pulumi azure sdk for go. I'm running into a strange issue where I am unable to retrieve the ip address of the private endpoint despite it being created and being able to see it in the Azure console.

I'm attempting to get the ip address through the NetworkInteraces field, which in turn I get the IP configurations field from. However, this leads to a index out of bounds error since the network interface does not have any ip configuration. I find this bizarre since I can see it in the console. I also tried getting the ip address through the custom dns config field, however that is empty. I also tried getting it through the privatednszonegroup but the ip configuration were also empty.

I consulted the below two issues raised previously, but they did not help unfortunately. I would appreciate any help on this issue, thank you. https://github.com/pulumi/pulumi-azure-native/issues/2830 https://github.com/pulumi/pulumi-azure-native/issues/1707

Example

package traffic_privatelink_probing_infra

import (
    "fmt"
    azureNative "github.com/pulumi/pulumi-azure-native-sdk"
    "github.com/pulumi/pulumi-azure-native-sdk/network"
    "github.com/pulumi/pulumi-azure-native-sdk/resources"
    "github.com/pulumi/pulumi/sdk/v3/go/pulumi"
    "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
    "strings"
)

type ProbingInfra struct {
    Config           *config.Config
    Ctx              *pulumi.Context
    PulumiStackRef   *pulumi.StackReference
    Provider         *azureNative.Provider // Instance of Azure provider where resources will be created
    PrivatelinkInfos []*PrivatelinkInfo
    SubscriptionId   string
    Tags             pulumi.StringMap // Tags to apply to generated resources
}

type PrivatelinkInfo struct {
    Deployment              string
    PrivatelinkServiceId    string
    PrivatelinkUrl          string
    TrafficEnvoyTestAccount string
}

func (p *ProbingInfra) BuildResources() (err error) {
    deploymentToTrafficPrivateEndpointID := pulumi.Map{}
    // k8s app cluster vnet
    k8sAppClusterVnet := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s-rg/providers/Microsoft.Network/"+
        "virtualNetworks/%s-vnet",
        p.SubscriptionId, p.Ctx.Stack(), p.Ctx.Stack())
    resourceGroupName := fmt.Sprintf("%s-rg", p.Ctx.Stack())
    virtualNetworkLinkName := fmt.Sprintf("%s-private-zone-link", p.Ctx.Stack())

    // Get resource group
    k8sAppClusterResourceGroup, _ := resources.LookupResourceGroup(p.Ctx, &resources.LookupResourceGroupArgs{
        ResourceGroupName: resourceGroupName,
    })

    // Create new private dns zone
    privatelinkZoneName := fmt.Sprintf("privatelink-zone-%s", p.Ctx.Stack())
    dnsZone, err := network.NewPrivateZone(p.Ctx, privatelinkZoneName, &network.PrivateZoneArgs{
        Location:          pulumi.String("Global"),
        PrivateZoneName:   pulumi.String("privatelink.snowflakecomputing.com"),
        ResourceGroupName: pulumi.String(k8sAppClusterResourceGroup.Name),
        Tags:              p.Tags,
    })

    if err != nil {
        return err
    }

    // Create vnet link with the app cluster vnet
    virtualNetworkLinkArgs := &network.VirtualNetworkLinkArgs{
        Location:          pulumi.String("Global"),
        ResourceGroupName: pulumi.String(k8sAppClusterResourceGroup.Name),
        PrivateZoneName:   dnsZone.Name,
        VirtualNetwork: &network.SubResourceArgs{
            Id: pulumi.String(k8sAppClusterVnet),
        },
        RegistrationEnabled:    pulumi.Bool(false),
        VirtualNetworkLinkName: pulumi.String(virtualNetworkLinkName),
    }
    _, err = network.NewVirtualNetworkLink(p.Ctx, virtualNetworkLinkName, virtualNetworkLinkArgs)

    if err != nil {
        return err
    }

    for _, plInfo := range p.PrivatelinkInfos {
        // Azure only accept relative record sets, so privatelink.snowflakecomputing.com needs to be truncated
        truncPrivatelinkMainUrl := strings.ReplaceAll(plInfo.PrivatelinkUrl, ".privatelink.snowflakecomputing.com", "")

        privateEndpointName := fmt.Sprintf("test-traffic-privatelink-private-endpoint-%s", plInfo.Deployment)
        privateDNSZoneGroupName := fmt.Sprintf("privateDNSZoneGroup-%s", plInfo.Deployment)
        privatelinkServiceConnectionName := fmt.Sprintf("%s-privatelink-connection", plInfo.Deployment)
        trafficTestAccountRegion := fmt.Sprintf("%s.%s", plInfo.TrafficEnvoyTestAccount, truncPrivatelinkMainUrl)
        // TODO: Output org into GDS. Some deployments do not have the sfengineering org.
        trafficTestAccountRegionless := fmt.Sprintf("sfengineering-%s", plInfo.TrafficEnvoyTestAccount)

        privateEndpoint, err := network.NewPrivateEndpoint(p.Ctx, privateEndpointName, &network.PrivateEndpointArgs{
            Location:            pulumi.String(k8sAppClusterResourceGroup.Location),
            PrivateEndpointName: pulumi.String(privateEndpointName),
            ManualPrivateLinkServiceConnections: network.PrivateLinkServiceConnectionArray{
                &network.PrivateLinkServiceConnectionArgs{
                    Name:                 pulumi.String(privatelinkServiceConnectionName),
                    PrivateLinkServiceId: pulumi.String(plInfo.PrivatelinkServiceId),
                    RequestMessage:       pulumi.String("Please approve this cloudprober connection. This is owned by CloudEng Traffic."),
                },
            },
            ResourceGroupName: pulumi.String(k8sAppClusterResourceGroup.Name),
            Subnet: &network.SubnetTypeArgs{
                Id: pulumi.String(k8sAppClusterVnet + "/subnets/aks"),
            },
        })

        if err != nil {
            return err
        }

        deploymentToTrafficPrivateEndpointID[plInfo.Deployment] = privateEndpoint.ID()

        _, err = network.NewPrivateDnsZoneGroup(p.Ctx, privateDNSZoneGroupName, &network.PrivateDnsZoneGroupArgs{
            PrivateDnsZoneConfigs: network.PrivateDnsZoneConfigArray{
                &network.PrivateDnsZoneConfigArgs{
                    Name:             dnsZone.Name,
                    PrivateDnsZoneId: dnsZone.ID(),
                },
            },
            PrivateDnsZoneGroupName: pulumi.String(privateDNSZoneGroupName),
            PrivateEndpointName:     pulumi.String(privateEndpointName),
            ResourceGroupName:       pulumi.String(k8sAppClusterResourceGroup.Name),
        })

        if err != nil {
            return err
        }
**** This is where the index out of bounds error is occurring
        privateEndpointIp := privateEndpoint.NetworkInterfaces.Index(pulumi.Int(0)).IpConfigurations().Index(pulumi.Int(0)).PrivateIPAddress().Elem()
        _, err = network.NewPrivateRecordSet(p.Ctx, trafficTestAccountRegionless, &network.PrivateRecordSetArgs{
            ARecords:              network.ARecordArray{network.ARecordArgs{Ipv4Address: privateEndpointIp}},
            RecordType:            pulumi.String("A"),
            RelativeRecordSetName: pulumi.String(trafficTestAccountRegionless),
            ResourceGroupName:     pulumi.String(k8sAppClusterResourceGroup.Name),
            Ttl:                   pulumi.Float64(3600),
            PrivateZoneName:       dnsZone.Name,
        })

        if err != nil {
            return err
        }

        _, err = network.NewPrivateRecordSet(p.Ctx, trafficTestAccountRegion, &network.PrivateRecordSetArgs{
            ARecords:              network.ARecordArray{network.ARecordArgs{Ipv4Address: privateEndpointIp}},
            RecordType:            pulumi.String("A"),
            RelativeRecordSetName: pulumi.String(trafficTestAccountRegion),
            ResourceGroupName:     pulumi.String(k8sAppClusterResourceGroup.Name),
            Ttl:                   pulumi.Float64(3600),
            PrivateZoneName:       dnsZone.Name,
        })

        if err != nil {
            return err
        }
    }
    p.Ctx.Export("trafficPrivateEndpointIds", deploymentToTrafficPrivateEndpointID)
    return nil
}

Output of pulumi about

Pulumi version: v3.74.0

Diagnostics:
  pulumi:pulumi:Stack (k8s-pulumi-tenants-azure-eastus2k8sdev1):
    panic: runtime error: index out of range [0] with length 0
    goroutine 1913 [running]:
    github.com/pulumi/pulumi-azure-native-sdk/network.NetworkInterfaceIPConfigurationResponseArrayOutput.Index.func1({_, _, _})
        /tmp/avalancheK8s/eastus2k8sdev1/userActions/pulumiPreview/k8s-pulumi-app-resources/tenants/azure/vendor/github.com/pulumi/pulumi-azure-native-sdk/network/pulumiTypes.go:44344 +0xc8
    reflect.Value.call({0x1e929c0?, 0x27da898?, 0x30?}, {0x273710d, 0x4}, {0xc0002fbe30, 0x1, 0xc0002fbe30?})
        /usr/lib/go/src/reflect/value.go:596 +0xca6
    reflect.Value.Call({0x1e929c0?, 0x27da898?, 0x101000001fd6e80?}, {0xc0002fbe30?, 0x0?, 0x1?})
        /usr/lib/go/src/reflect/value.go:380 +0xb9
    github.com/pulumi/pulumi/sdk/v3/go/internal.(*applier).Call(0xc000825780, {0x29a6110, 0x4dd6c00}, {0x1c40040?, 0xc000465848?, 0xc000575fb0?})
        /tmp/avalancheK8s/eastus2k8sdev1/userActions/pulumiPreview/k8s-pulumi-app-resources/tenants/azure/vendor/github.com/pulumi/pulumi/sdk/v3/go/internal/types.go:510 +0x22d
    github.com/pulumi/pulumi/sdk/v3/go/internal.(*OutputState).applyTWithApplier.func1()
        /tmp/avalancheK8s/eastus2k8sdev1/userActions/pulumiPreview/k8s-pulumi-app-resources/tenants/azure/vendor/github.com/pulumi/pulumi/sdk/v3/go/internal/types.go:622 +0x16a
    created by github.com/pulumi/pulumi/sdk/v3/go/internal.(*OutputState).applyTWithApplier in goroutine 1
        /tmp/avalancheK8s/eastus2k8sdev1/userActions/pulumiPreview/k8s-pulumi-app-resources/tenants/azure/vendor/github.com/pulumi/pulumi/sdk/v3/go/internal/types.go:609 +0x265
    error: an unhandled error occurred: program exited with non-zero exit code: 2

Additional context

No response

Contributing

Vote on this issue by adding a 👍 reaction. To contribute a fix for this issue, leave a comment (and link to your pull request, if you've opened one already).

[danielrbradley]

Hi @sfc-gh-raram thanks for getting in touch.

1. What version of the provider were you using when running this (you can find this by running pulumi about).
2. Does this still occur with the latest version of the provider (v2.39.0 at the time of writing).
3. You can also try enabling verbose logs to see what the Azure API is returning to help identify if it's the API that's missing the expected data.

[sfc-gh-raram]

Hi @danielrbradley . I am using the pulumi provider v1.103.0.

It will be difficult to upgrade the provider version due to the current set up, so I may not be able to answer your second question.

I will work on enabling verbose logging, and let you know.

[sfc-gh-raram]

I also tested it in v2.34.0 and encountered the same issue.

[sfc-gh-raram]

@danielrbradley Is my way of getting the ip address from the private endpoint correct? I also tried using the custom config struct field and attempted to get it through the private dns zone, but no luck.

privateEndpoint.NetworkInterfaces.Index(pulumi.Int(0)).IpConfigurations().Index(pulumi.Int(0)).PrivateIPAddress().Elem()

[danielrbradley]

Thanks for the extra info @sfc-gh-raram

Recapping what we know:

1. The ip address property is shown in the Azure Console
2. It appears that either the NetworkInterfaces or IpConfigurations lists are empty and therefore throw an error when attempting to access the first element.

Does that look correct?

When you look at the state, do you see these properties in the outputs (run pulumi stack export then search for test-traffic-privatelink-private-endpoint and look at the outputs)? Having a redacted copy of the resource's state in this issue might also help us to narrow down the issue and be easier than looking at the verbose logs for the HTTP responses.

You could also look at what the ARM template looks like for the resource in the Azure portal as that should match the shape of the Pulumi resource properties.

[DjordjeTosic]

Hello everyone, I have same issue with retrieving IP address from Private Endpoint, IP address is dynamically populated during the creation of resource. Code is following: kv_private_endpoint = azure_native.network.PrivateEndpoint( "KeyVaulPE", resource_group_name=resource_group.name, location=resource_group.location, private_link_service_connections=[azure_native.network.PrivateLinkServiceConnectionArgs( name="KV_private_link_service_connection", private_link_service_id=key_vault.id, group_ids=["vault"] )], subnet=azure_native.network.SubnetArgs( id=next(iter(snets.values())) ), opts=pulumi.ResourceOptions( provider=provider) ) kv_private_endpoint.network_interfaces.apply(lambda plsc: print(f"{plsc[0].ip_configurations}")) This code prints None value, even though resource is created and IP address is associated, so i can't even print IP address because it will drop indexation error, cause ip_configurations is none. I'm using pulumi_azure_native provider v2.44.0

[maribowman]

Hello everyone, I have same issue with retrieving IP address from Private Endpoint, IP address is dynamically populated during the creation of resource. Code is following: kv_private_endpoint = azure_native.network.PrivateEndpoint( "KeyVaulPE", resource_group_name=resource_group.name, location=resource_group.location, private_link_service_connections=[azure_native.network.PrivateLinkServiceConnectionArgs( name="KV_private_link_service_connection", private_link_service_id=key_vault.id, group_ids=["vault"] )], subnet=azure_native.network.SubnetArgs( id=next(iter(snets.values())) ), opts=pulumi.ResourceOptions( provider=provider) ) kv_private_endpoint.network_interfaces.apply(lambda plsc: print(f"{plsc[0].ip_configurations}")) This code prints None value, even though resource is created and IP address is associated, so i can't even print IP address because it will drop indexation error, cause ip_configurations is none. I'm using pulumi_azure_native provider v2.44.0

having the same issue with python provider v2.36.0 and worked around it with this snippet:

pep = azure_native.network.PrivateEndpoint(...)
ip = pep.network_interfaces.apply(lambda nics:
                                  azure_native.network.get_network_interface(
                                      network_interface_name=nics[0].id.split("/")[-1],
                                      resource_group_name="my-azure-rg",
                                  ).ip_configurations[0].private_ip_address)

wouldn't mind ditching the workaround though.

[danielrbradley]

Thanks for the workaround @maribowman - that looks like a good approach.

It's also interesting narrowing the issue down to the fact that the PrivateEndpoint does include the ids of each network interface, but is missing the IP configurations.

It would appear that this is likely an issue with the Azure service returning responses which don't satisfy the schema (missing the ipConfigurations). To confirm this we either need a minimal program with which we quickly reproduce this issue or sanitised verbose logs which show the raw responses from the Azure service to identify if it's missing there or it's a bug in the provider.