Closed oliverholliday closed 1 year ago
oliverholliday
commented
4 years ago This seems to be related to updating from one @pulumi/azure package to latest, and at the same time trying to change from Standard load balancer to Basic.
Once I manually edited the state to remove "addonProfile" from the inputs and set the LoadBalancerSku to "Standard" previews completed successfully.
oliverholliday
commented
4 years ago To add to this report, I've since had it with a stable cluster with no changes. Normal updates work fine, but any pulumi refresh invalidates the state in such a way that any further updates fail with the above errors. pulumi up -r has the same issue.
"@pulumi/azure": "^3.12.1",
"@pulumi/azuread": "^2.4.0",
"@pulumi/kubernetes": "^2.4.0",
"@pulumi/pulumi": "^2.6.1",I've been doing some more experimentation on this and not managed to get anywhere, the full code that's causing my problem is below:
import * as azure from "@pulumi/azure";
import * as azuread from "@pulumi/azuread";
import * as random from "@pulumi/random";
const application = new azuread.Application("test-application", {
name: name
});
const randomPassword = new random.RandomString(`test-password`, {
length: 32
});
const servicePrincipal = new azuread.ServicePrincipal("test-principal", {
applicationId: application.applicationId
});
const servicePrincipalPassword = new azuread.ServicePrincipalPassword("test-principal-password", {
servicePrincipalId: servicePrincipal.id,
value: randomPassword.result,
endDate: "2099-01-01T00:00:00Z"
});
const vnet = new azure.network.VirtualNetwork("test", {
name: "test",
resourceGroupName: "test",
location: "westeurope",
addressSpaces: ["10.192.0.0/12"]
});
const subnet = new azure.network.Subnet("test-subnet", {
addressPrefixes: ["10.193.0.0/16"],
resourceGroupName: vnet.resourceGroupName,
virtualNetworkName: vnet.name
});
const kubeCluster = new azure.containerservice.KubernetesCluster("test", {
name: "test",
resourceGroupName: "test",
location: "westeurope",
dnsPrefix: "test",
servicePrincipal: {
clientId: application.applicationId,
clientSecret: servicePrincipalPassword.value
},
defaultNodePool: {
name: "aksagentpool",
nodeCount: 1,
maxPods: 250,
type: "VirtualMachineScaleSets",
enableAutoScaling: false,
enableNodePublicIp: false,
osDiskSizeGb: 64,
vmSize: "Standard_B8ms",
vnetSubnetId: subnet.id
},
networkProfile: {
networkPlugin: "azure",
loadBalancerSku: "Standard",
serviceCidr: "192.168.200.0/22",
dnsServiceIp: "192.168.200.2",
dockerBridgeCidr: "172.17.0.1/16"
},
apiServerAuthorizedIpRanges: undefined,
addonProfile: {
kubeDashboard: {
enabled: false
},
httpApplicationRouting: {
enabled: false
}
}
}, {
additionalSecretOutputs: ["kubeConfigs", "kubeConfigRaw", "kubeAdminConfigs", "kubeAdminConfigRaw"],
ignoreChanges: ["defaultNodePool.nodeCount", "addonProfile", "kubeConfigRaw"]
});This update works correctly, but when doing a pulumi refresh the problems begin.
After running a refresh, the state looks like this:
{
"urn": "urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::mpp:Cluster$azure:containerservice/kubernetesCluster:KubernetesCluster::test",
"custom": true,
"id": "/subscriptions/e3e25af1/resourcegroups/test/providers/Microsoft.ContainerService/managedClusters/test",
"type": "azure:containerservice/kubernetesCluster:KubernetesCluster",
"inputs": {
"__defaults": [
"skuTier"
],
"addonProfile": {
"__defaults": [],
"aciConnectorLinux": {
"enabled": false,
"subnetName": ""
},
"azurePolicy": {
"enabled": false
},
"httpApplicationRouting": {
"enabled": false,
"httpApplicationRoutingZoneName": ""
},
"kubeDashboard": {
"enabled": false
},
"omsAgent": {
"enabled": false,
"logAnalyticsWorkspaceId": "",
"omsAgentIdentities": []
}
},
"apiServerAuthorizedIpRanges": [],
"autoScalerProfile": null,
"defaultNodePool": {
"__defaults": [],
"availabilityZones": [],
"enableAutoScaling": false,
"enableNodePublicIp": false,
"maxCount": 0,
"maxPods": 250,
"minCount": 0,
"name": "aksagentpool",
"nodeCount": 2,
"nodeLabels": {},
"nodeTaints": [],
"orchestratorVersion": "1.16.10",
"osDiskSizeGb": 64,
"tags": {},
"type": "VirtualMachineScaleSets",
"vmSize": "Standard_B8ms",
"vnetSubnetId": "/subscriptions/e3e25af1/resourceGroups/test/providers/Microsoft.Network/virtualNetworks/test/subnets/test"
},
"diskEncryptionSetId": "",
"dnsPrefix": "deveu3",
"enablePodSecurityPolicy": false,
"identity": null,
"kubernetesVersion": "1.16.10",
"linuxProfile": null,
"location": "westeurope",
"name": "deveu3",
"networkProfile": {
"__defaults": [
"outboundType"
],
"dnsServiceIp": "192.168.200.2",
"dockerBridgeCidr": "172.17.0.1/16",
"loadBalancerProfile": {
"effectiveOutboundIps": [
"/subscriptions/e3e25af1/resourceGroups/MC_test_test_westeurope/providers/Microsoft.Network/publicIPAddresses/3cb9a462-0754-4274-8a70-dd7bbed150ca"
],
"idleTimeoutInMinutes": 0,
"managedOutboundIpCount": 1,
"outboundIpAddressIds": [],
"outboundIpPrefixIds": [],
"outboundPortsAllocated": 0
},
"loadBalancerSku": "Standard",
"networkPlugin": "azure",
"networkPolicy": "",
"outboundType": "loadBalancer",
"podCidr": "",
"serviceCidr": "192.168.200.0/22"
},
"nodeResourceGroup": "MC_test_test_westeurope",
"privateClusterEnabled": false,
"privateLinkEnabled": false,
"resourceGroupName": "deveu",
"roleBasedAccessControl": {
"azureActiveDirectory": null,
"enabled": false
},
"servicePrincipal": {
"__defaults": [],
"clientId": "bd5b50c1",
"clientSecret": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:/gYNgcKddtmnEiBl:rzRKRBfvmZ8ARxGnGHszmuVl2SVV3dDl48aLOKuafmpDYcRnJn2mncDbdyAGEjyX1YYYqV3ttA=="
}
},
"skuTier": "Free",
"tags": {
"__defaults": [],
"environment": "Development"
},
"windowsProfile": {
"adminPassword": "",
"adminUsername": "azureuser"
}
},
"outputs": {
"__meta": "{\"e2bfb730\":{\"create\":5400000000000,\"delete\":5400000000000,\"read\":300000000000,\"update\":5400000000000}}",
"addonProfile": {
"aciConnectorLinux": {
"enabled": false,
"subnetName": ""
},
"azurePolicy": {
"enabled": false
},
"httpApplicationRouting": {
"enabled": false,
"httpApplicationRoutingZoneName": ""
},
"kubeDashboard": {
"enabled": false
},
"omsAgent": {
"enabled": false,
"logAnalyticsWorkspaceId": "",
"omsAgentIdentities": []
}
},
"apiServerAuthorizedIpRanges": [],
"autoScalerProfile": null,
"defaultNodePool": {
"availabilityZones": [],
"enableAutoScaling": false,
"enableNodePublicIp": false,
"maxCount": 0,
"maxPods": 250,
"minCount": 0,
"name": "aksagentpool",
"nodeCount": 2,
"nodeLabels": {},
"nodeTaints": [],
"orchestratorVersion": "1.16.10",
"osDiskSizeGb": 64,
"tags": {},
"type": "VirtualMachineScaleSets",
"vmSize": "Standard_B8ms",
"vnetSubnetId": "/subscriptions/e3e25af1/resourceGroups/test/providers/Microsoft.Network/virtualNetworks/test/subnets/test"
},
"diskEncryptionSetId": "",
"dnsPrefix": "test",
"enablePodSecurityPolicy": false,
"fqdn": "test.hcp.westeurope.azmk8s.io",
"id": "/subscriptions/e3e25af1/resourcegroups/test/providers/Microsoft.ContainerService/managedClusters/test",
"identity": null,
"kubeAdminConfigRaw": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:iY9BDN9yCSdJhgnE:ON/h0EwF7PmF8WKD00PQsv1q"
},
"kubeAdminConfigs": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:N+XLLeTbhHpJ4/zB:pXejk5qPkWbXdPJntSo2Nwb3"
},
"kubeConfigRaw": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:S0OGHvw99x8pR"
},
"kubeConfigs": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:ll+jek/AnOmBy"
},
"kubeletIdentities": [],
"kubernetesVersion": "1.16.10",
"linuxProfile": null,
"location": "westeurope",
"name": "deveu3",
"networkProfile": {
"dnsServiceIp": "192.168.200.2",
"dockerBridgeCidr": "172.17.0.1/16",
"loadBalancerProfile": {
"effectiveOutboundIps": [
"/subscriptions/e3e25af1/resourceGroups/MC_test_test_westeurope/providers/Microsoft.Network/publicIPAddresses/3cb9a462-0754-4274-8a70-dd7bbed150ca"
],
"idleTimeoutInMinutes": 0,
"managedOutboundIpCount": 1,
"outboundIpAddressIds": [],
"outboundIpPrefixIds": [],
"outboundPortsAllocated": 0
},
"loadBalancerSku": "Standard",
"networkPlugin": "azure",
"networkPolicy": "",
"outboundType": "loadBalancer",
"podCidr": "",
"serviceCidr": "192.168.200.0/22"
},
"nodeResourceGroup": "MC_test_test_westeurope",
"privateClusterEnabled": false,
"privateFqdn": "",
"privateLinkEnabled": false,
"resourceGroupName": "deveu",
"roleBasedAccessControl": {
"azureActiveDirectory": null,
"enabled": false
},
"servicePrincipal": {
"clientId": "bd5b50c1-451b-4e28-bec5-0e36110415b3",
"clientSecret": {
"4dabf18193072939515e22adb298388d": "1b47061264138c4ac30d75fd1eb44270",
"ciphertext": "v1:ioc+vXtzFL8jwOhM:cJaQlJOuq68Bg0qjT/EXpdmtQ81hCyVMS0JFKHw90Qj9IjHtum7p4j+9sFxBKBQU0rJzVZX81A=="
}
},
"skuTier": "Free",
"tags": {
"environment": "Development"
},
"windowsProfile": {
"adminPassword": "",
"adminUsername": "azureuser"
}
},
"dependencies": [
"urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::mpp:Cluster$mpp:AzureAdApplication$azuread:index/application:Application::test",
"urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::mpp:Cluster$mpp:AzureAdApplication$azuread:index/servicePrincipalPassword:ServicePrincipalPassword::test"
],
"provider": "urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::pulumi:providers:azure::default_3_12_1::ea23655a-69d8-40b3-900f-ae32c67110a4",
"propertyDependencies": {
"addonProfile": null,
"defaultNodePool": null,
"dnsPrefix": null,
"location": null,
"name": null,
"networkProfile": null,
"resourceGroupName": null,
"servicePrincipal": [
"urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::mpp:Cluster$mpp:AzureAdApplication$azuread:index/application:Application::test",
"urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::mpp:Cluster$mpp:AzureAdApplication$azuread:index/servicePrincipalPassword:ServicePrincipalPassword::test"
],
"tags": null
},
"additionalSecretOutputs": [
"kubeConfigs",
"kubeConfigRaw",
"kubeAdminConfigs",
"kubeAdminConfigRaw"
],
"aliases": [
"urn:pulumi:deployments-infra-dev-eu-cluster3::deployments-infra-dev-eu-cluster3::azure:containerservice/kubernetesCluster:KubernetesCluster::test"
]
}
mikhailshilkov
commented
1 year ago It sounds like this was a point-in-time problem related to migration between provider versions. Since we got no additional reports in last three years, I'll go ahead and close the issue as not relevant anymore.
I've started getting this when trying to update an existing AKS cluster:
I tried removing those various properties from inputs and outputs in the state file, but when I did a refresh they came back - previews and updates won't proceed again.
The inputs block for the kubernetesCluster section of the state file looks like this: