microsoft / azure-container-apps

Roadmap and issues for Azure Container Apps
MIT License
372 stars 29 forks source link

Bicep, using an object to compose Key vault URL results in a deployment error #749

Open abouroubi opened 1 year ago

abouroubi commented 1 year ago

Please provide us with the following information:

This issue is a: (mark with an x)

Issue description

I'm deploying Container Apps using Bicep, in the secrets array if I want to use Key vault, and generate the Key Vault URL using an object, this error happens:

{
  "status": "Failed",
  "error": {
    "code": "DeploymentFailed",
    "target": "/subscriptions/REDACTED/providers/Microsoft.Resources/deployments/main",
    "message": "At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.",
    "details": [
      {
        "code": "InvalidTemplateDeployment",
        "message": "The template deployment 'containerapps' is not valid according to the validation procedure. The tracking id is '2cc51ab5-4273-4ae6-ae62-b251c5a4f5d0'. See inner errors for details.",
        "details": [
          {
            "code": "ValidationForResourceFailed",
            "message": "Validation failed for a resource. Check 'Error.Details[0]' for more information.",
            "details": [
              {
                "code": "ContainerAppSecretKeyVaultUrlInvalid",
                "message": "Invalid Request: Container app secret(s) with name(s) 'mongodb-connection-string-2' are with invalid keyVaultUrl values, the value should be at format 'https://mykeyvault.vault.azure.net/secrets/mysecret', or 'https://mykeyvault.vault.azure.net/secrets/mysecret/994e967bb9b5468b92a4737aeee97276', and Container App could not reference keyvault secret from different cloud."
              }
            ]
          }
        ]
      }
    ]
  }
}

Oddly enough, if I use plain string variables or params, it works.

Steps to reproduce

  1. Run this code
    
    // Parameters
    param location string 

param keyVaultName string

param acrName string

param acrDefaultImage string

param cosmosAccountName string

param workspaceName string

resource cosmosAccount 'Microsoft.DocumentDB/databaseAccounts@2021-03-15' existing = { name: cosmosAccountName }

var databaseUrlSecretName = 'mongodb-uri'

var databaseUrlSecret = { secretName: databaseUrlSecretName secretValue: cosmosAccount.listConnectionStrings().connectionStrings[0].connectionString }

resource appInsightWorkspace 'Microsoft.OperationalInsights/workspaces@2020-10-01' existing = { name: workspaceName }

resource uai 'Microsoft.ManagedIdentity/userAssignedIdentities@2022-01-31-preview' = { name: 'id-ai' location: location }

resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-12-01' existing = { name: acrName }

var acrPullRole = resourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')

// Give the User Assigned Identity Pull authorization on Azure Container Registry module uaiRbac './acr.role.assignment.bicep' = { name: guid(resourceGroup().id, uai.id, acrPullRole) params: { acrName: acrName principalId: uai.properties.principalId roleId: acrPullRole } }

var subnetName = 'snet-test' resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-09-01' = { name: 'vnet-test' location: location properties: { addressSpace: { addressPrefixes: [ '10.0.0.0/16' ] } subnets: [ { name: subnetName properties: { addressPrefix: '10.0.0.0/23' } } ] } }

resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' existing = { name: keyVaultName }

resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2022-10-01' = { name: 'cae-test' location: location sku: { name: 'Consumption' } properties: { appLogsConfiguration: { destination: 'log-analytics' logAnalyticsConfiguration: { customerId: appInsightWorkspace.properties.customerId sharedKey: appInsightWorkspace.listKeys().primarySharedKey } } vnetConfiguration: { infrastructureSubnetId: virtualNetwork.properties.subnets[0].id } zoneRedundant: false } }

resource containerApp 'Microsoft.App/containerApps@2022-11-01-preview' = { name: 'ca-test' location: location identity: { type: 'SystemAssigned,UserAssigned' userAssignedIdentities: { '${uai.id}': {} } } properties: { managedEnvironmentId: containerAppsEnvironment.id configuration: { ingress: { external: false targetPort: 80 allowInsecure: false traffic: [ { latestRevision: true weight: 100 } ] } secrets: [ { name: 'mongodb-connection-string' //This WORKS keyVaultUrl: 'https://${keyVaultName}.vault.azure.net/secrets/${databaseUrlSecretName}' identity: 'System' } { name: 'mongodb-connection-string-2' // This DOESN'T keyVaultUrl: '${keyVault.properties.vaultUri}secrets/${databaseUrlSecret.secretName}' identity: 'System' } { name: 'mongodb-connection-string-3' // This DOESN'T keyVaultUrl: '${keyVault.properties.vaultUri}secrets/${databaseUrlSecretName}' identity: 'System' } { name: 'mongodb-connection-string-4' // This DOESN'T keyVaultUrl: '${keyVaultName}secrets/${databaseUrlSecret.secretName}' identity: 'System' } ] registries: [ { identity: uai.id server: containerRegistry.properties.loginServer } ] } template: { revisionSuffix: 'firstrevision' containers: [ { name: 'ca-test' image: acrDefaultImage env: [

        {
          name: 'mongodbConnectionString'
          secretRef: 'mongodb-connection-string'
        }

      ]

      resources: {
        cpu: json('.25')
        memory: '.5Gi'
      }
    }
  ]
  scale: {
    minReplicas: 1
    maxReplicas: 10
    rules: [
      {
        name: 'http-requests'
        http: {
          metadata: {
            concurrentRequests: '10'
          }
        }
      }
    ]
  }
}

} }

module keyVaultRoleAssignement './keyvault.role.assignement.bicep' = { name: 'keyVaultRoleAssignement' params: { keyVaultName: keyVaultName objectIds: [ containerApp.identity.principalId ] } }



**Expected behavior** [What you expected to happen.]
The template to be deployed without error

**Actual behavior** [What actually happened.]
The deployment fails, when an object or a resource is used to generate the key vault url.

**Screenshots**         
If applicable, add screenshots to help explain your problem.

### Additional context
Ex. Did this issue occur in the CLI or the Portal?
torosent commented 1 year ago

Duplicate of #736

sanchitmehta commented 1 year ago

@abouroubi This happened because during the ARM Template deployment a pre-flight validation check is done. This check doesn't have the resolved secret value (as the template hasn't run), and hence the deployment fails.

We are working fixing this by removing this check during the pre-flight api call. We would revert when the deployment starts

abouroubi commented 1 year ago

@sanchitmehta thanks for your answer. I don't know all the validation steps, but this property ${databaseUrlSecret.secretName} is supposed to be available before the template is done. It's just constant inside an object.

Anyway, maybe it's the same issue as #736 as @torosent mentioned.

jordanchang commented 1 year ago

I'm experiencing issues too, but not the error message mentioned above. Instead my deployment is stuck in an error state (InternalServerError) and my ADO pipeline is stuck in In-progress. I have to manually cancel my deployment and also my pipeline.

The message in my Azure Portal is:

{
    "status": "Failed",
    "error": {
        "code": "InternalServerError",
        "message": "Internal server error occurred. correlation ID: 89937157-f50b-4b8f-ac27-bd0e5ad36988"
 } 

Here's how I'm setting the Keyvault reference:

{
   name : 'MySecret'
   keyVaultUrl: 'https://${mykeyvault}.vault.azure.net/secrets/MyKey'
  identity: myIdent.id
}