search

hashicorp / terraform-provider-azurerm

Terraform provider for Azure Resource Manager
https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs
Mozilla Public License 2.0
4.61k stars 4.65k forks source link

Azure App Service Wont Write to Storage Account #13755

Closed jasric89 closed 2 years ago

jasric89 commented 3 years ago

Community Note

Terraform (and AzureRM Provider) Version

Terraform v1.0.7 on linux_amd64

Affected Resource(s)

Terraform Configuration Files

# Terraform Block
terraform {
  required_version = ">= 1.0"
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = ">= 2.0"
    }
    random = {
      source  = "hashicorp/random"
      version = ">= 3.0"
    }
  }
  #Terraform State Storage Account
  backend "azurerm" {}
}

# Providers Block
provider "azurerm" {
  features {}
}
provider "azuread" {
  tenant_id     = "VALUE"
  client_id     = "VALUE"
  client_secret = "VALUE"
}

provider "random" {}
provider "time" {}

# Random String Resource

resource "random_string" "myrandom" {
  length  = 6
  number  = false
  upper   = false
  special = false
}

resource "time_rotating" "main" {
  rotation_rfc3339 = null
  rotation_years   = 2

  triggers = {
    end_date = null
    years    = 2
  }
}

resource "azurerm_storage_account" "website_log_storage" {
  name                     = "cicweblogsstorageacc"
  resource_group_name      = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.name
  location                 = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.location
  account_tier             = "Standard"
  account_replication_type = "LRS"

  identity {
    type = "SystemAssigned"
  }
}

resource "azurerm_storage_container" "website_logs_container" {
  name                  = "${var.website_name}-logscont"
  storage_account_name  = azurerm_storage_account.website_log_storage.name
  container_access_type = "private"
}

data "azurerm_storage_account_blob_container_sas" "website_logs_container_sas" {
  connection_string = azurerm_storage_account.website_log_storage.primary_connection_string
  container_name    = azurerm_storage_container.website_logs_container.name

  start  = timestamp()
  expiry = time_rotating.main.rotation_rfc3339

  permissions {
    read   = true
    add    = true
    create = true
    write  = true
    delete = true
    list   = true
  }

  cache_control       = "max-age=5"
  content_disposition = "inline"
  content_encoding    = "deflate"
  content_language    = "en-US"
  content_type        = "application/json"
}

resource "azurerm_storage_account" "website_installers_account" {
  name                     = "websiteinstallersac"
  resource_group_name      = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.name
  location                 = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.location
  account_tier             = "Standard"
  account_replication_type = "LRS"
  #primary_access_key       = azurerm_storage_account_customer_managed_key.guides_key.name 

  identity {
    type = "SystemAssigned"
  }
}

resource "azurerm_storage_container" "website_installers_container" {
  depends_on = [
    azurerm_storage_account.website_installers_account
  ]
  name                  = "websiteinstallerscont"
  storage_account_name  = azurerm_storage_account.website_installers_account.name
  container_access_type = "private"
}

resource "azurerm_app_service_plan" "websiteappserviceplan" {
  name                = "appserviceplan-dgyn27h2dfoyojc"
  location            = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.location
  resource_group_name = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.name

  sku {
    tier = "Basic"
    size = "B1"
  }
}

resource "azurerm_app_service" "website_app" {
  depends_on = [
    azurerm_key_vault_access_policy.service_principal,
    azurerm_key_vault_access_policy.client,
    azurerm_key_vault_access_policy.website_installers_storage_accesspolicy,
    azurerm_storage_container.website_installers_container
  ]

  name                = var.website_name
  location            = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.location
  resource_group_name = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.name
  app_service_plan_id = azurerm_app_service_plan.websiteappserviceplan.id

  app_settings = {
    "KEY_VAULT_URL" = azurerm_key_vault.nscsecrets.vault_uri

  }

  site_config {
    always_on                = true
    dotnet_framework_version = "v5.0"
    app_command_line         = "dotnet EventManagement.Web.dll"

  }

  storage_account {
    name         = azurerm_storage_account.website_installers_account.id
    type         = "AzureBlob"
    account_name = azurerm_storage_account.website_installers_account.name
    access_key   = data.azurerm_key_vault_secret.Guides_AccessKey.id
    share_name   = azurerm_storage_container.website_installers_container.name
    mount_path   = "/var/lib/guides"
  }

  logs {
    detailed_error_messages_enabled = true
    failed_request_tracing_enabled  = true
    application_logs {
      azure_blob_storage {
        level             = "Information"
        sas_url           = format("https://${azurerm_storage_account.website_log_storage.name}.blob.core.windows.net/${azurerm_storage_container.website_logs_container.name}%s", data.azurerm_storage_account_blob_container_sas.website_logs_container_sas.sas)
        retention_in_days = 365
      }
    }

    http_logs {
      azure_blob_storage {
        sas_url           = format("https://${azurerm_storage_account.website_log_storage.name}.blob.core.windows.net/${azurerm_storage_container.website_logs_container.name}%s", data.azurerm_storage_account_blob_container_sas.website_logs_container_sas.sas)
        retention_in_days = 365
      }
    }

  }

  connection_string {
    name  = "StorageAccount"
    type  = "Custom"
    value = azurerm_storage_account.website_log_storage.primary_connection_string
  }

  identity {
    type = "SystemAssigned"
  }
}

// Users & Groups which I want to give permissions to be able to access the keyvault.
data "azuread_user" "user" {
  user_principal_name = "J.Content@netsupportsoftware.com"
}

data "azuread_group" "Classroom_In_The_Cloud_AZ_AD_Group" {
  display_name     = "NSL Development Dept"
  security_enabled = true
}

// This gets the Azure AD Tenant ID information to deploy for KeyVault. 
resource "azurerm_key_vault" "nscsecrets" {
  name                       = "${var.key_vault_name}-${random_string.myrandom.id}"
  resource_group_name        = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.name
  location                   = azurerm_resource_group.Classroom_In_The_Cloud_Terraform.location
  sku_name                   = "standard"
  tenant_id                  = data.azurerm_client_config.current.tenant_id
  soft_delete_retention_days = 7
  purge_protection_enabled   = false
}

resource "azurerm_key_vault_secret" "Website_Logs_Storage_URI" {
  name         = "WebsiteLogsStorageURI"
  value        = format("https://${azurerm_storage_account.website_log_storage.name}.blob.core.windows.net/${azurerm_storage_container.website_logs_container.name}%s", data.azurerm_storage_account_blob_container_sas.website_logs_container_sas.sas)
  key_vault_id = azurerm_key_vault.nscsecrets.id

  depends_on = [
    azurerm_key_vault_access_policy.client,
    azurerm_key_vault_access_policy.service_principal,
  ]

}

resource "azurerm_key_vault_secret" "Website_Guides_Access_key" {
  name         = "WebsiteGuidesAccessKey"
  value        = azurerm_storage_account.website_installers_account.primary_access_key
  key_vault_id = azurerm_key_vault.nscsecrets.id

  depends_on = [
    azurerm_key_vault_access_policy.client,
    azurerm_key_vault_access_policy.service_principal,
  ]

}

data "azurerm_key_vault_secret" "Guides_AccessKey" {
  depends_on = [
    azurerm_storage_container.website_installers_container
  ]
  name         = azurerm_key_vault_secret.Website_Guides_Access_key.name
  key_vault_id = azurerm_key_vault.nscsecrets.id
}

resource "azurerm_key_vault_key" "website_logs_key" {
  name         = "${var.website_name}-logskey"
  key_vault_id = azurerm_key_vault.nscsecrets.id
  key_type     = "RSA"
  key_size     = 2048
  key_opts     = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"]

  depends_on = [
    azurerm_key_vault_access_policy.client,
    azurerm_key_vault_access_policy.service_principal,
  ]
}

resource "azurerm_key_vault_key" "website_guides_key" {
  name         = "${var.website_name}-guideskey"
  key_vault_id = azurerm_key_vault.nscsecrets.id
  key_type     = "RSA"
  key_size     = 2048
  key_opts     = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"]

  depends_on = [
    azurerm_key_vault_access_policy.client,
    azurerm_key_vault_access_policy.service_principal,
  ]
}

resource "azurerm_key_vault_access_policy" "client" { // This is for AD Users Logged into Azure to give them the right access when creating resources. 
  key_vault_id            = azurerm_key_vault.nscsecrets.id
  tenant_id               = data.azurerm_client_config.current.tenant_id
  object_id               = data.azuread_group.Classroom_In_The_Cloud_AZ_AD_Group.object_id
  secret_permissions      = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set", ]
  key_permissions         = ["Backup", "Create", "Decrypt", "Delete", "Encrypt", "Get", "Import", "List", "Purge", "Recover", "Restore", "Sign", "UnwrapKey", "Update", "Verify", "WrapKey", ]
  storage_permissions     = ["Backup", "Delete", "DeleteSAS", "Get", "GetSAS", "List", "ListSAS", "Purge", "Recover", "RegenerateKey", "Restore", "Set", "SetSAS", "Update", ]
  certificate_permissions = ["create", "delete", "deleteissuers", "get", "getissuers", "import", "list", "listissuers", "managecontacts", "manageissuers", "setissuers", "update", ]
}

resource "azurerm_key_vault_access_policy" "service_principal" { // This is for the Service Principal in the pipeline to be able to make changes to Key Vault. 
  key_vault_id            = azurerm_key_vault.nscsecrets.id
  tenant_id               = data.azurerm_client_config.current.tenant_id
  object_id               = data.azurerm_client_config.current.object_id
  secret_permissions      = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set", ]
  key_permissions         = ["Backup", "Create", "Decrypt", "Delete", "Encrypt", "Get", "Import", "List", "Purge", "Recover", "Restore", "Sign", "UnwrapKey", "Update", "Verify", "WrapKey", ]
  storage_permissions     = ["Backup", "Delete", "DeleteSAS", "Get", "GetSAS", "List", "ListSAS", "Purge", "Recover", "RegenerateKey", "Restore", "Set", "SetSAS", "Update", ]
  certificate_permissions = ["create", "delete", "deleteissuers", "get", "getissuers", "import", "list", "listissuers", "managecontacts", "manageissuers", "setissuers", "update", ]
}

resource "azurerm_key_vault_access_policy" "website_logs_storage_accesspolicy" { // This is for the Storage Account for Website Logs. 
  key_vault_id            = azurerm_key_vault.nscsecrets.id
  tenant_id               = data.azurerm_client_config.current.tenant_id
  object_id               = azurerm_storage_account.website_log_storage.identity[0].principal_id
  key_permissions         = ["get", "create", "delete", "list", "restore", "recover", "unwrapkey", "wrapkey", "purge", "encrypt", "decrypt", "sign", "verify", ]
  secret_permissions      = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set", ]
  certificate_permissions = ["create", "delete", "deleteissuers", "get", "getissuers", "import", "list", "listissuers", "managecontacts", "manageissuers", "setissuers", "update", ]
}

resource "azurerm_key_vault_access_policy" "website_installers_storage_accesspolicy" { // This is for the Storage Account for Website Logs. 
  depends_on = [
    azurerm_storage_container.website_installers_container
  ]
  key_vault_id            = azurerm_key_vault.nscsecrets.id
  tenant_id               = data.azurerm_client_config.current.tenant_id
  object_id               = azurerm_storage_account.website_installers_account.identity[0].principal_id
  key_permissions         = ["get", "create", "delete", "list", "restore", "recover", "unwrapkey", "wrapkey", "purge", "encrypt", "decrypt", "sign", "verify", ]
  secret_permissions      = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set", ]
  certificate_permissions = ["create", "delete", "deleteissuers", "get", "getissuers", "import", "list", "listissuers", "managecontacts", "manageissuers", "setissuers", "update", ]
}

# Copy-paste your Terraform configurations here - for large Terraform configs,
# please use a service like Dropbox and share a link to the ZIP file. For
# security, you can also encrypt the files using our GPG public key: https://keybase.io/hashicorp

Debug Output

Panic Output

Expected Behaviour

Terraform App Service should write to storage account.

Actual Behaviour

Errors and doesn't write to storage account. I believe this may be a bug when specifying Storage Account Name in Web App Settings.

I first pointed to the storage account name but got the following error: 

Error: updating Storage Accounts for App Service "nsclassroom-dgyn27h2dfoyojc": web.AppsClient#UpdateAzureStorageAccounts: Failure sending request: StatusCode=409 -- Original Error: autorest/azure: Service returned an error. Status=<nil> <nil>

Upon researching more and according to this website: https://github.com/kumarvna/terraform-azurerm-app-service it should be the storage account identifier.

Error Message is the following: 

 Error: updating Storage Accounts for App Service "nsclassroom-dgyn27h2dfoyojc": web.AppsClient#UpdateAzureStorageAccounts: Failure responding to request: StatusCode=400 -- Original Error: autorest/azure: Service returned an error. Status=400 Code="BadRequest" Message="AzureStoragePropertyDictionary is invalid.  ID in AzureStoragePropertyDictionary contains invalid characters: /subscriptions/3b92ad75-8bb4-44a3-92df-394bc15085ef/resourceGroups/Classroom_In_The_Cloud_Terraform/providers/Microsoft.Storage/storageAccounts/websiteinstallersac" Details=[{"Message":"AzureStoragePropertyDictionary is invalid.  ID in AzureStoragePropertyDictionary contains invalid characters: /subscriptions/3b92ad75-8bb4-44a3-92df-394bc15085ef/resourceGroups/Classroom_In_The_Cloud_Terraform/providers/Microsoft.Storage/storageAccounts/websiteinstallersac"},{"Code":"BadRequest"},{"ErrorEntity":{"Code":"BadRequest","ExtendedCode":"51021","Message":"AzureStoragePropertyDictionary is invalid.  ID in AzureStoragePropertyDictionary contains invalid characters: /subscriptions/3b92ad75-8bb4-44a3-92df-394bc15085ef/resourceGroups/Classroom_In_The_Cloud_Terraform/providers/Microsoft.Storage/storageAccounts/websiteinstallersac","MessageTemplate":"{0} is invalid.  {1}","Parameters":["AzureStoragePropertyDictionary","ID in AzureStoragePropertyDictionary contains invalid characters: /subscriptions/3b92ad75-8bb4-44a3-92df-394bc15085ef/resourceGroups/Classroom_In_The_Cloud_Terraform/providers/Microsoft.Storage/storageAccounts/websiteinstallersac"]}}]

Steps to Reproduce

  1. terraform apply

Important Factoids

References

jasric89 commented 2 years ago

Please can I get an update on this bug?

heoelri commented 2 years ago

I think I'm facing a similar issue with my Linux container on AppSvc. I'm now trying Log Analytics (via Diagnostic Settings instead) but I'm curious to know if this should work or not. 

resource "azurerm_app_service_plan" "deployment" {
  name                = random_pet.deployment.id
  location            = azurerm_resource_group.deployment.location
  resource_group_name = azurerm_resource_group.deployment.name

  kind = "Linux"

  reserved = true

  sku {
    tier = "Basic"
    size = "B1"
  }
}

resource "azurerm_app_service" "deployment" {
  name                = random_pet.deployment.id
  location            = azurerm_resource_group.deployment.location
  resource_group_name = azurerm_resource_group.deployment.name
  app_service_plan_id = azurerm_app_service_plan.deployment.id

  identity {
    type = "SystemAssigned"
  }

  logs {
    application_logs {
      azure_blob_storage {
        level             = "Verbose"
        retention_in_days = 7
        sas_url           = format("https://${azurerm_storage_account.deployment.name}.blob.core.windows.net/${azurerm_storage_container.applicationlogs.name}%s", data.azurerm_storage_account_blob_container_sas.applicationlogs.sas)
        #data.azurerm_storage_account_blob_container_sas.applicationlogs.sas
      }
    }
    http_logs {
      azure_blob_storage {
        retention_in_days = 7
        sas_url           = format("https://${azurerm_storage_account.deployment.name}.blob.core.windows.net/${azurerm_storage_container.httplogs.name}%s", data.azurerm_storage_account_blob_container_sas.httplogs.sas)
      }
    }
  }

  site_config {
    acr_use_managed_identity_credentials = true

    linux_fx_version = "DOCKER|${azurerm_container_registry.deployment.name}.azurecr.io/frontend/frontend:latest"
  }

}

resource "azurerm_role_assignment" "acrpull_role" {
  scope                = azurerm_container_registry.deployment.id
  role_definition_name = "AcrPull"
  principal_id         = azurerm_app_service.deployment.identity.0.principal_id
}

resource "azurerm_storage_container" "applicationlogs" {
  name                  = "applicationlogs"
  storage_account_name  = azurerm_storage_account.deployment.name
  container_access_type = "private"
}

data "azurerm_storage_account_blob_container_sas" "applicationlogs" {
  connection_string = azurerm_storage_account.deployment.primary_connection_string
  container_name    = azurerm_storage_container.applicationlogs.name
  https_only        = true

  start  = timestamp()
  expiry = timeadd(timestamp(), "168h")

  permissions {
    read   = true
    add    = true
    create = false
    write  = false
    delete = true
    list   = true
  }
}

This results in a, at a first glance, viable Microsot.Web/sites/config resource:

        {
            "type": "Microsoft.Web/sites/config",
            "apiVersion": "2021-02-01",
            "name": "[concat(parameters('sites_becomingporpoise_name'), '/web')]",
            "location": "West Europe",
            "dependsOn": [
                "[resourceId('Microsoft.Web/sites', parameters('sites_becomingporpoise_name'))]"
            ],
            "properties": {
                "numberOfWorkers": 1,
                "defaultDocuments": [],
                "netFrameworkVersion": "v4.0",
                "linuxFxVersion": "DOCKER|becomingporpoise.azurecr.io/frontend/frontend:latest",
                "requestTracingEnabled": false,
                "remoteDebuggingEnabled": false,
                "remoteDebuggingVersion": "VS2019",
                "httpLoggingEnabled": true,
                "acrUseManagedIdentityCreds": true,
                "logsDirectorySizeLimit": 35,
                "detailedErrorLoggingEnabled": false,
                "publishingUsername": "$becomingporpoise",
                "scmType": "None",
                "use32BitWorkerProcess": false,
                "webSocketsEnabled": false,
                "alwaysOn": false,
                "managedPipelineMode": "Integrated",
                "virtualApplications": [
                    {
                        "virtualPath": "/",
                        "physicalPath": "site\\wwwroot",
                        "preloadEnabled": false
                    }
                ],
                "loadBalancing": "LeastRequests",
                "experiments": {
                    "rampUpRules": []
                },
                "autoHealEnabled": false,
                "vnetRouteAllEnabled": false,
                "vnetPrivatePortsCount": 0,
                "cors": {
                    "allowedOrigins": [],
                    "supportCredentials": false
                },
                "localMySqlEnabled": false,
                "managedServiceIdentityId": 16213,
                "ipSecurityRestrictions": [
                    {
                        "ipAddress": "Any",
                        "action": "Allow",
                        "priority": 1,
                        "name": "Allow all",
                        "description": "Allow all access"
                    }
                ],
                "scmIpSecurityRestrictions": [
                    {
                        "ipAddress": "Any",
                        "action": "Allow",
                        "priority": 1,
                        "name": "Allow all",
                        "description": "Allow all access"
                    }
                ],
                "scmIpSecurityRestrictionsUseMain": false,
                "http20Enabled": false,
                "minTlsVersion": "1.2",
                "scmMinTlsVersion": "1.0",
                "ftpsState": "AllAllowed",
                "preWarmedInstanceCount": 0,
                "functionAppScaleLimit": 0,
                "functionsRuntimeScaleMonitoringEnabled": false,
                "minimumElasticInstanceCount": 0,
                "azureStorageAccounts": {
                    "becomingporpoise": {
                        "type": "AzureBlob",
                        "accountName": "becomingporpoise",
                        "shareName": "becomingporpoise"
                    }
                }
            }
        },

The result in the portal looks like this - http_logs and application_logs are pointing to two different containers in the same storage account - the hcl snipped above contains only one container/sas definition.

image

But, my blob storage container remains empty - even after days:

image

@madsd do you know if this is supposed to work?

madsd commented 2 years ago

Hi, no - I am not deep in the monitoring and diagnostics. Does it work if you configure it through portal or CLI, and can you see a delta between the raw ARM configuration if configured through TF?

jasric89 commented 2 years ago

Can we have an update on this please?

tombuildsstuff commented 2 years ago

The azurerm_app_service resource has been superseded in v3.0 of the Azure Provider and is now feature-frozen to maintain compatibility - as such you'll need to migrate to those new resources instead: https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/3.0-upgrade-guide#new-resources-and-data-sources-for-app-service which I believe should work for you here.

If the new resources don't work for you then please let us know in a new issue and we can take another look, but since the older azurerm_app_service resource has been superseded I'm going to close this issue for the moment.

Thanks!

github-actions[bot] commented 2 years ago

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues. If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.