search

hashicorp / terraform-provider-kubernetes-alpha

A Terraform provider for Kubernetes that uses dynamic resource types and server-side apply. Supports all Kubernetes resources.
https://registry.terraform.io/providers/hashicorp/kubernetes-alpha/latest
Mozilla Public License 2.0
490 stars 63 forks source link

silent failure of terraform apply for kubernetes_manifest resource #240

Open chrisadkin-zz opened 3 years ago

chrisadkin-zz commented 3 years ago

Terraform, Provider, Kubernetes versions

Terraform version: 1.0
Provider version: 0.5
Kubernetes version: 1.20.7

Affected Resource(s)

kubernetes_alpha.terraform_manifest

Terraform Configuration Files

# Copy-paste your Terraform configurations here - for large Terraform configs,
# please use a service like Dropbox and share a link to the ZIP file. For
# security, you can also encrypt the files using our GPG public key.

Debug Output

https://gist.github.com/chrisadkin/ccb3f63e8b09ed2cb6b551b0edbdc0d5

Steps to Reproduce

  1. kubectl create ns arc-ds-controller

  2. terraform apply (for the following configuration)

main.tf

provider "kubernetes" {
  config_path = "~/.kube/config"
}

resource "kubernetes_role" "role_bootstrapper" {
  metadata {
    name      = "role-bootstrapper"
    namespace = "arc-ds-controller"
  }

  rule {
    api_groups = [""]
    resources = ["pods", "configmaps", "services", "persistentvolumeclaims", "secrets", "serviceaccounts", "events"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["apps"]
    resources = ["replicasets", "statefulsets"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["rbac.authorization.k8s.io"]
    resources = ["roles", "rolebindings"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["sql.arcdata.microsoft.com", "tasks.sql.arcdata.microsoft.com", "arcdata.microsoft.com"]
    resources = ["*"]
    verbs     = ["*"]
  }
}

resource "kubernetes_role_binding" "rb_bootstrapper" {
  metadata {
    name      = "rb-bootstrapper"
    namespace = "arc-ds-controller"
  }
  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "Role"
    name      = "role-bootstrapper"
  }
  subject {
    kind      = "ServiceAccount"
    name      = "sa-bootstrapper"
  }

  depends_on = [
    kubernetes_role.role_bootstrapper
  ]
}

resource "kubernetes_service_account" "sa_bootstrapper" {
  metadata {
    name = "sa-bootstrapper"
    namespace = "arc-ds-controller"
  }
}

resource "kubernetes_deployment" "bootstrapper" {
  metadata {
    name = "bootstrapper"
    namespace = "arc-ds-controller"
    labels = {
      app = "bootstrapper"
    }
  }

  spec {
    replicas = 1

    selector {
      match_labels = {
        app = "bootstrapper"
      }
    }

    template {
      metadata {
        labels = {
          app = "bootstrapper"
        }
      }

      spec {
        node_selector = {
          "kubernetes.io/os" = "linux"
        }

        service_account_name = "sa-bootstrapper"

        image_pull_secrets {
          name = "arc-private-registry"
        }

        container {
          image = "mcr.microsoft.com/arcdata/arc-bootstrapper:latest"
          name  = "bootstrapper"
          image_pull_policy = "Always"
          security_context {
            run_as_user = 21000
          }
        }
      }
    }
  }

  depends_on = [
    kubernetes_service_account.sa_bootstrapper
  ]
}

resource "kubernetes_secret" "controller_login_secret" {
  metadata {
    name      = "controller-login-secret"
    namespace = "arc-ds-controller"
  }

  binary_data = {
    "username" = "YXJjdXNlcg=="
    "password" = "T3NtaXVtNzY="
  }

  type = "Opaque"

  depends_on = [
    kubernetes_deployment.bootstrapper 
  ]
}

resource "kubernetes_service_account" "sa_mssql_controller" {
  metadata {
    name = "sa-mssql-controller"
    namespace = "arc-ds-controller"
  }
  secret {
    name = "controller-login-secret"
  }

  depends_on = [
    kubernetes_secret.controller_login_secret 
  ]
}

crd.tf

provider "kubernetes-alpha" {
  config_path = "~/.kube/config"
}

resource "kubernetes_manifest" "data_controller_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "datacontrollers.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "State"
          "type" = "string"
        },
      ]
      "group" = "arcdata.microsoft.com"
      "names" = {
        "kind" = "datacontroller"
        "plural" = "datacontrollers"
      }
      "scope" = "Namespaced"
      "subresources" = {
         "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "sql_mi_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "sqlmanagedinstances.sql.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "Status"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.readyReplicas"
          "name" = "Replicas"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.primaryEndpoint"
          "name" = "Primary-Endpoint"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "sql.arcdata.microsoft.com"
      "names" = {
        "kind" = "sqlmanagedinstance"
        "plural" = "sqlmanagedinstances"
        "shortNames" = [
          "sqlmi",
        ]
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "sql_mi_restore_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "sqlmanagedinstancerestoretasks.tasks.sql.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "Status"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "tasks.sql.arcdata.microsoft.com"
      "names" = {
        "kind" = "SqlManagedInstanceRestoreTask"
        "plural" = "sqlmanagedinstancerestoretasks"
        "shortNames" = [
          "sqlmirestoretask",
        ]
        "singular" = "sqlmanagedinstancerestoretask"
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "postgres_sql_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "postgresqls.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "State"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.readyPods"
          "name" = "Ready-Pods"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.primaryEndpoint"
          "name" = "Primary-Endpoint"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "arcdata.microsoft.com"
      "names" = {
        "kind" = "postgresql"
        "plural" = "postgresqls"
        "shortNames" = [
          "postgres",
        ]
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}
  1. terraform apply (for the following configuration): 
    
provider "kubernetes-alpha" {
config_path = "~/.kube/config" 
}

resource "kubernetes_manifest" "arc" { provider = kubernetes-alpha

manifest = { "apiVersion" = "arcdata.microsoft.com/v1alpha1" "kind" = "datacontroller" "metadata" = { "name" = "arc" "namespace" = "arc-ds-controller" } "spec" = { "credentials" = { "controllerAdmin" = "controller-login-secret" "serviceAccount" = "sa-mssql-controller" } "docker" = { "imagePullPolicy" = "Always" "imageTag" = "latest" "registry" = "mcr.microsoft.com" "repository" = "arcdata" } "security" = { "allowDumps" = true "allowNodeMetricsCollection" = true "allowPodMetricsCollection" = true "allowRunAsRoot" = false } "services" = [ { "name" = "controller" "port" = 30080 "serviceType" = "LoadBalancer" }, { "name" = "serviceProxy" "port" = 30777 "serviceType" = "LoadBalancer" }, ] "settings" = { "ElasticSearch" = { "vm.max_map_count" = "-1" } "azure" = { "connectionMode" = "indirect" "displayName" = "arc" "enableBilling" = "True" "location" = "eastus" "logs.rotation.days" = "7" "logs.rotation.size" = "5000" "resourceGroup" = "AzureArcTestEastUS" "subscription" = "< Put your own subscription string here>" } } "storage" = { "data" = { "accessMode" = "ReadWriteOnce" "className" = "portworx-sc" "size" = "15Gi" } "logs" = { "accessMode" = "ReadWriteOnce" "className" = "portworx-sc" "size" = "10Gi" } } } } }

### Expected Behavior

When I create the Kubernetes objects using the YAML from this article https://docs.microsoft.com/en-us/azure/azure-arc/data/create-data-controller-using-kubernetes-native-tools and issue:

```kubectl get all -n arc-ds-controller```

this is what I see:

NAME READY STATUS RESTARTS AGE pod/bootstrapper-lxlzl 1/1 Running 0 3m26s pod/control-xxxgw 2/2 Running 0 11s pod/controldb-0 2/2 Terminating 0 11s

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/controldb-svc ClusterIP 10.233.42.205 1433/TCP,8311/TCP,8411/TCP 12s service/controller-svc ClusterIP 10.233.22.220 443/TCP,8311/TCP,8301/TCP,8411/TCP,8401/TCP 12s service/controller-svc-external LoadBalancer 10.233.59.180 10.XXX.YYY.93 30080:32754/TCP 12s

NAME DESIRED CURRENT READY AGE replicaset.apps/bootstrapper 1 1 1 3m26s replicaset.apps/control 1 1 1 11s

NAME READY AGE statefulset.apps/controldb 1/1 11s


### Actual Behavior

When I issue a kubectl get all -n arc-ds-controller after applying the Terraform configurations, I see:

NAME READY STATUS RESTARTS AGE pod/bootstrapper-d7f9fcc65-x6sgl 1/1 Running 0 36m

NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/bootstrapper 1/1 1 1 36m

NAME DESIRED CURRENT READY AGE replicaset.apps/bootstrapper-d7f9fcc65 1 1 1 36m



### Important Factoids
- I'm running Kubernetes 1.20.7 that has been deployed to Ubuntu virtual machines running on VMware via Kubespray
- terraform plan reports zero issues for any of config except for a warning about the fact that the ability to specify password for
Azure service principals will soon be deprecated

### References
NA

### Community Note
<!--- Please keep this note for the community --->
* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
* If you are interested in working on this issue or have submitted a pull request, please leave a comment