rancher / terraform-provider-rancher2

Terraform Rancher2 provider
https://www.terraform.io/docs/providers/rancher2/
Mozilla Public License 2.0
253 stars 216 forks source link

Failure when updating rancher2_cluster_template #524

Open caiconkhicon opened 3 years ago

caiconkhicon commented 3 years ago

Rancher2 provider version: 1.10.6

I have an RKE template with 6 revisions:

Annotation 2020-11-16 153509

I update it by replacing the first 2 revisions (1.16.6 and 1.17.2) in the TF code by 2 new revisions. Then I run terraform plan and terraform apply.

terraform plan shows the changelog:

  # module.rancher.rancher2_cluster_template.default will be updated in-place
  ~ resource "rancher2_cluster_template" "default" {
        annotations         = {}
        default_revision_id = "cattle-global-data:ctr-qmh4p"
        description         = "K8S cluster template with hardening configuration"
        id                  = "cattle-global-data:ct-f86d2"
        labels              = {
            "cattle.io/creator" = "norman"
        }
        name                = "default"
        members {
            access_type = "owner"
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
          - enabled             = true -> null
            id                  = "cattle-global-data:ctr-rs2mh"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          - name                = "v.1.16.6-rancher1-2" -> null
          - cluster_config {
              - default_pod_security_policy_template_id = "restricted" -> null
              - docker_root_dir                         = "/var/lib/docker" -> null
              - enable_cluster_alerting                 = false -> null
              - enable_cluster_monitoring               = false -> null
              - enable_network_policy                   = false -> null
              - windows_prefered_cluster                = false -> null
              - rke_config {
                  - addon_job_timeout     = 30 -> null
                  - addons_include        = [] -> null
                  - ignore_docker_version = true -> null
                  - kubernetes_version    = "v1.18.3-rancher2-2" -> null
                  - ssh_agent_auth        = false -> null
                  - network {
                      - mtu     = 0 -> null
                      - options = {} -> null
                      - plugin  = "canal" -> null
                    }
                  - services {
                      - etcd {
                          - creation      = "12h" -> null
                          - external_urls = [] -> null
                          - extra_args    = {
                              - "client-cert-auth"      = "true"
                              - "peer-client-cert-auth" = "true"
                            } -> null
                          - extra_binds   = [] -> null
                          - extra_env     = [] -> null
                          - gid           = 0 -> null
                          - retention     = "72h" -> null
                          - snapshot      = false -> null
                          - uid           = 0 -> null
                        }
                      - kube_api {
                          - admission_configuration = {} -> null
                          - always_pull_images      = false -> null
                          - extra_args              = {} -> null
                          - extra_binds             = [] -> null
                          - extra_env               = [] -> null
                          - pod_security_policy     = true -> null
                          - service_node_port_range = "30000-32767" -> null
                          - audit_log {
                              - enabled = true -> null
                              - configuration {
                                  - format     = "json" -> null
                                  - max_age    = 30 -> null
                                  - max_backup = 10 -> null
                                  - max_size   = 100 -> null
                                  - path       = "/var/log/kube-audit/audit-log.json" -> null
                                }
                            }
                          - event_rate_limit {
                              - enabled = true -> null
                            }
                          - secrets_encryption_config {
                              - enabled = true -> null
                            }
                        }
                      - kube_controller {
                          - extra_args  = {
                              - "address"                     = "127.0.0.1"
                              - "feature-gates"               = "RotateKubeletServerCertificate=true"
                              - "profiling"                   = "false"
                              - "terminated-pod-gc-threshold" = "1000"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                      - kubelet {
                          - extra_args                   = {
                              - "feature-gates"     = "RotateKubeletServerCertificate=true"
                              - "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            } -> null
                          - extra_binds                  = [] -> null
                          - extra_env                    = [] -> null
                          - fail_swap_on                 = false -> null
                          - generate_serving_certificate = true -> null
                        }
                      - scheduler {
                          - extra_args  = {
                              - "address"   = "127.0.0.1"
                              - "profiling" = "false"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                    }
                  - upgrade_strategy {
                      - drain                        = false -> null
                      - max_unavailable_controlplane = "1" -> null
                      - max_unavailable_worker       = "10%" -> null
                    }
                }
            }
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
          - enabled             = true -> null
            id                  = "cattle-global-data:ctr-x5775"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          - name                = "v1.17.2-rancher1-2" -> null
          - cluster_config {
              - default_pod_security_policy_template_id = "restricted" -> null
              - docker_root_dir                         = "/var/lib/docker" -> null
              - enable_cluster_alerting                 = false -> null
              - enable_cluster_monitoring               = false -> null
              - enable_network_policy                   = false -> null
              - windows_prefered_cluster                = false -> null
              - rke_config {
                  - addon_job_timeout     = 30 -> null
                  - addons_include        = [] -> null
                  - ignore_docker_version = true -> null
                  - kubernetes_version    = "v1.17.2-rancher1-2" -> null
                  - ssh_agent_auth        = false -> null
                  - ingress {
                      - extra_args    = {} -> null
                      - node_selector = {} -> null
                      - options       = {
                          - "http2-max-field-size"   = "16k"
                          - "log-format-escape-json" = "true"
                          - "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                          - "proxy-body-size"        = "900m"
                        } -> null
                      - provider      = "nginx" -> null
                    }
                  - network {
                      - mtu     = 0 -> null
                      - options = {} -> null
                      - plugin  = "canal" -> null
                    }
                  - services {
                      - etcd {
                          - creation      = "12h" -> null
                          - external_urls = [] -> null
                          - extra_args    = {
                              - "client-cert-auth"      = "true"
                              - "peer-client-cert-auth" = "true"
                            } -> null
                          - extra_binds   = [] -> null
                          - extra_env     = [] -> null
                          - gid           = 0 -> null
                          - retention     = "72h" -> null
                          - snapshot      = false -> null
                          - uid           = 0 -> null
                        }
                      - kube_api {
                          - admission_configuration = {} -> null
                          - always_pull_images      = false -> null
                          - extra_args              = {} -> null
                          - extra_binds             = [] -> null
                          - extra_env               = [] -> null
                          - pod_security_policy     = true -> null
                          - service_node_port_range = "30000-32767" -> null
                          - audit_log {
                              - enabled = true -> null
                              - configuration {
                                  - format     = "json" -> null
                                  - max_age    = 30 -> null
                                  - max_backup = 10 -> null
                                  - max_size   = 100 -> null
                                  - path       = "/var/log/kube-audit/audit-log.json" -> null
                                }
                            }
                          - event_rate_limit {
                              - enabled = true -> null
                            }
                          - secrets_encryption_config {
                              - enabled = true -> null
                            }
                        }
                      - kube_controller {
                          - extra_args  = {
                              - "address"                     = "127.0.0.1"
                              - "feature-gates"               = "RotateKubeletServerCertificate=true"
                              - "profiling"                   = "false"
                              - "terminated-pod-gc-threshold" = "1000"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                      - kubelet {
                          - extra_args                   = {
                              - "feature-gates"     = "RotateKubeletServerCertificate=true"
                              - "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            } -> null
                          - extra_binds                  = [
                              - "/var/openebs/local:/var/openebs/local",
                            ] -> null
                          - extra_env                    = [] -> null
                          - fail_swap_on                 = false -> null
                          - generate_serving_certificate = true -> null
                        }
                      - scheduler {
                          - extra_args  = {
                              - "address"   = "127.0.0.1"
                              - "profiling" = "false"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                    }
                  - upgrade_strategy {
                      - drain                        = false -> null
                      - max_unavailable_controlplane = "1" -> null
                      - max_unavailable_worker       = "10%" -> null
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-bqnfh"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.18.3-rancher2-2"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.18.3-rancher2-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {}
                        node_selector = {}
                        options       = {
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-b52fp"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.18.3-rancher2-3"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.18.3-rancher2-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {}
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
          ~ default             = false -> true
            enabled             = true
          ~ id                  = "cattle-global-data:ctr-bvrgd" -> (known after apply)
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          ~ name                = "v1.18.3-rancher2-4" -> "v1.18.3-rancher2-6"
          ~ cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
              ~ rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                  ~ kubernetes_version    = "v1.18.3-rancher2-2" -> "v1.18.9-rancher1-1"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {
                            "tcp-services-configmap" = "$(POD_NAMESPACE)/tcp-services-custom"
                        }
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                  ~ services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                      ~ kubelet {
                          ~ extra_args                   = {
                              + "eviction-hard"            = "memory.available
                              + "eviction-minimum-reclaim" = "memory.available=512Mi"
                                "feature-gates"            = "RotateKubeletServerCertificate=true"
                              + "kube-reserved"            = "cpu=500m,memory=1512Mi"
                              + "system-reserved"          = "cpu=500m,memory=512Mi"
                                "tls-cipher-suites"        = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
          ~ default             = true -> false
            enabled             = true
          ~ id                  = "cattle-global-data:ctr-qmh4p" -> (known after apply)
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          ~ name                = "v1.18.3-rancher2-5" -> "v1.18.9-rancher1-1"
          ~ cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
              ~ rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                  ~ kubernetes_version    = "v1.18.3-rancher2-2" -> "v1.18.9-rancher1-1"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {
                            "tcp-services-configmap" = "$(POD_NAMESPACE)/tcp-services-custom"
                        }
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                  + private_registries {
                      + is_default = true
                      + password   = (sensitive value)
                      + url        = "docker.mydomain.custom"
                      + user       = (sensitive value)
                    }
                  ~ services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                      ~ kubelet {
                          ~ extra_args                   = {
                              + "eviction-hard"            = "memory.available
                              + "eviction-minimum-reclaim" = "memory.available=512Mi"
                                "feature-gates"            = "RotateKubeletServerCertificate=true"
                              + "kube-reserved"            = "cpu=500m,memory=1512Mi"
                              + "system-reserved"          = "cpu=500m,memory=512Mi"
                                "tls-cipher-suites"        = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
    }

terrraform apply returns an error:

Error: [ERROR] Expanding cluster template revisions: name "" is repeated
  on ../../modules/rancher/rancher_config.tf line 16, in resource "rancher2_cluster_template" "default":
  16: resource "rancher2_cluster_template" "default" {

I don't know what's is the issue and where/how to debug it.

caiconkhicon commented 3 years ago

One more thing I noticed from the output of the plan: the replaced revisions are wrong. They should be 1.16.6 and 1.17.2, not 1.18.3-rancher-2-5 and 1.18.3-rancher-2-6 (plus 1.16.6 and 1.17.2 are said to be deleted, not replaced).

rawmind0 commented 3 years ago

@caiconkhicon , template revisions are tricky due to they are managed by a terraform list, what it means that it's dependant of the order. Provider is shifting the list to set proper order, but sometimes the plan showed is not showed in an accurated way, but executing it fine.

Anyway, it seems there is a bug in the provider removing more than one template revision at once. You are trying to remove 2 template revisions, isn't it?? If so, as workaround, could you please test if removing one, applying and removing the other one and applying is working fine??

In the meanwhile, we'll work on a fix for it.

caiconkhicon commented 3 years ago

@rawmind0 : thanks for your fast reply. I try it now and give you feedback soon.

caiconkhicon commented 3 years ago

@rawmind0 : I tested like you suggested, so I reverted the change with the revision 1.17.2. Now terraform apply succeeded, but after that I check on the Rancher UI and nothing has changed!? Do you have any idea?

terraform plan

  # module.rancher.rancher2_cluster_template.default will be updated in-place
  ~ resource "rancher2_cluster_template" "default" {
        annotations         = {}
        default_revision_id = "cattle-global-data:ctr-qmh4p"
        description         = "K8S cluster template with hardening configuration"
        id                  = "cattle-global-data:ct-f86d2"
        labels              = {
            "cattle.io/creator" = "norman"
        }
        name                = "default"
        members {
            access_type = "owner"
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
          - enabled             = true -> null
            id                  = "cattle-global-data:ctr-rs2mh"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          - name                = "v.1.16.6-rancher1-2" -> null
          - cluster_config {
              - default_pod_security_policy_template_id = "restricted" -> null
              - docker_root_dir                         = "/var/lib/docker" -> null
              - enable_cluster_alerting                 = false -> null
              - enable_cluster_monitoring               = false -> null
              - enable_network_policy                   = false -> null
              - windows_prefered_cluster                = false -> null
              - rke_config {
                  - addon_job_timeout     = 30 -> null
                  - addons_include        = [] -> null
                  - ignore_docker_version = true -> null
                  - kubernetes_version    = "v1.18.3-rancher2-2" -> null
                  - ssh_agent_auth        = false -> null
                  - network {
                      - mtu     = 0 -> null
                      - options = {} -> null
                      - plugin  = "canal" -> null
                    }
                  - services {
                      - etcd {
                          - creation      = "12h" -> null
                          - external_urls = [] -> null
                          - extra_args    = {
                              - "client-cert-auth"      = "true"
                              - "peer-client-cert-auth" = "true"
                            } -> null
                          - extra_binds   = [] -> null
                          - extra_env     = [] -> null
                          - gid           = 0 -> null
                          - retention     = "72h" -> null
                          - snapshot      = false -> null
                          - uid           = 0 -> null
                        }
                      - kube_api {
                          - admission_configuration = {} -> null
                          - always_pull_images      = false -> null
                          - extra_args              = {} -> null
                          - extra_binds             = [] -> null
                          - extra_env               = [] -> null
                          - pod_security_policy     = true -> null
                          - service_node_port_range = "30000-32767" -> null
                          - audit_log {
                              - enabled = true -> null
                              - configuration {
                                  - format     = "json" -> null
                                  - max_age    = 30 -> null
                                  - max_backup = 10 -> null
                                  - max_size   = 100 -> null
                                  - path       = "/var/log/kube-audit/audit-log.json" -> null
                                }
                            }
                          - event_rate_limit {
                              - enabled = true -> null
                            }
                          - secrets_encryption_config {
                              - enabled = true -> null
                            }
                        }
                      - kube_controller {
                          - extra_args  = {
                              - "address"                     = "127.0.0.1"
                              - "feature-gates"               = "RotateKubeletServerCertificate=true"
                              - "profiling"                   = "false"
                              - "terminated-pod-gc-threshold" = "1000"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                      - kubelet {
                          - extra_args                   = {
                              - "feature-gates"     = "RotateKubeletServerCertificate=true"
                              - "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            } -> null
                          - extra_binds                  = [] -> null
                          - extra_env                    = [] -> null
                          - fail_swap_on                 = false -> null
                          - generate_serving_certificate = true -> null
                        }
                      - scheduler {
                          - extra_args  = {
                              - "address"   = "127.0.0.1"
                              - "profiling" = "false"
                            } -> null
                          - extra_binds = [] -> null
                          - extra_env   = [] -> null
                        }
                    }
                  - upgrade_strategy {
                      - drain                        = false -> null
                      - max_unavailable_controlplane = "1" -> null
                      - max_unavailable_worker       = "10%" -> null
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-x5775"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.17.2-rancher1-2"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.17.2-rancher1-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {}
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-bqnfh"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.18.3-rancher2-2"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.18.3-rancher2-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {}
                        node_selector = {}
                        options       = {
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-b52fp"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.18.3-rancher2-3"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.18.3-rancher2-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {}
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
        template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = false
            enabled             = true
            id                  = "cattle-global-data:ctr-bvrgd"
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
            name                = "v1.18.3-rancher2-4"
            cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
                rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                    kubernetes_version    = "v1.18.3-rancher2-2"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {
                            "tcp-services-configmap" = "$(POD_NAMESPACE)/tcp-services-custom"
                        }
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                    services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                        kubelet {
                            extra_args                   = {
                                "feature-gates"     = "RotateKubeletServerCertificate=true"
                                "tls-cipher-suites" = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
      ~ template_revisions {
            annotations         = {}
            cluster_template_id = "cattle-global-data:ct-f86d2"
            default             = true
            enabled             = true
          ~ id                  = "cattle-global-data:ctr-qmh4p" -> (known after apply)
            labels              = {
                "io.cattle.field/clusterTemplateId" = "ct-f86d2"
            }
          ~ name                = "v1.18.3-rancher2-5" -> "v1.18.3-rancher2-6"
          ~ cluster_config {
                default_pod_security_policy_template_id = "restricted"
                docker_root_dir                         = "/var/lib/docker"
                enable_cluster_alerting                 = false
                enable_cluster_monitoring               = false
                enable_network_policy                   = false
                windows_prefered_cluster                = false
              ~ rke_config {
                    addon_job_timeout     = 30
                    addons_include        = []
                    ignore_docker_version = true
                  ~ kubernetes_version    = "v1.18.3-rancher2-2" -> "v1.18.9-rancher1-1"
                    ssh_agent_auth        = false
                    ingress {
                        extra_args    = {
                            "tcp-services-configmap" = "$(POD_NAMESPACE)/tcp-services-custom"
                        }
                        node_selector = {}
                        options       = {
                            "http2-max-field-size"   = "16k"
                            "log-format-escape-json" = "true"
                            "log-format-upstream"    = "{\\\"time\\\": \\\"$time_iso8601\\\", \\\"remote_addr\\\": \\\"$proxy_protocol_addr\\\", \\\"x-forward-for\\\": \\\"$proxy_add_x_forwarded_for\\\", \\\"request_id\\\": \\\"$req_id\\\", \\\"remote_user\\\": \\\"$remote_user\\\", \\\"bytes_sent\\\": $bytes_sent, \\\"request_time\\\": $request_time, \\\"status\\\":$status, \\\"vhost\\\": \\\"$host\\\", \\\"request_proto\\\": \\\"$server_protocol\\\", \\\"path\\\": \\\"$uri\\\", \\\"request_query\\\": \\\"$args\\\", \\\"request_length\\\": $request_length, \\\"duration\\\": $request_time,\\\"method\\\": \\\"$request_method\\\", \\\"http_referrer\\\": \\\"$http_referer\\\", \\\"http_user_agent\\\": \\\"$http_user_agent\\\" }"
                            "proxy-body-size"        = "900m"
                        }
                        provider      = "nginx"
                    }
                    network {
                        mtu     = 0
                        options = {}
                        plugin  = "canal"
                    }
                  ~ services {
                        etcd {
                            creation      = "12h"
                            external_urls = []
                            extra_args    = {
                                "client-cert-auth"      = "true"
                                "peer-client-cert-auth" = "true"
                            }
                            extra_binds   = []
                            extra_env     = []
                            gid           = 0
                            retention     = "72h"
                            snapshot      = false
                            uid           = 0
                        }
                        kube_api {
                            admission_configuration = {}
                            always_pull_images      = false
                            extra_args              = {}
                            extra_binds             = []
                            extra_env               = []
                            pod_security_policy     = true
                            service_node_port_range = "30000-32767"
                            audit_log {
                                enabled = true
                                configuration {
                                    format     = "json"
                                    max_age    = 30
                                    max_backup = 10
                                    max_size   = 100
                                    path       = "/var/log/kube-audit/audit-log.json"
                                }
                            }
                            event_rate_limit {
                                enabled = true
                            }
                            secrets_encryption_config {
                                enabled = true
                            }
                        }
                        kube_controller {
                            extra_args  = {
                                "address"                     = "127.0.0.1"
                                "feature-gates"               = "RotateKubeletServerCertificate=true"
                                "profiling"                   = "false"
                                "terminated-pod-gc-threshold" = "1000"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                      ~ kubelet {
                          ~ extra_args                   = {
                              + "eviction-hard"            = "memory.available
                              + "eviction-minimum-reclaim" = "memory.available=512Mi"
                                "feature-gates"            = "RotateKubeletServerCertificate=true"
                              + "kube-reserved"            = "cpu=500m,memory=1512Mi"
                              + "system-reserved"          = "cpu=500m,memory=512Mi"
                                "tls-cipher-suites"        = "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256"
                            }
                            extra_binds                  = [
                                "/var/openebs/local:/var/openebs/local",
                            ]
                            extra_env                    = []
                            fail_swap_on                 = false
                            generate_serving_certificate = true
                        }
                        scheduler {
                            extra_args  = {
                                "address"   = "127.0.0.1"
                                "profiling" = "false"
                            }
                            extra_binds = []
                            extra_env   = []
                        }
                    }
                    upgrade_strategy {
                        drain                        = false
                        max_unavailable_controlplane = "1"
                        max_unavailable_worker       = "10%"
                    }
                }
            }
        }
    }

terraform apply

module.rancher.rancher2_cluster_template.default: Modifying... [id=cattle-global-data:ct-f86d2]
module.rancher.rancher2_cluster_template.default: Modifications complete after 2s [id=cattle-global-data:ct-f86d2]