search

vmware / terraform-provider-tanzu-mission-control

Terraform Provider for VMware Tanzu Mission Control
https://registry.terraform.io/providers/vmware/tanzu-mission-control/latest/docs
Mozilla Public License 2.0
37 stars 33 forks source link

TMC Rate Limiting issue #241

Open vrabbi opened 1 year ago

vrabbi commented 1 year ago

Describe the bug

when applying large configurations, rate limiting can often be encountered.

Reproduction steps

  1. create a terraform module with around 50 resources
  2. try to apply
  3. try to delete

Expected behavior

a retry logic should be added to the provider or a field on the provider to specify concurrency may help as well.

Additional context

No response

ramya-bangera commented 1 year ago

@vrabbi - Can you share the reference TF file and a screenshot of the error which you are hitting?

vrabbi commented 1 year ago

terraform example file: 

variable "workspaces" {
  type = list(object({
    name         = string
    description  = string
    view_groups  = optional(list(string))
    edit_groups  = optional(list(string))
    admin_groups = optional(list(string))
    labels       = optional(map(string), { "tf-managed" = "yes" })
  }))
}

variable "clusters" {
  type = list(string)
}

variable "allowed_registry_hostname" {
  type = string
}

variable "ingress_controller_namespace" {
  type = string
  default = "tanzu-system-ingress"
}

variable "coredns_namespace" {
  type = string
  default = "kube-system"
}

variable "coredns_pod_selector" {
  type = map(string)
  default = {
    "k8s-app" = "kube-dns"
  }
}

data "tanzu-mission-control_cluster" "demo" {
  for_each = toset(var.clusters)
  name     = each.value
}
resource "tanzu-mission-control_iam_policy" "demo_operator" {
  depends_on = [
    tanzu-mission-control_workspace.demo
  ]
  count = length(var.workspaces)
  scope {
    workspace {
      name = var.workspaces[count.index].name
    }
  }
  role_bindings {
    role = "namespace.admin"

    dynamic "subjects" {
      for_each = toset(var.workspaces[count.index]["admin_groups"])
      content {
        name = subjects.value
        kind = "GROUP"
      }
    }
  }
}

resource "tanzu-mission-control_iam_policy" "demo_editor" {
  depends_on = [
    tanzu-mission-control_iam_policy.demo_operator
  ]
  count = length(var.workspaces)
  scope {
    workspace {
      name = var.workspaces[count.index].name
    }
  }
  role_bindings {
    role = "namespace.edit"

    dynamic "subjects" {
      for_each = toset(var.workspaces[count.index]["edit_groups"])
      content {
        name = subjects.value
        kind = "GROUP"
      }
    }
  }
}

resource "tanzu-mission-control_iam_policy" "demo_viewer" {
  depends_on = [
    tanzu-mission-control_iam_policy.demo_editor
  ]
  count = length(var.workspaces)
  scope {
    workspace {
      name = var.workspaces[count.index].name
    }
  }
  role_bindings {
    role = "namespace.view"

    dynamic "subjects" {
      for_each = toset(var.workspaces[count.index]["view_groups"])
      content {
        name = subjects.value
        kind = "GROUP"
      }
    }
  }
}

resource "tanzu-mission-control_image_policy" "workspace_scoped_custom_image_policy" {
  name = "secure-image-policy"

  depends_on = [
    tanzu-mission-control_network_policy.coredns_egress
  ]

  count = length(var.workspaces)

  scope {
    workspace {
      workspace = var.workspaces[count.index].name
    }
  }

  spec {
    input {
      custom {
        audit = false
        rules {
          hostname = var.allowed_registry_hostname
          imagename = "*"
          port = "*"
          requiredigest = true
        }
      }
    }
  }
}

resource "tanzu-mission-control_network_policy" "ingress_controller_only" {
  name = "allow-ingress-only-from-ingress-controller"

  depends_on = [
    tanzu-mission-control_iam_policy.demo_viewer
  ]

  count = length(var.workspaces)

  scope {
    workspace {
      workspace = var.workspaces[count.index].name
    }
  }

  spec {
    input {
      custom_ingress {
        to_pod_labels = {
          "ingress-needed" = "true"
        }
        rules {
          ports {}
          rule_spec {
            custom_selector {
              namespace_selector = {
                "kubernetes.io/metadata.name" = var.ingress_controller_namespace
              }
            }
          }
        }
      }
    }

  }
}

resource "tanzu-mission-control_network_policy" "coredns_egress" {
  name = "allow-egress-only-to-coredns"

  depends_on = [
    tanzu-mission-control_network_policy.ingress_controller_only
  ]

  count = length(var.workspaces)

  scope {
    workspace {
      workspace = var.workspaces[count.index].name
    }
  }

  spec {
    input {
      custom_egress {
        to_pod_labels = {"dns-needed" = "true"}
        rules {
          ports {
            port = "53"
            protocol = "TCP"
          }
          ports {
            port = "53"
            protocol = "UDP"
          }
          rule_spec {
            custom_selector {
              namespace_selector = {
                "kubernetes.io/metadata.name" = var.coredns_namespace
              }
              pod_selector = var.coredns_pod_selector
            }
          }
        }
      }
    }
  }
}

locals {
  namespaces = distinct(flatten([
    for cluster in data.tanzu-mission-control_cluster.demo : [
      for workspace in var.workspaces : {
        description        = workspace["description"]
        name               = workspace["name"]
        cluster            = cluster.name
        provisioner        = cluster.provisioner_name
        management_cluster = cluster.management_cluster_name
        labels             = workspace["labels"]

      }
    ]
  ]))
}

resource "tanzu-mission-control_workspace" "demo" {
  for_each = { for entry in var.workspaces : "${entry.description}.${entry.name}" => entry }
  name     = each.value.name

  meta {
    description = each.value.description
    labels      = each.value.labels
  }
}

resource "tanzu-mission-control_namespace" "demo" {
  depends_on = [
    tanzu-mission-control_workspace.demo
  ]
  for_each                = { for entry in local.namespaces : "${entry.description}.${entry.name}.${entry.cluster}.${entry.provisioner}.${entry.management_cluster}" => entry }
  name                    = each.value.name
  cluster_name            = each.value.cluster
  provisioner_name        = each.value.provisioner
  management_cluster_name = each.value.management_cluster

  meta {
    description = each.value.description
    labels      = each.value.labels
  }

  spec {
    workspace_name = each.value.name
  }
}

example tfvars: 

workspaces = [
  {
    name         = "sec-ns-01"
    description  = "first app demo"
    view_groups  = ["ts-view"]
    edit_groups  = ["ts-edit"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "test" = "value"
    }
  },
  {
    name         = "sec-ns-02"
    description  = "app demo 2"
    view_groups  = ["monitoring-team-02"]
    edit_groups  = ["dev-team-02"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "sample-label" = "awesome-value"
    }
  },
  {
    name         = "sec-ns-03"
    description  = "first app demo"
    view_groups  = ["ts-view"]
    edit_groups  = ["ts-edit"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "test" = "value"
    }
  },
  {
    name         = "sec-ns-04"
    description  = "app demo 2"
    view_groups  = ["monitoring-team-02"]
    edit_groups  = ["dev-team-02"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "sample-label" = "awesome-value"
    }
  },
  {
    name         = "sec-ns-05"
    description  = "first app demo"
    view_groups  = ["ts-view"]
    edit_groups  = ["ts-edit"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "test" = "value"
    }
  },
  {
    name         = "sec-ns-06"
    description  = "app demo 2"
    view_groups  = ["monitoring-team-02"]
    edit_groups  = ["dev-team-02"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "sample-label" = "awesome-value"
    }
  },
  {
    name         = "sec-ns-07"
    description  = "first app demo"
    view_groups  = ["ts-view"]
    edit_groups  = ["ts-edit"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "test" = "value"
    }
  },
  {
    name         = "sec-ns-08"
    description  = "app demo 2"
    view_groups  = ["monitoring-team-02"]
    edit_groups  = ["dev-team-02"]
    admin_groups = ["ts-devops", "ts-admins"]
    labels = {
      "sample-label" = "awesome-value"
    }
  }
]
clusters = [
  "tap-16-cls"
]
allowed_registry_hostname = "harbor.vrabbi.cloud"

output: 

Plan: 64 to add, 0 to change, 0 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

tanzu-mission-control_workspace.demo["first app demo.sec-ns-07"]: Creating...
tanzu-mission-control_workspace.demo["first app demo.sec-ns-03"]: Creating...
tanzu-mission-control_workspace.demo["first app demo.sec-ns-01"]: Creating...
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-04"]: Creating...
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-02"]: Creating...
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-06"]: Creating...
tanzu-mission-control_workspace.demo["first app demo.sec-ns-05"]: Creating...
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-08"]: Creating...
tanzu-mission-control_workspace.demo["first app demo.sec-ns-03"]: Creation complete after 0s [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-02"]: Creation complete after 0s [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-01"]: Creation complete after 0s [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-05"]: Creation complete after 0s [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-04"]: Creation complete after 0s [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-08"]: Creation complete after 0s [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-07"]: Creation complete after 0s [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-06"]: Creation complete after 0s [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-04.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-02.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-07.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_iam_policy.demo_operator[0]: Creating...
tanzu-mission-control_iam_policy.demo_operator[6]: Creating...
tanzu-mission-control_iam_policy.demo_operator[5]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-03.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_iam_policy.demo_operator[3]: Creating...
tanzu-mission-control_iam_policy.demo_operator[4]: Creating...
tanzu-mission-control_iam_policy.demo_operator[1]: Creating...
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-02.tap-16-cls.attached.attached"]: Creation complete after 1s [id=ns:01H9FJHCVNN7X9868KV4ECST3W]
tanzu-mission-control_iam_policy.demo_operator[5]: Creation complete after 1s [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_operator[3]: Creation complete after 1s [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_iam_policy.demo_operator[7]: Creating...
tanzu-mission-control_iam_policy.demo_operator[6]: Creation complete after 1s [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-04.tap-16-cls.attached.attached"]: Creation complete after 1s [id=ns:01H9FJHCW6G1TE4DKWDKE8NJ9Y]
tanzu-mission-control_iam_policy.demo_operator[1]: Creation complete after 1s [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_operator[2]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-01.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-07.tap-16-cls.attached.attached"]: Creation complete after 1s [id=ns:01H9FJHCVGF4Q9XTFBCRQCNVNV]
tanzu-mission-control_iam_policy.demo_operator[0]: Creation complete after 1s [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_operator[4]: Creation complete after 1s [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-06.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-08.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-05.tap-16-cls.attached.attached"]: Creating...
tanzu-mission-control_namespace.demo["first app demo.sec-ns-03.tap-16-cls.attached.attached"]: Creation complete after 1s [id=ns:01H9FJHCWBXPQT7HTXNQTC82R3]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-05.tap-16-cls.attached.attached"]: Creation complete after 0s [id=ns:01H9FJHD5SYKSNHTC2G32R5XAR]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-08.tap-16-cls.attached.attached"]: Creation complete after 0s [id=ns:01H9FJHD89X2P424EGDTYG465C]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-06.tap-16-cls.attached.attached"]: Creation complete after 0s [id=ns:01H9FJHD943AB7AVF18N8HV9Z8]
tanzu-mission-control_iam_policy.demo_operator[7]: Creation complete after 0s [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-01.tap-16-cls.attached.attached"]: Creation complete after 0s [id=ns:01H9FJHDA811S8W3DVRT6BY7VV]
tanzu-mission-control_iam_policy.demo_operator[2]: Creation complete after 0s [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_editor[1]: Creating...
tanzu-mission-control_iam_policy.demo_editor[7]: Creating...
tanzu-mission-control_iam_policy.demo_editor[0]: Creating...
tanzu-mission-control_iam_policy.demo_editor[6]: Creating...
tanzu-mission-control_iam_policy.demo_editor[5]: Creating...
tanzu-mission-control_iam_policy.demo_editor[2]: Creating...
tanzu-mission-control_iam_policy.demo_editor[3]: Creating...
tanzu-mission-control_iam_policy.demo_editor[4]: Creating...
tanzu-mission-control_iam_policy.demo_editor[5]: Creation complete after 1s [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_editor[2]: Creation complete after 1s [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_editor[4]: Creation complete after 1s [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_editor[0]: Creation complete after 1s [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_editor[1]: Creation complete after 1s [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_editor[7]: Creation complete after 1s [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_iam_policy.demo_editor[3]: Creation complete after 1s [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_iam_policy.demo_editor[6]: Creation complete after 1s [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_viewer[5]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[1]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[6]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[0]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[4]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[3]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[7]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[2]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[1]: Creation complete after 0s [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_viewer[4]: Creation complete after 0s [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_viewer[6]: Creation complete after 0s [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_viewer[3]: Creation complete after 0s [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
╷
│ Error: unable to create Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│   with tanzu-mission-control_iam_policy.demo_viewer[2],
│   on iam-policies.tf line 47, in resource "tanzu-mission-control_iam_policy" "demo_viewer":
│   47: resource "tanzu-mission-control_iam_policy" "demo_viewer" {
│
╵
╷
│ Error: unable to create Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│   with tanzu-mission-control_iam_policy.demo_viewer[7],
│   on iam-policies.tf line 47, in resource "tanzu-mission-control_iam_policy" "demo_viewer":
│   47: resource "tanzu-mission-control_iam_policy" "demo_viewer" {
│
╵
╷
│ Error: unable to create Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│   with tanzu-mission-control_iam_policy.demo_viewer[5],
│   on iam-policies.tf line 47, in resource "tanzu-mission-control_iam_policy" "demo_viewer":
│   47: resource "tanzu-mission-control_iam_policy" "demo_viewer" {
│
╵
╷
│ Error: unable to create Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│   with tanzu-mission-control_iam_policy.demo_viewer[0],
│   on iam-policies.tf line 47, in resource "tanzu-mission-control_iam_policy" "demo_viewer":
│   47: resource "tanzu-mission-control_iam_policy" "demo_viewer" {
vrabbi commented 1 year ago

after 1 minute wait i then can continue by doing an apply again which works with the following output:

Plan: 28 to add, 0 to change, 0 to destroy.

Do you want to perform these actions?
  Terraform will perform the actions described above.
  Only 'yes' will be accepted to approve.

  Enter a value: yes

tanzu-mission-control_iam_policy.demo_viewer[0]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[2]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[5]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[7]: Creating...
tanzu-mission-control_iam_policy.demo_viewer[2]: Creation complete after 0s [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_viewer[0]: Creation complete after 0s [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_viewer[5]: Creation complete after 0s [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_viewer[7]: Creation complete after 0s [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_network_policy.ingress_controller_only[0]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[3]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[4]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[5]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[2]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[7]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[6]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[1]: Creating...
tanzu-mission-control_network_policy.ingress_controller_only[2]: Creation complete after 0s [id=p:01H9FJXP5693A1SNKT2ZR5DMY6]
tanzu-mission-control_network_policy.ingress_controller_only[5]: Creation complete after 0s [id=p:01H9FJXP5JKCR47H4GZB0QK2SQ]
tanzu-mission-control_network_policy.ingress_controller_only[6]: Creation complete after 0s [id=p:01H9FJXP4WB1ZRHN1AHFCBZZY4]
tanzu-mission-control_network_policy.ingress_controller_only[7]: Creation complete after 0s [id=p:01H9FJXP57W7YR7APPZR0AQR94]
tanzu-mission-control_network_policy.ingress_controller_only[3]: Creation complete after 0s [id=p:01H9FJXP4DW84PTV4F5RRT27ZN]
tanzu-mission-control_network_policy.ingress_controller_only[1]: Creation complete after 0s [id=p:01H9FJXP4CF9G3VKS2E6C8J8HC]
tanzu-mission-control_network_policy.ingress_controller_only[0]: Creation complete after 0s [id=p:01H9FJXP64NCHCCZSY4DX972X6]
tanzu-mission-control_network_policy.ingress_controller_only[4]: Creation complete after 0s [id=p:01H9FJXP4EYC0FE89SCSBMG7QA]
tanzu-mission-control_network_policy.coredns_egress[4]: Creating...
tanzu-mission-control_network_policy.coredns_egress[2]: Creating...
tanzu-mission-control_network_policy.coredns_egress[0]: Creating...
tanzu-mission-control_network_policy.coredns_egress[5]: Creating...
tanzu-mission-control_network_policy.coredns_egress[7]: Creating...
tanzu-mission-control_network_policy.coredns_egress[6]: Creating...
tanzu-mission-control_network_policy.coredns_egress[3]: Creating...
tanzu-mission-control_network_policy.coredns_egress[1]: Creating...
tanzu-mission-control_network_policy.coredns_egress[3]: Creation complete after 0s [id=p:01H9FJXPKZ40BVSHN53SHRNGYV]
tanzu-mission-control_network_policy.coredns_egress[2]: Creation complete after 0s [id=p:01H9FJXPKG7A7H493TEWXRXN8J]
tanzu-mission-control_network_policy.coredns_egress[7]: Creation complete after 0s [id=p:01H9FJXPKQPBZR3KPSETBDFD0M]
tanzu-mission-control_network_policy.coredns_egress[0]: Creation complete after 0s [id=p:01H9FJXPKWP7NK2BKPBC3KZM6M]
tanzu-mission-control_network_policy.coredns_egress[6]: Creation complete after 0s [id=p:01H9FJXPKB6MJYKDQH53BG5QZG]
tanzu-mission-control_network_policy.coredns_egress[1]: Creation complete after 0s [id=p:01H9FJXPK2Z1X37AGP5HV8EGEF]
tanzu-mission-control_network_policy.coredns_egress[4]: Creation complete after 0s [id=p:01H9FJXPKD8N0SASTKTXDJK85Z]
tanzu-mission-control_network_policy.coredns_egress[5]: Creation complete after 0s [id=p:01H9FJXPRFBYDSJEWVQH1W3CH1]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[7]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[0]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[2]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[5]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[3]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[4]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[1]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[6]: Creating...
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[0]: Creation complete after 0s [id=p:01H9FJXPZ7CYQRBYW0XYYWEGR4]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[3]: Creation complete after 0s [id=p:01H9FJXQ44ZKZD3SVFG29CHGC4]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[1]: Creation complete after 0s [id=p:01H9FJXQ3Z08K4XF9RA3RCAV08]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[7]: Creation complete after 0s [id=p:01H9FJXQ4190ZC83M6GCYZY12Q]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[2]: Creation complete after 0s [id=p:01H9FJXQ42AYNS25J76JBR5ZG4]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[6]: Creation complete after 0s [id=p:01H9FJXQ44BH2M0S5XH6KS4F4P]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[5]: Creation complete after 0s [id=p:01H9FJXQ3TW7BNJXAPGSE81G37]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[4]: Creation complete after 1s [id=p:01H9FJXQ8KETHQJXHDDEZYPW6B]

Apply complete! Resources: 28 added, 0 changed, 0 destroyed.

then doing a delete fails again with the same rate limit error:

Plan: 0 to add, 0 to change, 64 to destroy.

Do you really want to destroy all resources?
  Terraform will destroy all your managed infrastructure, as shown above.
  There is no undo. Only 'yes' will be accepted to confirm.

  Enter a value: yes

tanzu-mission-control_namespace.demo["app demo 2.sec-ns-02.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHCVNN7X9868KV4ECST3W]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[3]: Destroying... [id=p:01H9FJXQ44ZKZD3SVFG29CHGC4]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[2]: Destroying... [id=p:01H9FJXQ42AYNS25J76JBR5ZG4]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-05.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHD5SYKSNHTC2G32R5XAR]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-04.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHCW6G1TE4DKWDKE8NJ9Y]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-06.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHD943AB7AVF18N8HV9Z8]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-08.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHD89X2P424EGDTYG465C]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-07.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHCVGF4Q9XTFBCRQCNVNV]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[7]: Destroying... [id=p:01H9FJXQ4190ZC83M6GCYZY12Q]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[4]: Destroying... [id=p:01H9FJXQ8KETHQJXHDDEZYPW6B]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[3]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[4]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[7]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[0]: Destroying... [id=p:01H9FJXPZ7CYQRBYW0XYYWEGR4]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-07.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_namespace.demo["first app demo.sec-ns-01.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHDA811S8W3DVRT6BY7VV]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-02.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[6]: Destroying... [id=p:01H9FJXQ44BH2M0S5XH6KS4F4P]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[1]: Destroying... [id=p:01H9FJXQ3Z08K4XF9RA3RCAV08]
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[5]: Destroying... [id=p:01H9FJXQ3TW7BNJXAPGSE81G37]
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-06.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-04.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_namespace.demo["first app demo.sec-ns-03.tap-16-cls.attached.attached"]: Destroying... [id=ns:01H9FJHCWBXPQT7HTXNQTC82R3]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-05.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_namespace.demo["app demo 2.sec-ns-08.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[1]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[6]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[5]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[2]: Destruction complete after 0s
tanzu-mission-control_namespace.demo["first app demo.sec-ns-01.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_image_policy.workspace_scoped_custom_image_policy[0]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[7]: Destroying... [id=p:01H9FJXPKQPBZR3KPSETBDFD0M]
tanzu-mission-control_network_policy.coredns_egress[3]: Destroying... [id=p:01H9FJXPKZ40BVSHN53SHRNGYV]
tanzu-mission-control_network_policy.coredns_egress[6]: Destroying... [id=p:01H9FJXPKB6MJYKDQH53BG5QZG]
tanzu-mission-control_network_policy.coredns_egress[5]: Destroying... [id=p:01H9FJXPRFBYDSJEWVQH1W3CH1]
tanzu-mission-control_network_policy.coredns_egress[4]: Destroying... [id=p:01H9FJXPKD8N0SASTKTXDJK85Z]
tanzu-mission-control_network_policy.coredns_egress[1]: Destroying... [id=p:01H9FJXPK2Z1X37AGP5HV8EGEF]
tanzu-mission-control_network_policy.coredns_egress[0]: Destroying... [id=p:01H9FJXPKWP7NK2BKPBC3KZM6M]
tanzu-mission-control_network_policy.coredns_egress[2]: Destroying... [id=p:01H9FJXPKG7A7H493TEWXRXN8J]
tanzu-mission-control_namespace.demo["first app demo.sec-ns-03.tap-16-cls.attached.attached"]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[7]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[5]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[1]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[0]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[6]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[2]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[3]: Destruction complete after 0s
tanzu-mission-control_network_policy.coredns_egress[4]: Destruction complete after 0s
tanzu-mission-control_network_policy.ingress_controller_only[6]: Destroying... [id=p:01H9FJXP4WB1ZRHN1AHFCBZZY4]
tanzu-mission-control_network_policy.ingress_controller_only[2]: Destroying... [id=p:01H9FJXP5693A1SNKT2ZR5DMY6]
tanzu-mission-control_network_policy.ingress_controller_only[7]: Destroying... [id=p:01H9FJXP57W7YR7APPZR0AQR94]
tanzu-mission-control_network_policy.ingress_controller_only[3]: Destroying... [id=p:01H9FJXP4DW84PTV4F5RRT27ZN]
tanzu-mission-control_network_policy.ingress_controller_only[0]: Destroying... [id=p:01H9FJXP64NCHCCZSY4DX972X6]
tanzu-mission-control_network_policy.ingress_controller_only[5]: Destroying... [id=p:01H9FJXP5JKCR47H4GZB0QK2SQ]
tanzu-mission-control_network_policy.ingress_controller_only[4]: Destroying... [id=p:01H9FJXP4EYC0FE89SCSBMG7QA]
tanzu-mission-control_network_policy.ingress_controller_only[1]: Destroying... [id=p:01H9FJXP4CF9G3VKS2E6C8J8HC]
tanzu-mission-control_network_policy.ingress_controller_only[2]: Destruction complete after 0s
tanzu-mission-control_network_policy.ingress_controller_only[3]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[7]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[4]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[5]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[1]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[0]: Destruction complete after 1s
tanzu-mission-control_network_policy.ingress_controller_only[6]: Destruction complete after 1s
tanzu-mission-control_iam_policy.demo_viewer[3]: Destroying... [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_iam_policy.demo_viewer[7]: Destroying... [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_iam_policy.demo_viewer[1]: Destroying... [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_viewer[6]: Destroying... [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_viewer[5]: Destroying... [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_viewer[4]: Destroying... [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_viewer[2]: Destroying... [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_viewer[0]: Destroying... [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_viewer[5]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[0]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[1]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[4]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[7]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[3]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[6]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_viewer[2]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[5]: Destroying... [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_editor[2]: Destroying... [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_editor[7]: Destroying... [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_iam_policy.demo_editor[1]: Destroying... [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_editor[6]: Destroying... [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_editor[4]: Destroying... [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_editor[3]: Destroying... [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_iam_policy.demo_editor[0]: Destroying... [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_editor[7]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[1]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[6]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[2]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[3]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[4]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[0]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_editor[5]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_operator[0]: Destroying... [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_iam_policy.demo_operator[2]: Destroying... [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_iam_policy.demo_operator[1]: Destroying... [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_operator[4]: Destroying... [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_operator[3]: Destroying... [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_iam_policy.demo_operator[6]: Destroying... [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_operator[5]: Destroying... [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_iam_policy.demo_operator[7]: Destroying... [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_iam_policy.demo_operator[2]: Destruction complete after 1s
tanzu-mission-control_iam_policy.demo_operator[3]: Destruction complete after 1s
tanzu-mission-control_iam_policy.demo_operator[0]: Destruction complete after 1s
tanzu-mission-control_iam_policy.demo_operator[5]: Destruction complete after 1s
╷
│ Error: unable to delete Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│
╵
╷
│ Error: unable to delete Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│
╵
╷
│ Error: unable to delete Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}
│
│
╵
╷
│ Error: unable to delete Role Binding for workspace: PATCH request failed with status : 429 Too Many Requests, response: {"error":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later.","code":8,"message":"/vmware.tanzu.manage.v1alpha1.workspace.WorkspaceIAMPolicy/Patch is rejected due to rate limiting, please retry later."}

1 minute later with another destroy it then finishes:

Plan: 0 to add, 0 to change, 12 to destroy.

Do you really want to destroy all resources?
  Terraform will destroy all your managed infrastructure, as shown above.
  There is no undo. Only 'yes' will be accepted to confirm.

  Enter a value: yes

tanzu-mission-control_iam_policy.demo_operator[1]: Destroying... [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_iam_policy.demo_operator[6]: Destroying... [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_iam_policy.demo_operator[7]: Destroying... [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_iam_policy.demo_operator[4]: Destroying... [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_iam_policy.demo_operator[7]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_operator[1]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_operator[4]: Destruction complete after 0s
tanzu-mission-control_iam_policy.demo_operator[6]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-08"]: Destroying... [id=ws:01H9FJHC7FRC1RQE78WGGZ7R3X]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-03"]: Destroying... [id=ws:01H9FJHC7SRV6VA8NQ51YEJQNT]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-04"]: Destroying... [id=ws:01H9FJHC7KYZSFB71PC9KFDMT2]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-05"]: Destroying... [id=ws:01H9FJHC7HE9V5B9CRW8Q0N66T]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-06"]: Destroying... [id=ws:01H9FJHCC5XM26VC6H0K50N53G]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-07"]: Destroying... [id=ws:01H9FJHCC2C97CY21GSCRY1RQX]
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-02"]: Destroying... [id=ws:01H9FJHC7KBZWX3SJCVVJ4S945]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-01"]: Destroying... [id=ws:01H9FJHC7N2V96R0EB1FT672R4]
tanzu-mission-control_workspace.demo["first app demo.sec-ns-05"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-02"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-06"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["first app demo.sec-ns-03"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["first app demo.sec-ns-07"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["first app demo.sec-ns-01"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-04"]: Destruction complete after 0s
tanzu-mission-control_workspace.demo["app demo 2.sec-ns-08"]: Destruction complete after 1s

Destroy complete! Resources: 12 destroyed.
derekmceachern commented 3 months ago

Having the same issue here. Any update on this issue?

╷
│ Error: Unable to update Tanzu Mission Control cluster entry, name : k8scluster: PUT request failed with status : 429 Too Many Requests, response: {"error":"(target=mc:01J3RA188EY49V9G5YKEKB3JB0, intentId=01J6JK2XJ3ABB3CENF9ZG3ZA5H): error receiving watch event from DB","code":8,"message":"(target=mc:01J3RA188EY49V9G5YKEKB3JB0, intentId=01J6JK2XJ3ABB3CENF9ZG3ZA5H): error receiving watch event from DB"}
│
│   with module.clusters.tanzu-mission-control_cluster.create_tkgs_workload_k8scluster,
│   on clusters/k8scluster.tf line 52, in resource "tanzu-mission-control_cluster" "create_tkgs_workload_k8scluster":
│   52: resource "tanzu-mission-control_cluster" "create_tkgs_workload_k8scluster" {
│
github-actions[bot] commented 1 week ago

'Marking this issue as stale due to inactivity. This helps us focus on the active issues. If this issue receives no comments in the next 30 days it will automatically be closed.

If this issue was automatically closed and you feel this issue should be reopened, we encourage creating a new issue linking back to this one for added context. Thank you!'