ArchiFleKs
commented
2 years ago Same issue here
Description
I'm trying to migrate from v17 to v18. Here is my v17 config:
node_groups_defaults = {
disk_size = 10
create_launch_template = true
ami_type = "AL2_x86_64"
min_capacity = 0
max_capacity = 10
desired_capacity = 0
capacity_type = "ON_DEMAND"
bootstrap_env = {
USE_MAX_PODS = false
CONTAINER_RUNTIME = "containerd"
}
taints = [
{
key = "dedicated"
value = "true"
effect = "NO_SCHEDULE"
}
]
}
node_groups = {
"default-a-" = {
desired_capacity = 1
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[0]]
kubelet_extra_args = "--max-pods=${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.9.1 --cni-prefix-delegation-enabled")}"
taints = []
k8s_labels = {
size = "medium"
network = "private"
arch = "arm64"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}a"
}
}
}Here is my new v18 config:
eks_managed_node_group_defaults = {
disk_size = 10
min_capacity = 0
max_capacity = 10
desired_capacity = 0
capacity_type = "ON_DEMAND"
enable_bootstrap_user_data = true
}
eks_managed_node_groups = {
"default-a" = {
desired_capacity = 1
ami_type = "AL2_x86_64"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[0]]
enable_bootstrap_user_data = false
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}a"
}
}
"default-b" = {
desired_capacity = 1
ami_type = "AL2_x86_64"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[1]]
enable_bootstrap_user_data = false
bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.10.1 --cni-prefix-delegation-enabled")}'"
pre_bootstrap_user_data = <<-EOT
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
EOT
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}b"
}
}
"default-c" = {
desired_capacity = 1
ami_type = "AL2_x86_64"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[2]]
enable_bootstrap_user_data = true
bootstrap_extra_args = "--container-runtime containerd --kubelet-extra-args '--max-pods=${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.10.1 --cni-prefix-delegation-enabled")}'"
pre_bootstrap_user_data = <<-EOT
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
EOT
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}b"
}
}
"bottlerocket-a" = {
desired_capacity = 1
ami_type = "BOTTLEROCKET_x86_64"
platform = "bottlerocket"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[0]]
bootstrap_extra_args = <<-EOT
"max-pods" = ${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.10.1 --cni-prefix-delegation-enabled")}
EOT
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}a"
}
}
"bottlerocket-b" = {
desired_capacity = 1
ami_type = "BOTTLEROCKET_x86_64"
platform = "bottlerocket"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[1]]
bootstrap_extra_args = <<-EOT
"max-pods" = ${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.10.1 --cni-prefix-delegation-enabled")}
EOT
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}b"
}
}
"bottlerocket-c" = {
desired_capacity = 1
ami_type = "BOTTLEROCKET_x86_64"
platform = "bottlerocket"
instance_types = ["t3a.large"]
subnets = [dependency.vpc.outputs.private_subnets[2]]
bootstrap_extra_args = <<-EOT
"max-pods" = ${run_cmd("/bin/sh", "-c", "../../../../../../../tools/max-pods-calculator.sh --instance-type t3a.large --cni-version 1.10.1 --cni-prefix-delegation-enabled")}
EOT
k8s_labels = {
network = "private"
"topology.ebs.csi.aws.com/zone" = "${include.root.locals.merged.aws_region}c"
}
}
}The bottlerocket pools are working fine, I have issue with the Amazon Linux Pool, where I cannot get Containerd and bootstrap to work as per https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/examples/eks_managed_node_group/main.tf#L166
Let's go pool by pool
default-a
pre-merged:
No CUSTOM USER DATApost-merged:
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="//"
--//
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
set -ex
B64_CLUSTER_CA=REDACTED
API_SERVER_URL=https://732CC77068B4A2F46DC40497FD380C96.sk1.eu-west-1.eks.amazonaws.com
K8S_CLUSTER_DNS_IP=172.20.0.10
/etc/eks/bootstrap.sh pio-teks-demo-demo --kubelet-extra-args '--node-labels=eks.amazonaws.com/sourceLaunchTemplateVersion=5,eks.amazonaws.com/nodegroup-image=ami-06efe6b9d402e8674,eks.amazonaws.com/capacityType=ON_DEMAND,eks.amazonaws.com/nodegroup=default-a-20220112161956265600000004,eks.amazonaws.com/sourceLaunchTemplateId=lt-08c0df7227547bf5b --max-pods=110' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL --dns-cluster-ip $K8S_CLUSTER_DNS_IP --use-max-pods false
--//--This seems to be the standard user data, although I'm not sure how use-max-pods false and max-pods ended up here by default but it might be relicas of something I tried before (or is EKS able to calculate this on it's own now ? )
Node in the end is running Docker and not containerd.
default-b
pre-merged:
#!/bin/bash
set -e
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
B64_CLUSTER_CA=REDACTED
API_SERVER_URL=https://732CC77068B4A2F46DC40497FD380C96.sk1.eu-west-1.eks.amazonaws.com
/etc/eks/bootstrap.sh pio-teks-demo-demo --container-runtime containerd --kubelet-extra-args '--max-pods=110' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URLpost-merged:
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="//"
--//
Content-Transfer-Encoding: 7bit
Content-Type: text/x-shellscript
Mime-Version: 1.0
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
--//
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
set -ex
B64_CLUSTER_CA=REDACTED
API_SERVER_URL=https://732CC77068B4A2F46DC40497FD380C96.sk1.eu-west-1.eks.amazonaws.com
K8S_CLUSTER_DNS_IP=172.20.0.10
/etc/eks/bootstrap.sh pio-teks-demo-demo --kubelet-extra-args '--node-labels=eks.amazonaws.com/sourceLaunchTemplateVersion=3,eks.amazonaws.com/nodegroup-image=ami-06efe6b9d402e8674,eks.amazonaws.com/capacityType=ON_DEMAND,eks.amazonaws.com/nodegroup=default-b-20220112162947978700000003,eks.amazonaws.com/sourceLaunchTemplateId=lt-04b8a55955eb3eaf6 --max-pods=110' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL --dns-cluster-ip $K8S_CLUSTER_DNS_IP --use-max-pods false
--//--Here my previous custom args are missing form the final LT config, probably because of the missing. Node in the end is running Docker and not containerd.
default-c
Just fail to create with:
╷
│ Error: error waiting for EKS Node Group (pio-teks-demo-demo:default-c-20220112162947978700000003) version update (fcf678f4-bfbb-3f56-bbf5-f20dce780e0b): unexpected state 'Failed', wanted target 'Successful'. last error: 1 error occurred:
│ * : Unknown: User data was not in the MIME multipart format.
│
│
│
│ with module.eks_managed_node_group["default-c"].aws_eks_node_group.this[0],
│ on modules/eks-managed-node-group/main.tf line 260, in resource "aws_eks_node_group" "this":
│ 260: resource "aws_eks_node_group" "this" {
│
╵ Here is the pre merged userdata:
#!/bin/bash
set -e
export CONTAINER_RUNTIME="containerd"
export USE_MAX_PODS=false
B64_CLUSTER_CA=REDACTED
API_SERVER_URL=https://732CC77068B4A2F46DC40497FD380C96.sk1.eu-west-1.eks.amazonaws.com
/etc/eks/bootstrap.sh pio-teks-demo-demo --container-runtime containerd --kubelet-extra-args '--max-pods=110' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URLNot sure what to make of it as it is the same as default-b but without the enable_bootstrap_user_data which seems to break it.
Versions
- Terraform: 1.1.3
- Provider(s):
Terraform v1.1.3 on linux_amd64 + provider registry.terraform.io/hashicorp/aws v3.71.0 + provider registry.terraform.io/hashicorp/cloudinit v2.2.0 + provider registry.terraform.io/hashicorp/github v4.19.1 + provider registry.terraform.io/hashicorp/kubernetes v2.7.1 + provider registry.terraform.io/hashicorp/tls v3.1.0 - Module: v18.0.6
Expected behavior
Node to be launched with containerd as runtime and the max-prod specify by ENI max pod calculator
Actual behavior
Launching nodes fails.
Terminal Output Screenshot(s)
╷
│ Error: error waiting for EKS Node Group (pio-teks-demo-demo:default-c-20220112162947978700000003) version update (fcf678f4-bfbb-3f56-bbf5-f20dce780e0b): unexpected state 'Failed', wanted target 'Successful'. last error: 1 error occurred:
│ * : Unknown: User data was not in the MIME multipart format.
│
│
│
│ with module.eks_managed_node_group["default-c"].aws_eks_node_group.this[0],
│ on modules/eks-managed-node-group/main.tf line 260, in resource "aws_eks_node_group" "this":
│ 260: resource "aws_eks_node_group" "this" {
│
╵
ERRO[0285] 1 error occurred:
* exit status 1
bryantbiggs
antonbabenko
andronux
stevehipwell
youwalther65
mj3c
Description
I can't create eks_managed_node_groups as described in README.md Combination of this parameters
doesn't work with or without below parameters
if I use enable_bootstrap_user_data = true I get error
If enable_bootstrap_user_data = false I have such user data in launch template
Versions
Reproduction
Steps to reproduce the behavior: create such group using terraform-aws-eks v18.0.6
Expected behavior
I need the kubernetes node with CONTAINER-RUNTIME containerd and --kubelet-extra-args '--max-pods=20'