ManApart
commented
5 years ago Basic summary of our changes against the last time we merged upstream into our fork:
- Added variables for rolling update counts - not applicable for non cloudformation based ASG
- Changed out the terraform autoscaling group resource for a CloudFormation based one, which does rolling updates (the tf one just didn't do them) - we'll test if this actually works with the upstream's addition of the
asg_recreate_on_changeandasg_initial_lifecycle_hooks - Added the ability to lock the AMI version so we weren't constantly taking updates with us - addressed in upstream with
worker_ami_name_filtervariable
diff --git a/data.tf b/data.tf
index d4b5106..8a6d8c7 100644
--- a/data.tf
+++ b/data.tf
@@ -18,7 +18,7 @@ data "aws_iam_policy_document" "workers_assume_role_policy" {
data "aws_ami" "eks_worker" {
filter {
name = "name"
- values = ["amazon-eks-node-${var.cluster_version}-v*"]
+ values = ["amazon-eks-node-${var.cluster_version}-v${var.ami_version}"]
}
most_recent = true
diff --git a/local.tf b/local.tf
index 5597cdd..4c7e062 100644
--- a/local.tf
+++ b/local.tf
@@ -15,6 +15,8 @@ locals {
asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group.
asg_max_size = "3" # Maximum worker capacity in the autoscaling group.
asg_min_size = "1" # Minimum worker capacity in the autoscaling group.
+ asg_rolling_update_max_batch_size = "1" # Specifies the maximum number of instances that will be updated at once during an autoscaling group rolling update.
+ asg_rolling_update_min_instances_in_service = "0" # Specifies the minimum number of instances that must be in service within the autoscaling group during a rolling update.
instance_type = "m4.large" # Size of the workers instances.
spot_price = "" # Cost of spot instance.
placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated".
diff --git a/outputs.tf b/outputs.tf
index 12041d9..3cb2def 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -39,14 +39,11 @@ output "kubeconfig" {
value = "${data.template_file.kubeconfig.rendered}"
}
-output "workers_asg_arns" {
- description = "IDs of the autoscaling groups containing workers."
- value = "${concat(aws_autoscaling_group.workers.*.arn, aws_autoscaling_group.workers_launch_template.*.arn)}"
-}
-
output "workers_asg_names" {
description = "Names of the autoscaling groups containing workers."
- value = "${concat(aws_autoscaling_group.workers.*.id, aws_autoscaling_group.workers_launch_template.*.id)}"
+ value = [
+ "${data.template_file.workers_names.*.rendered}"
+ ]
}
output "worker_security_group_id" {
diff --git a/variables.tf b/variables.tf
index 1937585..6712004 100644
--- a/variables.tf
+++ b/variables.tf
@@ -12,6 +12,11 @@ variable "cluster_version" {
default = "1.11"
}
+variable "ami_version" {
+ description = "Version for the EKS-optimized worker AMI."
+ default = "*"
+}
+
variable "config_output_path" {
description = "Where to save the Kubectl config file (if `write_kubeconfig = true`). Should end in a forward slash `/` ."
default = "./"
diff --git a/workers.tf b/workers.tf
index 0c7acd4..1e3cb5b 100644
--- a/workers.tf
+++ b/workers.tf
@@ -1,29 +1,45 @@
-# Worker Groups using Launch Configurations
-
-resource "aws_autoscaling_group" "workers" {
- name_prefix = "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}"
- desired_capacity = "${lookup(var.worker_groups[count.index], "asg_desired_capacity", local.workers_group_defaults["asg_desired_capacity"])}"
- max_size = "${lookup(var.worker_groups[count.index], "asg_max_size", local.workers_group_defaults["asg_max_size"])}"
- min_size = "${lookup(var.worker_groups[count.index], "asg_min_size", local.workers_group_defaults["asg_min_size"])}"
- target_group_arns = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "target_group_arns", ""), local.workers_group_defaults["target_group_arns"])))}"]
- launch_configuration = "${element(aws_launch_configuration.workers.*.id, count.index)}"
- vpc_zone_identifier = ["${split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), local.workers_group_defaults["subnets"]))}"]
- protect_from_scale_in = "${lookup(var.worker_groups[count.index], "protect_from_scale_in", local.workers_group_defaults["protect_from_scale_in"])}"
- suspended_processes = ["${compact(split(",", coalesce(lookup(var.worker_groups[count.index], "suspended_processes", ""), local.workers_group_defaults["suspended_processes"])))}"]
- count = "${var.worker_group_count}"
-
- tags = ["${concat(
- list(
- map("key", "Name", "value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg", "propagate_at_launch", true),
- map("key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "value", "owned", "propagate_at_launch", true),
- map("key", "k8s.io/cluster-autoscaler/${lookup(var.worker_groups[count.index], "autoscaling_enabled", local.workers_group_defaults["autoscaling_enabled"]) == 1 ? "enabled" : "disabled" }", "value", "true", "propagate_at_launch", false)
- ),
- local.asg_tags)
- }"]
-
- lifecycle {
- ignore_changes = ["desired_capacity"]
- }
+resource "aws_cloudformation_stack" "workers" {
+ count = "${var.worker_group_count}"
+ name = "${replace("${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}", "/[^-a-zA-Z0-9]/", "-")}"
+ template_body = <<EOF
+---
+AWSTemplateFormatVersion: "2010-09-09"
+Description: Terraform-managed CF Stack for Auto-Scaling Group
+Resources:
+ AutoScalingGroup:
+ Type: AWS::AutoScaling::AutoScalingGroup
+ Properties:
+ AutoScalingGroupName: ${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}
+ VPCZoneIdentifier: ${jsonencode(split(",", coalesce(lookup(var.worker_groups[count.index], "subnets", ""), join(",", var.subnets))))}
+ LaunchConfigurationName: ${element(aws_launch_configuration.workers.*.id, count.index)}
+ MinSize: ${lookup(var.worker_groups[count.index], "asg_min_size",lookup(local.workers_group_defaults, "asg_min_size"))}
+ MaxSize: ${lookup(var.worker_groups[count.index], "asg_max_size",lookup(local.workers_group_defaults, "asg_max_size"))}
+ Tags: ${jsonencode(concat(
+ list(
+ map("Key", "Name", "Value", "${aws_eks_cluster.this.name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg", "PropagateAtLaunch", "True"),
+ map("Key", "kubernetes.io/cluster/${aws_eks_cluster.this.name}", "Value", "owned", "PropagateAtLaunch", "True"),
+ ),
+ local.asg_tags))}
+ UpdatePolicy:
+ AutoScalingRollingUpdate:
+ MaxBatchSize: ${lookup(var.worker_groups[count.index], "asg_rolling_update_max_batch_size",lookup(local.workers_group_defaults, "asg_rolling_update_max_batch_size"))}
+ MinInstancesInService: ${lookup(var.worker_groups[count.index], "asg_rolling_update_min_instances_in_service",lookup(local.workers_group_defaults, "asg_rolling_update_min_instances_in_service"))}
+ SuspendProcesses:
+ - HealthCheck
+ - ReplaceUnhealthy
+ - AZRebalance
+ - AlarmNotification
+ - ScheduledActions
+Outputs:
+ AutoScalingGroupName:
+ Description: The name of the Auto-Scaling Group
+ Value: !Ref AutoScalingGroup
+EOF
+}
+
+data "template_file" "workers_names" {
+ count = "${var.worker_group_count}"
+ template = "${lookup(aws_cloudformation_stack.workers.*.outputs[count.index], "AutoScalingGroupName")}"
}
resource "aws_launch_configuration" "workers" {
@@ -136,11 +152,11 @@ resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryRea
resource "null_resource" "tags_as_list_of_maps" {
count = "${length(keys(var.tags))}"
- triggers = {
- key = "${element(keys(var.tags), count.index)}"
- value = "${element(values(var.tags), count.index)}"
- propagate_at_launch = "true"
- }
+ triggers = "${map(
+ "Key", "${element(keys(var.tags), count.index)}",
+ "Value", "${element(values(var.tags), count.index)}",
+ "PropagateAtLaunch", "True"
+ )}"
}
resource "aws_iam_role_policy_attachment" "workers_autoscaling" {
We forked terraform-aws-eks a long time ago. Determine what changes were made in the fork and if they are still needed it we update to the latest version. Either merge in the forked repo changes or change common to use the public repo.
Acceptance criteria of spike:
Spike is to help prep for these ultimate goals:
Tech note: