feat: Improve managed node group bootstrap revisited (#1577)

Author: stevehipwell

modules/node_groups/README.md

@@ -37,7 +37,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In
 | instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` |
 | k8s\_labels | Kubernetes labels | map(string) | No labels applied |
 | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` |
-| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" |
+| bootstrap_env | Provide environment variables to customise [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Require `create_launch_template` to be `true` | map(string) | `{}` |
+| kubelet_extra_args | Extra arguments for kubelet, this is automatically merged with `labels`. Require `create_launch_template` to be `true` | string | "" |
 | launch_template_id | The id of a aws_launch_template to use | string | No LT used |
 | launch\_template_version | The version of the LT to use | string | none |
 | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` |

modules/node_groups/launch_template.tf

@@ -9,13 +9,14 @@ data "cloudinit_config" "workers_userdata" {
     content_type = "text/x-shellscript"
     content = templatefile("${path.module}/templates/userdata.sh.tpl",
       {
-        kubelet_extra_args   = each.value["kubelet_extra_args"]
-        pre_userdata         = each.value["pre_userdata"]
-        ami_id               = lookup(each.value, "ami_id", "")
-        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
         cluster_name         = var.cluster_name
         cluster_endpoint     = var.cluster_endpoint
         cluster_auth_base64  = var.cluster_auth_base64
+        ami_id               = lookup(each.value, "ami_id", "")
+        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
+        bootstrap_env        = each.value["bootstrap_env"]
+        kubelet_extra_args   = each.value["kubelet_extra_args"]
+        pre_userdata         = each.value["pre_userdata"]
         capacity_type        = lookup(each.value, "capacity_type", "ON_DEMAND")
         append_labels        = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
       }

modules/node_groups/locals.tf

@@ -13,6 +13,7 @@ locals {
       min_capacity                         = var.workers_group_defaults["asg_min_size"]
       subnets                              = var.workers_group_defaults["subnets"]
       create_launch_template               = false
+      bootstrap_env                        = {}
       kubelet_extra_args                   = var.workers_group_defaults["kubelet_extra_args"]
       disk_size                            = var.workers_group_defaults["root_volume_size"]
       disk_type                            = var.workers_group_defaults["root_volume_type"]

modules/node_groups/templates/userdata.sh.tpl

@@ -1,13 +1,27 @@
 #!/bin/bash -e
 %{ if length(ami_id) == 0 ~}
 
-# Set variables directly into bootstrap.sh for default AMI
-sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh
+# Set bootstrap env
+printf '#!/bin/bash
+%{ for k, v in bootstrap_env ~}
+export ${k}="${v}"
+%{ endfor ~}
+export ADDITIONAL_KUBELET_EXTRA_ARGS="${kubelet_extra_args}"
+' > /etc/profile.d/eks-bootstrap-env.sh
+
+# Source extra environment variables in bootstrap script
+sed -i '/^set -o errexit/a\\nsource /etc/profile.d/eks-bootstrap-env.sh' /etc/eks/bootstrap.sh
+
+# Merge ADDITIONAL_KUBELET_EXTRA_ARGS into KUBELET_EXTRA_ARGS
+sed -i 's/^KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-}/KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-} $${ADDITIONAL_KUBELET_EXTRA_ARGS}/' /etc/eks/bootstrap.sh
 %{else ~}
 
 # Set variables for custom AMI
 API_SERVER_URL=${cluster_endpoint}
 B64_CLUSTER_CA=${cluster_auth_base64}
+%{ for k, v in bootstrap_env ~}
+${k}="${v}"
+%{ endfor ~}
 KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}'
 %{endif ~}