feat: Add support for AL2023 nodeadm user data

docs/UPGRADE-20.0.md

@@ -20,6 +20,7 @@ To give users advanced notice and provide some future direction for this module,
 1. The `aws-auth` sub-module will be removed entirely from the project. Since this sub-module is captured in the v20.x releases, users can continue using it even after the module moves forward with the next major version. The long term strategy and direction is cluster access entry and to rely only on the AWS Terraform provider.
 2. The default value for `authentication_mode` will change to `API`. Aligning with point 1 above, this is a one way change, but users are free to specify the value of their choosing in place of this default (when the change is made). This module will proceed with an EKS API first strategy.
 3. The launch template and autoscaling group usage contained within the EKS managed nodegroup and self-managed nodegroup sub-modules *might be replaced with the [`terraform-aws-autoscaling`](https://github.com/terraform-aws-modules/terraform-aws-autoscaling) module. At minimum, it makes sense to replace most of functionality in the self-managed nodegroup module with this external module, but its not yet clear if there is any benefit of using it in the EKS managed nodegroup sub-module. The interface that users interact with will stay the same, the changes will be internal to the implementation and we will do everything we can to keep the disruption to a minimum.
+4. The `platform` variable will be replaced and instead `ami_type` will become the standard across both self-managed nodegroup(s) and EKS managed nodegroup(s). As EKS expands its portfolio of supported operating systems, the `ami_type` is better suited to associate the correct user data format to the respective OS. The `platform` variable is a legacy artifact of self-managed nodegroups but not as descriptive as the `ami_type`, and therefore it will be removed in favor of `ami_type`.
 
 ## Additional changes
 

examples/eks_managed_node_group/main.tf

@@ -86,6 +86,29 @@ module "eks" {
       }
     }
 
+    # AL2023 node group utilizing new user data format which utilizes nodeadm
+    # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh)
+    al2023_nodeadm = {
+      platform = "al2023"
+
+      cloudinit_pre_nodeadm = [
+        {
+          content_type = "application/node.eks.aws"
+          content      = <<-EOT
+            ---
+            apiVersion: node.eks.aws/v1alpha
+            kind: NodeConfig
+            spec:
+              kubelet:
+                config:
+                  shutdownGracePeriod: 30s
+                  featureGates:
+                    DisableKubeletCloudCredentialProviders: true
+          EOT
+        }
+      ]
+    }
+
     # Default node group - as provided by AWS EKS using Bottlerocket
     bottlerocket_default = {
       # By default, the module creates a launch template to ensure tags are propagated to instances, etc.,

examples/self_managed_node_group/main.tf

@@ -72,6 +72,29 @@ module "eks" {
     # Default node group - as provisioned by the module defaults
     default_node_group = {}
 
+    # AL2023 node group utilizing new user data format which utilizes nodeadm
+    # to join nodes to the cluster (instead of /etc/eks/bootstrap.sh)
+    al2023_nodeadm = {
+      platform = "al2023"
+
+      cloudinit_pre_nodeadm = [
+        {
+          content_type = "application/node.eks.aws"
+          content      = <<-EOT
+            ---
+            apiVersion: node.eks.aws/v1alpha
+            kind: NodeConfig
+            spec:
+              kubelet:
+                config:
+                  shutdownGracePeriod: 30s
+                  featureGates:
+                    DisableKubeletCloudCredentialProviders: true
+          EOT
+        }
+      ]
+    }
+
     # Bottlerocket node group
     bottlerocket = {
       name = "bottlerocket-self-mng"

examples/user_data/README.md

@@ -30,6 +30,10 @@ $ terraform apply
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_eks_mng_al2023_additional"></a> [eks\_mng\_al2023\_additional](#module\_eks\_mng\_al2023\_additional) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_custom_ami"></a> [eks\_mng\_al2023\_custom\_ami](#module\_eks\_mng\_al2023\_custom\_ami) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_custom_template"></a> [eks\_mng\_al2023\_custom\_template](#module\_eks\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a |
+| <a name="module_eks_mng_al2023_no_op"></a> [eks\_mng\_al2023\_no\_op](#module\_eks\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_additional"></a> [eks\_mng\_al2\_additional](#module\_eks\_mng\_al2\_additional) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_custom_ami"></a> [eks\_mng\_al2\_custom\_ami](#module\_eks\_mng\_al2\_custom\_ami) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_al2_custom_template"></a> [eks\_mng\_al2\_custom\_template](#module\_eks\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a |
@@ -42,6 +46,9 @@ $ terraform apply
 | <a name="module_eks_mng_windows_custom_ami"></a> [eks\_mng\_windows\_custom\_ami](#module\_eks\_mng\_windows\_custom\_ami) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_windows_custom_template"></a> [eks\_mng\_windows\_custom\_template](#module\_eks\_mng\_windows\_custom\_template) | ../../modules/_user_data | n/a |
 | <a name="module_eks_mng_windows_no_op"></a> [eks\_mng\_windows\_no\_op](#module\_eks\_mng\_windows\_no\_op) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_bootstrap"></a> [self\_mng\_al2023\_bootstrap](#module\_self\_mng\_al2023\_bootstrap) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_custom_template"></a> [self\_mng\_al2023\_custom\_template](#module\_self\_mng\_al2023\_custom\_template) | ../../modules/_user_data | n/a |
+| <a name="module_self_mng_al2023_no_op"></a> [self\_mng\_al2023\_no\_op](#module\_self\_mng\_al2023\_no\_op) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_bootstrap"></a> [self\_mng\_al2\_bootstrap](#module\_self\_mng\_al2\_bootstrap) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_custom_template"></a> [self\_mng\_al2\_custom\_template](#module\_self\_mng\_al2\_custom\_template) | ../../modules/_user_data | n/a |
 | <a name="module_self_mng_al2_no_op"></a> [self\_mng\_al2\_no\_op](#module\_self\_mng\_al2\_no\_op) | ../../modules/_user_data | n/a |
@@ -56,6 +63,10 @@ $ terraform apply
 
 | Name | Type |
 |------|------|
+| [local_file.eks_mng_al2023_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.eks_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_additional](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
@@ -68,6 +79,9 @@ $ terraform apply
 | [local_file.eks_mng_windows_custom_ami](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_windows_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.eks_mng_windows_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+| [local_file.self_mng_al2023_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_bootstrap](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_custom_template](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
 | [local_file.self_mng_al2_no_op](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |

examples/user_data/main.tf

@@ -43,7 +43,6 @@ module "eks_mng_al2_custom_ami" {
   EOT
 }
 
-
 module "eks_mng_al2_custom_template" {
   source = "../../modules/_user_data"
 
@@ -65,6 +64,107 @@ module "eks_mng_al2_custom_template" {
   EOT
 }
 
+################################################################################
+# EKS managed node group - AL2023
+################################################################################
+
+module "eks_mng_al2023_no_op" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+}
+
+module "eks_mng_al2023_additional" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+}
+
+module "eks_mng_al2023_custom_ami" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cluster_name              = local.name
+  cluster_endpoint          = local.cluster_endpoint
+  cluster_auth_base64       = local.cluster_auth_base64
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
+  enable_bootstrap_user_data = true
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
+module "eks_mng_al2023_custom_template" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  enable_bootstrap_user_data = true
+  user_data_template_path    = "${path.module}/templates/al2023_custom.tpl"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
 ################################################################################
 # EKS managed node group - Bottlerocket
 ################################################################################
@@ -80,6 +180,9 @@ module "eks_mng_bottlerocket_additional" {
 
   platform = "bottlerocket"
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   bootstrap_extra_args = <<-EOT
     # extra args added
     [settings.kernel]
@@ -138,6 +241,9 @@ module "eks_mng_windows_additional" {
 
   platform = "windows"
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   pre_bootstrap_user_data = <<-EOT
     [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯'
   EOT
@@ -245,6 +351,90 @@ module "self_mng_al2_custom_template" {
   EOT
 }
 
+################################################################################
+# Self-managed node group - AL2023
+################################################################################
+
+module "self_mng_al2023_no_op" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  is_eks_managed_node_group = false
+}
+
+module "self_mng_al2023_bootstrap" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  enable_bootstrap_user_data = true
+  is_eks_managed_node_group  = false
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
+module "self_mng_al2023_custom_template" {
+  source = "../../modules/_user_data"
+
+  platform = "al2023"
+
+  enable_bootstrap_user_data = true
+  is_eks_managed_node_group  = false
+
+  cluster_name        = local.name
+  cluster_endpoint    = local.cluster_endpoint
+  cluster_auth_base64 = local.cluster_auth_base64
+
+  user_data_template_path = "${path.module}/templates/al2023_custom.tpl"
+
+  cloudinit_pre_nodeadm = [{
+    content      = <<-EOT
+      ---
+      apiVersion: node.eks.aws/v1alpha
+      kind: NodeConfig
+      spec:
+        kubelet:
+          config:
+            shutdownGracePeriod: 30s
+            featureGates:
+              DisableKubeletCloudCredentialProviders: true
+    EOT
+    content_type = "application/node.eks.aws"
+  }]
+
+  cloudinit_post_nodeadm = [{
+    content      = <<-EOT
+      echo "All done"
+    EOT
+    content_type = "text/x-shellscript; charset=\"us-ascii\""
+  }]
+}
+
 ################################################################################
 # Self-managed node group - Bottlerocket
 ################################################################################
@@ -269,6 +459,9 @@ module "self_mng_bottlerocket_bootstrap" {
   cluster_endpoint    = local.cluster_endpoint
   cluster_auth_base64 = local.cluster_auth_base64
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   bootstrap_extra_args = <<-EOT
     # extra args added
     [settings.kernel]
@@ -321,6 +514,9 @@ module "self_mng_windows_bootstrap" {
   cluster_endpoint    = local.cluster_endpoint
   cluster_auth_base64 = local.cluster_auth_base64
 
+  # Should do nothing
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
+
   pre_bootstrap_user_data = <<-EOT
     [string]$Something = 'IDoNotKnowAnyPowerShell ¯\_(ツ)_/¯'
   EOT