Added support for IPv6 and add AWS load balancer controller to support externally created load balancer in EKS

EKS-Blueprint/modules/kubernetes-addons/README.md

@@ -214,7 +214,6 @@
 | <a name="input_enable_karpenter"></a> [enable\_karpenter](#input\_enable\_karpenter) | Enable Karpenter autoscaler add-on | `bool` | `false` | no |
 | <a name="input_enable_keda"></a> [enable\_keda](#input\_enable\_keda) | Enable KEDA Event-based autoscaler add-on | `bool` | `false` | no |
 | <a name="input_enable_kube_prometheus_stack"></a> [enable\_kube\_prometheus\_stack](#input\_enable\_kube\_prometheus\_stack) | Enable Community kube-prometheus-stack add-on | `bool` | `false` | no |
-| <a name="input_enable_kubecost"></a> [enable\_kubecost](#input\_enable\_kubecost) | Enable Kubecost add-on | `bool` | `false` | no |
 | <a name="input_enable_kuberay_operator"></a> [enable\_kuberay\_operator](#input\_enable\_kuberay\_operator) | Enable KubeRay Operator add-on | `bool` | `false` | no |
 | <a name="input_enable_kubernetes_dashboard"></a> [enable\_kubernetes\_dashboard](#input\_enable\_kubernetes\_dashboard) | Enable Kubernetes Dashboard add-on | `bool` | `false` | no |
 | <a name="input_enable_kyverno"></a> [enable\_kyverno](#input\_enable\_kyverno) | Enable Kyverno add-on | `bool` | `false` | no |
@@ -265,6 +264,7 @@
 | <a name="input_keda_helm_config"></a> [keda\_helm\_config](#input\_keda\_helm\_config) | KEDA Event-based autoscaler add-on config | `any` | `{}` | no |
 | <a name="input_keda_irsa_policies"></a> [keda\_irsa\_policies](#input\_keda\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_kube_prometheus_stack_helm_config"></a> [kube\_prometheus\_stack\_helm\_config](#input\_kube\_prometheus\_stack\_helm\_config) | Community kube-prometheus-stack Helm Chart config | `any` | `{}` | no |
+| <a name="input_kubecost_enabled"></a> [kubecost\_enabled](#input\_kubecost\_enabled) | Enable Kubecost add-on | `bool` | `false` | no |
 | <a name="input_kubecost_helm_config"></a> [kubecost\_helm\_config](#input\_kubecost\_helm\_config) | Kubecost Helm Chart config | `any` | `{}` | no |
 | <a name="input_kuberay_operator_helm_config"></a> [kuberay\_operator\_helm\_config](#input\_kuberay\_operator\_helm\_config) | KubeRay Operator Helm Chart config | `any` | `{}` | no |
 | <a name="input_kubernetes_dashboard_helm_config"></a> [kubernetes\_dashboard\_helm\_config](#input\_kubernetes\_dashboard\_helm\_config) | Kubernetes Dashboard Helm Chart config | `any` | `null` | no |

EKS-Blueprint/modules/kubernetes-addons/aws-load-balancer-controller/data.tf

@@ -142,9 +142,14 @@ data "aws_iam_policy_document" "aws_lb" {
 
     condition {
       test     = "Null"
-      variable = "aws:ResourceTag/ingress.k8s.aws/cluster"
+      variable = "aws:ResourceTag/elbv2.k8s.aws/cluster"
       values   = ["false"]
     }
+    condition {
+      test     = "StringEquals"
+      variable = "elasticloadbalancing:CreateAction"
+      values   = ["CreateTargetGroup", "CreateLoadBalancer"]
+    }
   }
 
   statement {

EKS-Blueprint/modules/kubernetes-addons/aws-load-balancer-controller/locals.tf

@@ -7,7 +7,7 @@ locals {
     name        = local.name
     chart       = local.name
     repository  = "https://aws.github.io/eks-charts"
-    version     = "1.4.5"
+    version     = "1.5.4"
     namespace   = "kube-system"
     values      = local.default_helm_values
     description = "aws-load-balancer-controller Helm Chart for ingress resources"
@@ -33,6 +33,10 @@ locals {
       {
         name  = "serviceAccount.create"
         value = false
+      },
+      {
+        name  = "clusterName"
+        value = var.addon_context.eks_cluster_id
       }
     ],
     try(var.helm_config.set_values, [])

README.md

@@ -14,6 +14,7 @@ module "eks_bootstrap" {
   name                          = "skaf"
   vpc_id                        = "vpc-06e37f0786b7eskaf"
   environment                   = "production"
+  ipv6_enabled                  = true
   kms_key_arn                   = "arn:aws:kms:region:222222222222:key/kms_key_arn"
   keda_enabled                  = true
   istio_enabled                 = false
@@ -40,6 +41,7 @@ module "eks_bootstrap" {
     private_subnet_name    = "private_subnet_name"
     instance_capacity_type = ["spot"]
     excluded_instance_type = ["nano", "micro", "small"]
+    instance_hypervisor    = ["nitro"]   ## Instance hypervisor is picked up only if IPv6 enable is chosen
   }
   cert_manager_letsencrypt_email                = "email@example.com"
   internal_ingress_nginx_enabled                = true
@@ -74,6 +76,7 @@ module "eks_bootstrap" {
 | Release 2.0.0  | ✔  | ✔  | ✔ | ✗ |
 | Release 2.1.0  | ✔  | ✔  | ✔ | ✗  |
 | Release 3.0.0  | ✔  | ✔  | ✔ |  ✔ |
+| Release 3.1.0  | ✔  | ✔  | ✔ |  ✔ |
 
 ## IAM Permissions
 The required IAM permissions to create resources from this module can be found [here](https://github.com/squareops/terraform-aws-eks-bootstrap/blob/main/IAM.md)
@@ -180,7 +183,7 @@ Velero is designed to work with cloud native environments, making it a popular c
 
 ## Notes
 
-Before enabling the **Kubecost** addon for your Amazon EKS cluster, please make sure to subscribe to the **Kubecost - Amazon EKS cost monitoring** license. 
+Before enabling the **Kubecost** addon for your Amazon EKS cluster, please make sure to subscribe to the **Kubecost - Amazon EKS cost monitoring** license.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
@@ -247,7 +250,7 @@ Before enabling the **Kubecost** addon for your Amazon EKS cluster, please make
 | <a name="input_cert_manager_install_letsencrypt_http_issuers"></a> [cert\_manager\_install\_letsencrypt\_http\_issuers](#input\_cert\_manager\_install\_letsencrypt\_http\_issuers) | Enable or disable the HTTP issuer for cert-manager | `bool` | `false` | no |
 | <a name="input_cert_manager_install_letsencrypt_r53_issuers"></a> [cert\_manager\_install\_letsencrypt\_r53\_issuers](#input\_cert\_manager\_install\_letsencrypt\_r53\_issuers) | Enable or disable the creation of Route53 issuer while installing cert manager. | `bool` | `false` | no |
 | <a name="input_cert_manager_letsencrypt_email"></a> [cert\_manager\_letsencrypt\_email](#input\_cert\_manager\_letsencrypt\_email) | Specifies the email address to be used by cert-manager to request Let's Encrypt certificates | `string` | `""` | no |
-| <a name="input_cluster_autoscaler_chart_version"></a> [cluster\_autoscaler\_chart\_version](#input\_cluster\_autoscaler\_chart\_version) | Version of the cluster autoscaler helm chart | `string` | `"9.19.1"` | no |
+| <a name="input_cluster_autoscaler_chart_version"></a> [cluster\_autoscaler\_chart\_version](#input\_cluster\_autoscaler\_chart\_version) | Version of the cluster autoscaler helm chart | `string` | `"9.29.0"` | no |
 | <a name="input_cluster_autoscaler_enabled"></a> [cluster\_autoscaler\_enabled](#input\_cluster\_autoscaler\_enabled) | Whether to enable the Cluster Autoscaler add-on or not. | `bool` | `false` | no |
 | <a name="input_cluster_issuer"></a> [cluster\_issuer](#input\_cluster\_issuer) | Specify the letsecrypt cluster-issuer for ingress tls. | `string` | `"letsencrypt-prod"` | no |
 | <a name="input_cluster_propotional_autoscaler_enabled"></a> [cluster\_propotional\_autoscaler\_enabled](#input\_cluster\_propotional\_autoscaler\_enabled) | Enable or disable Cluster propotional autoscaler add-on | `bool` | `false` | no |
@@ -257,11 +260,12 @@ Before enabling the **Kubecost** addon for your Amazon EKS cluster, please make
 | <a name="input_environment"></a> [environment](#input\_environment) | Environment identifier for the Amazon Elastic Kubernetes Service (EKS) cluster. | `string` | `""` | no |
 | <a name="input_external_secrets_enabled"></a> [external\_secrets\_enabled](#input\_external\_secrets\_enabled) | Enable or disable External Secrets operator add-on for managing external secrets. | `bool` | `false` | no |
 | <a name="input_ingress_nginx_enabled"></a> [ingress\_nginx\_enabled](#input\_ingress\_nginx\_enabled) | Enable or disable Nginx Ingress Controller add-on for routing external traffic to Kubernetes services. | `bool` | `false` | no |
-| <a name="input_ingress_nginx_version"></a> [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Specify the version of the NGINX Ingress Controller | `string` | `"4.1.4"` | no |
+| <a name="input_ingress_nginx_version"></a> [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Specify the version of the NGINX Ingress Controller | `string` | `"4.7.0"` | no |
 | <a name="input_internal_ingress_nginx_enabled"></a> [internal\_ingress\_nginx\_enabled](#input\_internal\_ingress\_nginx\_enabled) | Enable or disable the deployment of an internal ingress controller for Kubernetes. | `bool` | `false` | no |
+| <a name="input_ipv6_enabled"></a> [ipv6\_enabled](#input\_ipv6\_enabled) | Whether enable IPv6 or not | `bool` | `false` | no |
 | <a name="input_istio_enabled"></a> [istio\_enabled](#input\_istio\_enabled) | Enable istio for service mesh. | `bool` | `false` | no |
 | <a name="input_karpenter_enabled"></a> [karpenter\_enabled](#input\_karpenter\_enabled) | Enable or disable Karpenter, a Kubernetes-native, multi-tenant, and auto-scaling solution for containerized workloads on Kubernetes. | `bool` | `false` | no |
-| <a name="input_karpenter_provisioner_config"></a> [karpenter\_provisioner\_config](#input\_karpenter\_provisioner\_config) | Configuration to provide settings for Karpenter, including which private subnet to use, instance capacity types, and excluded instance types. | `any` | <pre>{<br>  "excluded_instance_type": [<br>    "nano",<br>    "micro",<br>    "small"<br>  ],<br>  "instance_capacity_type": [<br>    "spot"<br>  ],<br>  "private_subnet_name": ""<br>}</pre> | no |
+| <a name="input_karpenter_provisioner_config"></a> [karpenter\_provisioner\_config](#input\_karpenter\_provisioner\_config) | Configuration to provide settings for Karpenter, including which private subnet to use, instance capacity types, and excluded instance types. | `any` | <pre>{<br>  "excluded_instance_type": [<br>    "nano",<br>    "micro",<br>    "small"<br>  ],<br>  "instance_capacity_type": [<br>    "spot"<br>  ],<br>  "instance_hypervisor": [<br>    "nitro"<br>  ],<br>  "private_subnet_name": ""<br>}</pre> | no |
 | <a name="input_karpenter_provisioner_enabled"></a> [karpenter\_provisioner\_enabled](#input\_karpenter\_provisioner\_enabled) | Enable or disable the installation of Karpenter, which is a Kubernetes cluster autoscaler. | `bool` | `false` | no |
 | <a name="input_keda_enabled"></a> [keda\_enabled](#input\_keda\_enabled) | Enable or disable Kubernetes Event-driven Autoscaling (KEDA) add-on for autoscaling workloads. | `bool` | `false` | no |
 | <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | ARN of the KMS key used to encrypt AWS resources in the EKS cluster. | `string` | `""` | no |

addons/internal_nginx_ingress/ingress_ipv6.yaml

@@ -0,0 +1,76 @@
+controller:
+  kind: Deployment
+  service:
+    enabled: true
+    annotations:
+      service.beta.kubernetes.io/aws-load-balancer-type: external
+      service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
+      service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+      service.beta.kubernetes.io/aws-load-balancer-ip-address-type: dualstack
+    externalTrafficPolicy: Cluster
+    ipFamilies:
+    - IPv6
+    ipFamilyPolicy: PreferDualStack
+    internal:
+      enabled: false
+      annotations:
+        service.beta.kubernetes.io/aws-load-balancer-type: nlb
+        service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+        service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
+        service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+        service.beta.kubernetes.io/aws-load-balancer-ip-address-type: dualstack
+      ingressClass: internal-nginx
+  ingressClassResource:
+    enabled: true
+    name: internal-nginx
+  ingressClass: internal-nginx
+
+
+  resources:
+    limits:
+      cpu: 500m
+      memory: 750Mi
+    requests:
+      cpu: 50m
+      memory: 200Mi
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+    maxReplicas: 10
+    targetCPUUtilizationPercentage: 80
+    targetMemoryUtilizationPercentage: 80
+  podAnnotations:
+    co.elastic.logs/enabled: "true"
+    co.elastic.logs/module: nginx
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchExpressions:
+          - key: app.kubernetes.io/name
+            operator: In
+            values:
+            - ingress-nginx
+          - key: app.kubernetes.io/instance
+            operator: In
+            values:
+            - ingress-nginx
+          - key: app.kubernetes.io/component
+            operator: In
+            values:
+            - controller
+        topologyKey: "kubernetes.io/hostname"
+
+
+
+
+
+## Enabling metrics for prometheus monitoring
+
+  metrics:
+    enabled: ${enable_service_monitor}
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        release: "prometheus-operator"

addons/karpenter_provisioner/README.md

@@ -26,6 +26,8 @@ No modules.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_excluded_karpenter_ec2_instance_type"></a> [excluded\_karpenter\_ec2\_instance\_type](#input\_excluded\_karpenter\_ec2\_instance\_type) | List of instance types that can be used by Karpenter | `list(string)` | <pre>[<br>  ""<br>]</pre> | no |
+| <a name="input_instance_hypervisor"></a> [instance\_hypervisor](#input\_instance\_hypervisor) | List of instance hypervisor that can be used by Karpenter | `list(string)` | <pre>[<br>  ""<br>]</pre> | no |
+| <a name="input_ipv6_enabled"></a> [ipv6\_enabled](#input\_ipv6\_enabled) | whether IPv6 enabled or not | `bool` | `false` | no |
 | <a name="input_karpenter_ec2_capacity_type"></a> [karpenter\_ec2\_capacity\_type](#input\_karpenter\_ec2\_capacity\_type) | EC2 provisioning capacity type | `list(string)` | <pre>[<br>  ""<br>]</pre> | no |
 | <a name="input_sg_selector_name"></a> [sg\_selector\_name](#input\_sg\_selector\_name) | Name of security group selector for karpenter provisioner. | `string` | `""` | no |
 | <a name="input_subnet_selector_name"></a> [subnet\_selector\_name](#input\_subnet\_selector\_name) | Name of subnet selector for karpenter provisioner. | `string` | `""` | no |

addons/karpenter_provisioner/karpenter-provisioner/ipv4-values.yaml

@@ -0,0 +1,4 @@
+subnet_selector_name: "${subnet_selector_name}"
+sg_selector_name: "${sg_selector_name}"
+karpenter_ec2_capacity_type: "${karpenter_ec2_capacity_type}"
+excluded_karpenter_ec2_instance_type: "${excluded_karpenter_ec2_instance_type}"

addons/karpenter_provisioner/karpenter-provisioner/ipv6-values.yaml

@@ -0,0 +1,5 @@
+subnet_selector_name: "${subnet_selector_name}"
+sg_selector_name: "${sg_selector_name}"
+karpenter_ec2_capacity_type: "${karpenter_ec2_capacity_type}"
+excluded_karpenter_ec2_instance_type: "${excluded_karpenter_ec2_instance_type}"
+karpenter_instance_hypervisor: "${instance_hypervisor}"

addons/karpenter_provisioner/karpenter-provisioner/templates/provisioner.yaml

@@ -12,6 +12,11 @@ spec:
     - key: karpenter.k8s.aws/instance-size
       operator: NotIn
       values: {{ .Values.karpenter_ec2_instance_type }}
+    {{- if .Values.karpenter_instance_hypervisor }}
+    - key: "karpenter.k8s.aws/instance-hypervisor"
+      operator: In
+      values: {{ .Values.karpenter_instance_hypervisor }}
+    {{- end }}
   providerRef:                                # optional, recommended to use instead of `provider`
     name: karpenter-node-template
   ttlSecondsAfterEmpty: 300                   # optional, but never scales down if not set

addons/karpenter_provisioner/main.tf

@@ -2,8 +2,16 @@ resource "helm_release" "karpenter_provisioner" {
   name    = "karpenter-provisioner"
   chart   = "${path.module}/karpenter-provisioner/"
   timeout = 600
-  values = [
-    templatefile("${path.module}/karpenter-provisioner/values.yaml", {
+  values = var.ipv6_enabled == true ? [
+    templatefile("${path.module}/karpenter-provisioner/ipv6-values.yaml", {
+      subnet_selector_name                 = var.subnet_selector_name,
+      sg_selector_name                     = var.sg_selector_name,
+      karpenter_ec2_capacity_type          = "[${join(",", [for s in var.karpenter_ec2_capacity_type : format("%s", s)])}]",
+      excluded_karpenter_ec2_instance_type = "[${join(",", var.excluded_karpenter_ec2_instance_type)}]"
+      instance_hypervisor                  = "[${join(",", var.instance_hypervisor)}]"
+    })
+    ] : [
+    templatefile("${path.module}/karpenter-provisioner/ipv4-values.yaml", {
       subnet_selector_name                 = var.subnet_selector_name,
       sg_selector_name                     = var.sg_selector_name,
       karpenter_ec2_capacity_type          = "[${join(",", [for s in var.karpenter_ec2_capacity_type : format("%s", s)])}]",

addons/karpenter_provisioner/variable.tf

@@ -21,3 +21,15 @@ variable "excluded_karpenter_ec2_instance_type" {
   type        = list(string)
   default     = [""]
 }
+
+variable "instance_hypervisor" {
+  description = "List of instance hypervisor that can be used by Karpenter"
+  type        = list(string)
+  default     = [""]
+}
+
+variable "ipv6_enabled" {
+  description = "whether IPv6 enabled or not"
+  type        = bool
+  default     = false
+}

addons/nginx_ingress/nginx_ingress_ipv6.yaml

@@ -0,0 +1,59 @@
+## Set kind to DaemonSet so no affinity is assigned to it
+
+controller:
+  kind: Deployment
+  service:
+    annotations:
+      service.beta.kubernetes.io/aws-load-balancer-type: external
+      service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
+      service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
+      service.beta.kubernetes.io/aws-load-balancer-ip-address-type: dualstack
+    externalTrafficPolicy: Cluster
+    ipFamilies:
+    - IPv6
+    ipFamilyPolicy: PreferDualStack
+  resources:
+    limits:
+      cpu: 500m
+      memory: 750Mi
+    requests:
+      cpu: 50m
+      memory: 200Mi
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+    maxReplicas: 10
+    targetCPUUtilizationPercentage: 80
+    targetMemoryUtilizationPercentage: 80
+  podAnnotations:
+    co.elastic.logs/enabled: "true"
+    co.elastic.logs/module: nginx
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchExpressions:
+          - key: app.kubernetes.io/name
+            operator: In
+            values:
+            - ingress-nginx
+          - key: app.kubernetes.io/instance
+            operator: In
+            values:
+            - ingress-nginx
+          - key: app.kubernetes.io/component
+            operator: In
+            values:
+            - controller
+        topologyKey: "kubernetes.io/hostname"
+
+
+
+## Enabling metrics for prometheus monitoring
+
+  metrics:
+    enabled: ${enable_service_monitor}
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        release: "prometheus-operator"