diff --git a/README.md b/README.md index 1a7a54e..de847b5 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,11 @@ This module provides an opinionated way to configure an AWS EKS cluster using: Here's an example using a VPC defined using the [terraform-aws-vpc](https://github.com/terraform-aws-modules/terraform-aws-vpc) module: ``` -data "aws_availability_zones" "default" {} +data "aws_availability_zones" "current" {} locals { cluster_name = "test-eks" - vpc_azs = slice(data.aws_availability_zones.default.names, 0, 2) + vpc_azs = slice(data.aws_availability_zones.current.names, 0, 2) vpc_cidr = "10.100.0.0/16" vpc_subnets = cidrsubnets(local.vpc_cidr, 6, 6, 4, 4) diff --git a/cert-manager.tf b/cert-manager.tf new file mode 100644 index 0000000..580fff9 --- /dev/null +++ b/cert-manager.tf @@ -0,0 +1,94 @@ +## cert-manager +locals { + cert_manager = length(var.cert_manager_route53_zone_id) > 0 +} + +module "cert_manager_irsa" { + count = local.cert_manager ? 1 : 0 + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-cert-manager-role" + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = [ + "${var.cert_manager_namespace}:cert-manager", + ] + } + } + tags = var.tags +} + +data "aws_iam_policy_document" "cert_manager" { + count = local.cert_manager ? 1 : 0 + statement { + actions = [ + "route53:GetChange" + ] + resources = ["arn:${local.aws_partition}:route53:::change/*"] + } + + statement { + actions = [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + ] + resources = ["arn:${local.aws_partition}:route53:::hostedzone/${var.cert_manager_route53_zone_id}"] + } +} + +resource "aws_iam_policy" "cert_manager" { + count = local.cert_manager ? 1 : 0 + name = "AmazonEKS_Cert_Manager_Policy-${var.cluster_name}" + description = "Provides permissions for cert-manager" + policy = data.aws_iam_policy_document.cert_manager[0].json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "cert_manager" { + count = local.cert_manager ? 1 : 0 + role = "${var.cluster_name}-cert-manager-role" + policy_arn = aws_iam_policy.cert_manager[0].arn + depends_on = [ + module.cert_manager_irsa[0] + ] +} + +resource "helm_release" "cert_manager" { + count = local.cert_manager ? 1 : 0 + name = "cert-manager" + namespace = var.cert_manager_namespace + create_namespace = var.cert_manager_namespace == "kube-system" ? false : true + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = "v${var.cert_manager_version}" + wait = var.cert_manager_wait + keyring = "${path.module}/cert-manager-keyring.gpg" + verify = var.helm_verify + + # Set up values so CRDs are installed with the chart, the service account has + # correct annotations, and that the pod's security context has permissions + # to read the account token: + # https://cert-manager.io/docs/configuration/acme/dns01/route53/#service-annotation + values = [ + yamlencode({ + "installCRDs" = true + "securityContext" = { + "fsGroup" = 1001 + } + "serviceAccount" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${var.cluster_name}-cert-manager-role" + } + } + }), + yamlencode(var.cert_manager_values), + ] + + depends_on = [ + module.cert_manager_irsa[0], + module.eks, + ] +} diff --git a/cni.tf b/cni.tf new file mode 100644 index 0000000..930ae87 --- /dev/null +++ b/cni.tf @@ -0,0 +1,18 @@ +# Authorize VPC CNI via IRSA. +module "eks_vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-vpc-cni-role" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = var.tags +} diff --git a/ebs-csi.tf b/ebs-csi.tf new file mode 100644 index 0000000..6039ef4 --- /dev/null +++ b/ebs-csi.tf @@ -0,0 +1,94 @@ +## EBS CSI Storage Driver + +# Allow PVCs backed by EBS +module "eks_ebs_csi_irsa" { + count = var.ebs_csi_driver ? 1 : 0 + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-ebs-csi-role" + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["${var.ebs_csi_driver_namespace}:ebs-csi-controller-sa"] + } + } + + tags = var.tags +} + +resource "helm_release" "aws_ebs_csi_driver" { + count = var.ebs_csi_driver ? 1 : 0 + chart = "aws-ebs-csi-driver" + create_namespace = var.ebs_csi_driver_namespace == "kube-system" ? false : true + name = "aws-ebs-csi-driver" + namespace = var.ebs_csi_driver_namespace + repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver" + version = var.ebs_csi_driver_version + wait = var.ebs_csi_driver_wait + + values = [ + yamlencode({ + "controller" = { + "extraVolumeTags" = var.tags + "serviceAccount" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${var.cluster_name}-ebs-csi-role" + } + } + } + "image" = { + "repository" = "${var.csi_ecr_repository_id}.dkr.ecr.${local.aws_region}.amazonaws.com/eks/aws-ebs-csi-driver" + } + }), + yamlencode(var.ebs_csi_driver_values), + ] + + depends_on = [ + module.eks_ebs_csi_irsa[0], + module.eks, + ] +} + +# Make EBS CSI with gp3 default storage driver +resource "kubernetes_storage_class" "eks_ebs_storage_class" { + count = var.ebs_csi_driver ? 1 : 0 + + metadata { + annotations = { + "storageclass.kubernetes.io/is-default-class" = "true" + } + labels = {} + name = "ebs-sc" + } + + mount_options = [] + parameters = {} + storage_provisioner = "ebs.csi.aws.com" + volume_binding_mode = "WaitForFirstConsumer" + + depends_on = [ + helm_release.aws_ebs_csi_driver[0], + ] +} + +# Don't want gp2 storageclass set as default. +resource "kubernetes_annotations" "eks_disable_gp2" { + count = var.ebs_csi_driver ? 1 : 0 + + api_version = "storage.k8s.io/v1" + kind = "StorageClass" + metadata { + name = "gp2" + } + annotations = { + "storageclass.kubernetes.io/is-default-class" = "false" + } + force = true + + depends_on = [ + kubernetes_storage_class.eks_ebs_storage_class[0] + ] +} diff --git a/efs-csi.tf b/efs-csi.tf new file mode 100644 index 0000000..af68673 --- /dev/null +++ b/efs-csi.tf @@ -0,0 +1,218 @@ +## EFS CSI Storage Driver + +locals { + efs_arns = [ + "arn:${local.aws_partition}:elasticfilesystem:${local.aws_region}:${local.aws_account_id}:file-system/*" + ] + efs_access_point_arns = [ + "arn:${local.aws_partition}:elasticfilesystem:${local.aws_region}:${local.aws_account_id}:access-point/*" + ] +} + +# Allow PVCs backed by EFS +module "eks_efs_csi_controller_irsa" { + count = var.efs_csi_driver ? 1 : 0 + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-efs-csi-controller-role" + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = [ + "${var.efs_csi_driver_namespace}:efs-csi-controller-sa", + ] + } + } + tags = var.tags +} + +module "eks_efs_csi_node_irsa" { + count = var.efs_csi_driver ? 1 : 0 + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-efs-csi-node-role" + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = [ + "${var.efs_csi_driver_namespace}:efs-csi-node-sa", + ] + } + } + tags = var.tags +} + +data "aws_iam_policy_document" "eks_efs_csi_driver" { + count = var.efs_csi_driver ? 1 : 0 + + statement { + sid = "AllowDescribeAvailabilityZones" + actions = ["ec2:DescribeAvailabilityZones"] + resources = ["*"] # tfsec:ignore:aws-iam-no-policy-wildcards + } + + statement { + sid = "AllowDescribeFileSystems" + actions = [ + "elasticfilesystem:DescribeAccessPoints", + "elasticfilesystem:DescribeFileSystems", + "elasticfilesystem:DescribeMountTargets" + ] + resources = flatten([ + local.efs_arns, + local.efs_access_point_arns, + ]) + } + + statement { + actions = [ + "elasticfilesystem:CreateAccessPoint", + "elasticfilesystem:TagResource", + ] + resources = local.efs_arns + + condition { + test = "StringLike" + variable = "aws:RequestTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + sid = "AllowDeleteAccessPoint" + actions = ["elasticfilesystem:DeleteAccessPoint"] + resources = local.efs_access_point_arns + + condition { + test = "StringLike" + variable = "aws:ResourceTag/efs.csi.aws.com/cluster" + values = ["true"] + } + } + + statement { + sid = "ClientReadWrite" + actions = [ + "elasticfilesystem:ClientRootAccess", + "elasticfilesystem:ClientWrite", + "elasticfilesystem:ClientMount", + ] + resources = local.efs_arns + + condition { + test = "Bool" + variable = "elasticfilesystem:AccessedViaMountTarget" + values = ["true"] + } + } +} + +resource "aws_iam_policy" "eks_efs_csi_driver" { + count = var.efs_csi_driver ? 1 : 0 + name = "AmazonEKS_EFS_CSI_Policy-${var.cluster_name}" + description = "Provides permissions to manage EFS volumes via the container storage interface driver" + policy = data.aws_iam_policy_document.eks_efs_csi_driver[0].json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "eks_efs_csi_controller" { + count = var.efs_csi_driver ? 1 : 0 + role = "${var.cluster_name}-efs-csi-controller-role" + policy_arn = aws_iam_policy.eks_efs_csi_driver[0].arn + depends_on = [ + module.eks_efs_csi_controller_irsa[0] + ] +} + +resource "aws_iam_role_policy_attachment" "eks_efs_csi_node" { + count = var.efs_csi_driver ? 1 : 0 + role = "${var.cluster_name}-efs-csi-node-role" + policy_arn = aws_iam_policy.eks_efs_csi_driver[0].arn + depends_on = [ + module.eks_efs_csi_node_irsa[0] + ] +} + +resource "aws_efs_file_system" "eks_efs" { + count = var.efs_csi_driver ? 1 : 0 + creation_token = "${var.cluster_name}-efs" + encrypted = true + kms_key_id = var.kms_manage ? aws_kms_key.this[0].arn : module.eks.kms_key_arn + tags = var.tags +} + +resource "aws_efs_mount_target" "eks_efs_private" { + count = var.efs_csi_driver ? length(var.private_subnets) : 0 + file_system_id = aws_efs_file_system.eks_efs[0].id + subnet_id = var.private_subnets[count.index] + security_groups = [module.eks.cluster_primary_security_group_id] +} + +resource "helm_release" "aws_efs_csi_driver" { + count = var.efs_csi_driver ? 1 : 0 + chart = "aws-efs-csi-driver" + create_namespace = var.efs_csi_driver_namespace == "kube-system" ? false : true + name = "aws-efs-csi-driver" + namespace = var.efs_csi_driver_namespace + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" + version = var.efs_csi_driver_version + wait = var.efs_csi_driver_wait + + + values = [ + yamlencode({ + "controller" = { + "serviceAccount" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${var.cluster_name}-efs-csi-controller-role" + } + } + "tags" = var.tags + } + "image" = { + "repository" = "${var.csi_ecr_repository_id}.dkr.ecr.${local.aws_region}.amazonaws.com/eks/aws-efs-csi-driver" + } + "node" = { + "serviceAccount" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${var.cluster_name}-efs-csi-node-role" + } + } + } + }), + yamlencode(var.efs_csi_driver_values), + ] + + depends_on = [ + module.eks_efs_csi_controller_irsa[0], + module.eks, + ] +} + +resource "kubernetes_storage_class" "eks_efs_storage_class" { + count = var.efs_csi_driver ? 1 : 0 + + metadata { + annotations = {} + name = "efs-sc" + labels = {} + } + + mount_options = [] + parameters = { + "provisioningMode" = "efs-ap" + "fileSystemId" = aws_efs_file_system.eks_efs[0].id + "directoryPerms" = "755" + "uid" = "0" + "gid" = "0" + } + storage_provisioner = "efs.csi.aws.com" + + depends_on = [ + helm_release.aws_efs_csi_driver[0], + ] +} diff --git a/karpenter.tf b/karpenter.tf new file mode 100644 index 0000000..2a46b8e --- /dev/null +++ b/karpenter.tf @@ -0,0 +1,58 @@ +module "karpenter" { + # XXX: Switch source back to module once there is an official v20 release, refs + # terraform-aws-modules/terraform-aws-eks#2858 + count = var.karpenter ? 1 : 0 + source = "github.com/radiant-maxar/terraform-aws-eks//modules/karpenter?ref=v20.0.0-alpha" + # source = "terraform-aws-modules/eks/aws//modules/karpenter" + # version = "20.x.x" + cluster_name = var.cluster_name + iam_role_attach_cni_policy = var.iam_role_attach_cni_policy + irsa_namespace_service_accounts = ["${var.karpenter_namespace}:karpenter"] + irsa_oidc_provider_arn = module.eks.oidc_provider_arn + tags = var.tags +} + +resource "helm_release" "karpenter_crd" { + count = var.karpenter ? 1 : 0 + create_namespace = var.karpenter_namespace == "kube-system" ? false : true + name = "karpenter-crd" + namespace = var.karpenter_namespace + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter-crd" + version = "v${var.karpenter_version}" + + depends_on = [ + module.eks, + module.karpenter[0], + ] +} + +resource "helm_release" "karpenter" { + count = var.karpenter ? 1 : 0 + name = "karpenter" + namespace = var.karpenter_namespace + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = "v${var.karpenter_version}" + wait = var.karpenter_wait + + values = [ + yamlencode({ + serviceAccount = { + annotations = { + "eks.amazonaws.com/role-arn" = module.karpenter[0].pod_identity_role_arn + } + } + settings = { + clusterEndpoint = module.eks.cluster_endpoint + clusterName = var.cluster_name + interruptionQueue = module.karpenter[0].queue_name + } + }), + yamlencode(var.karpenter_values), + ] + + depends_on = [ + helm_release.karpenter_crd[0] + ] +} diff --git a/kms.tf b/kms.tf new file mode 100644 index 0000000..c7619e2 --- /dev/null +++ b/kms.tf @@ -0,0 +1,7 @@ +resource "aws_kms_key" "this" { + count = var.kms_manage ? 1 : 0 + deletion_window_in_days = var.kms_key_deletion_window_in_days + description = "KMS Key for EKS Secrets" + enable_key_rotation = true + tags = var.tags +} diff --git a/lb.tf b/lb.tf new file mode 100644 index 0000000..c74adcf --- /dev/null +++ b/lb.tf @@ -0,0 +1,53 @@ +## AWS Load Balancer Controller + +# Authorize Amazon Load Balancer Controller +module "eks_lb_irsa" { + count = var.lb_controller ? 1 : 0 + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.33.0" + + role_name = "${var.cluster_name}-lb-role" + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["${var.lb_controller_namespace}:aws-load-balancer-controller"] + } + } + + tags = var.tags +} + +resource "helm_release" "aws_lb_controller" { + count = var.lb_controller ? 1 : 0 + chart = "aws-load-balancer-controller" + create_namespace = var.lb_controller_namespace == "kube-system" ? false : true + name = "aws-load-balancer-controller" + namespace = var.lb_controller_namespace + repository = "https://aws.github.io/eks-charts" + version = var.lb_controller_version + wait = var.lb_controller_wait + + values = [ + yamlencode({ + "clusterName" = var.cluster_name + "defaultTags" = var.tags + "region" = local.aws_region + "serviceAccount" = { + "annotations" = { + "eks.amazonaws.com/role-arn" = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${var.cluster_name}-lb-role" + "eks.amazonaws.com/sts-regional-endpoints" = "true" + } + "name" = "aws-load-balancer-controller" + } + "vpcId" = var.vpc_id + }), + yamlencode(var.lb_controller_values), + ] + + depends_on = [ + module.eks_lb_irsa[0], + module.eks, + ] +} diff --git a/main.tf b/main.tf index 02a64d4..09a5b27 100644 --- a/main.tf +++ b/main.tf @@ -1,72 +1,74 @@ -data "aws_caller_identity" "default" {} -data "aws_region" "default" {} +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} +data "aws_region" "current" {} locals { - aws_account_id = data.aws_caller_identity.default.account_id - aws_region = data.aws_region.default.name - aws_auth_roles = [ - for role in var.system_masters_roles : { - rolearn = "arn:aws:iam::${local.aws_account_id}:role/${role}" - username = role - groups = ["system:masters"] - } - ] - - cert_manager = length(var.cert_manager_route53_zone_id) > 0 -} - -resource "aws_kms_key" "this" { - deletion_window_in_days = 10 - description = "KMS Key for EKS Secrets" - enable_key_rotation = true - tags = var.tags -} - -resource "aws_security_group" "eks_efs_sg" { - name = "${var.cluster_name}-efs-sg" - description = "Security group for EFS clients in EKS VPC" - vpc_id = var.vpc_id - - ingress { - description = "Ingress NFS traffic for EFS" - from_port = 2049 - to_port = 2049 - protocol = "tcp" - cidr_blocks = [var.vpc_cidr] - } - - tags = var.tags + aws_account_id = data.aws_caller_identity.current.account_id + aws_partition = data.aws_partition.current.partition + aws_region = data.aws_region.current.name + aws_auth_karpenter_roles = var.karpenter ? [ + { + rolearn = module.karpenter[0].role_arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes", + ] + }, + ] : [] + aws_auth_roles = concat( + [ + for role in var.system_masters_roles : { + rolearn = "arn:${local.aws_partition}:iam::${local.aws_account_id}:role/${role}" + username = role + groups = ["system:masters"] + } + ], + local.aws_auth_karpenter_roles, + var.aws_auth_roles + ) } # EKS Cluster module "eks" { # tfsec:ignore:aws-ec2-no-public-egress-sgr tfsec:ignore:aws-eks-no-public-cluster-access tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr source = "terraform-aws-modules/eks/aws" - version = "19.17.2" + version = "19.21.0" cluster_name = var.cluster_name cluster_version = var.kubernetes_version - cluster_addons = { - coredns = { - most_recent = var.cluster_addons_most_recent - preserve = true - } - kube-proxy = { - most_recent = var.cluster_addons_most_recent - preserve = true - } - vpc-cni = { - most_recent = var.cluster_addons_most_recent - preserve = true - service_account_role_arn = module.eks_vpc_cni_irsa.iam_role_arn - } - } - cluster_addons_timeouts = var.cluster_addons_timeouts - cluster_enabled_log_types = var.cluster_enabled_log_types - cluster_encryption_config = { - provider_key_arn = aws_kms_key.this.arn + cluster_addons = merge( + var.cluster_addons_coredns ? { + coredns = { + most_recent = var.cluster_addons_most_recent + } + } : {}, + { + eks-pod-identity-agent = { + most_recent = var.cluster_addons_most_recent + } + kube-proxy = { + most_recent = var.cluster_addons_most_recent + } + vpc-cni = { + most_recent = var.cluster_addons_most_recent + service_account_role_arn = module.eks_vpc_cni_irsa.iam_role_arn + } + }, + var.cluster_addons_overrides + ) + cluster_addons_timeouts = var.cluster_addons_timeouts + cluster_enabled_log_types = var.cluster_enabled_log_types + create_cluster_security_group = var.create_cluster_security_group + create_node_security_group = var.create_node_security_group + + cluster_encryption_config = var.kms_manage ? { + provider_key_arn = aws_kms_key.this[0].arn resources = ["secrets"] - } + } : { resources = ["secrets"] } + create_kms_key = var.kms_manage ? false : true + kms_key_deletion_window_in_days = var.kms_key_deletion_window_in_days + kms_key_enable_default_policy = var.kms_key_enable_default_policy cluster_endpoint_private_access = var.cluster_endpoint_private_access cluster_endpoint_public_access = var.cluster_endpoint_public_access @@ -74,7 +76,6 @@ module "eks" { # tfsec:ignore:aws-ec2-no-public-egress-sgr tfsec:ignore:aws-eks- cluster_security_group_additional_rules = var.cluster_security_group_additional_rules aws_auth_roles = local.aws_auth_roles - create_kms_key = false manage_aws_auth_configmap = true enable_irsa = true subnet_ids = concat(var.public_subnets, var.private_subnets) @@ -90,14 +91,18 @@ module "eks" { # tfsec:ignore:aws-ec2-no-public-egress-sgr tfsec:ignore:aws-eks- iam_role_attach_cni_policy = var.iam_role_attach_cni_policy max_size = var.default_max_size min_size = var.default_min_size - vpc_security_group_ids = [ - aws_security_group.eks_efs_sg.id, - ] } eks_managed_node_groups = var.eks_managed_node_groups + fargate_profiles = var.fargate_profiles + fargate_profile_defaults = var.fargate_profile_defaults + node_security_group_tags = var.node_security_group_tags - tags = var.tags + + tags = merge( + var.tags, + var.karpenter ? { "karpenter.sh/discovery" = var.cluster_name } : {} + ) } # Add EKS to default kubeconfig and set context for it. @@ -110,136 +115,6 @@ resource "null_resource" "eks_kubeconfig" { ] } -# Authorize Amazon Load Balancer Controller -module "eks_lb_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-lb-role" - attach_load_balancer_controller_policy = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] - } - } - - tags = var.tags -} - -# Authorize VPC CNI via IRSA. -module "eks_vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-vpc-cni-role" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv4 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } - - tags = var.tags -} - -# Allow PVCs backed by EBS -module "eks_ebs_csi_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-ebs-csi-role" - attach_ebs_csi_policy = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] - } - } - - tags = var.tags -} - -# Allow PVCs backed by EFS -module "eks_efs_csi_controller_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-efs-csi-controller-role" - attach_efs_csi_policy = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = [ - "kube-system:efs-csi-controller-sa", - ] - } - } - tags = var.tags -} - -module "eks_efs_csi_node_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-efs-csi-node-role" - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = [ - "kube-system:efs-csi-node-sa", - ] - } - } - tags = var.tags -} - -data "aws_iam_policy_document" "eks_efs_csi_node" { - statement { - actions = [ - "elasticfilesystem:DescribeMountTargets", - "ec2:DescribeAvailabilityZones", - ] - resources = ["*"] # tfsec:ignore:aws-iam-no-policy-wildcards - } -} - -resource "aws_iam_policy" "eks_efs_csi_node" { - name = "AmazonEKS_EFS_CSI_Node_Policy-${var.cluster_name}" - description = "Provides node permissions to use the EFS CSI driver" - policy = data.aws_iam_policy_document.eks_efs_csi_node.json - tags = var.tags -} - -resource "aws_iam_role_policy_attachment" "eks_efs_csi_node" { - role = "${var.cluster_name}-efs-csi-node-role" - policy_arn = aws_iam_policy.eks_efs_csi_node.arn - depends_on = [ - module.eks_efs_csi_node_irsa - ] -} - -resource "aws_efs_file_system" "eks_efs" { - creation_token = "${var.cluster_name}-efs" - encrypted = true - kms_key_id = aws_kms_key.this.arn - tags = var.tags -} - -resource "aws_efs_mount_target" "eks_efs_private" { - count = length(var.private_subnets) - file_system_id = aws_efs_file_system.eks_efs.id - subnet_id = var.private_subnets[count.index] - security_groups = [aws_security_group.eks_efs_sg.id] -} - - ## Kubernetes-level configuration provider "helm" { @@ -266,259 +141,3 @@ provider "kubernetes" { args = ["eks", "get-token", "--region", local.aws_region, "--cluster-name", module.eks.cluster_name] } } - -## AWS Load Balancer Controller -resource "helm_release" "aws_lb_controller" { - name = "aws-load-balancer-controller" - namespace = "kube-system" - chart = "aws-load-balancer-controller" - repository = "https://aws.github.io/eks-charts" - version = var.lb_controller_version - - values = [ - yamlencode({ - "clusterName" = var.cluster_name - "defaultTags" = var.tags - "region" = local.aws_region - "serviceAccount" = { - "annotations" = { - "eks.amazonaws.com/role-arn" = "arn:aws:iam::${local.aws_account_id}:role/${var.cluster_name}-lb-role" - "eks.amazonaws.com/sts-regional-endpoints" = "true" - } - "name" = "aws-load-balancer-controller" - } - "vpcId" = var.vpc_id - }) - ] - - depends_on = [ - module.eks_lb_irsa, - module.eks, - ] -} - -## EBS CSI Storage Driver -resource "helm_release" "aws_ebs_csi_driver" { - name = "aws-ebs-csi-driver" - namespace = "kube-system" - chart = "aws-ebs-csi-driver" - repository = "https://kubernetes-sigs.github.io/aws-ebs-csi-driver" - version = var.ebs_csi_driver_version - - values = [ - yamlencode({ - "controller" = { - "extraVolumeTags" = var.tags - "serviceAccount" = { - "annotations" = { - "eks.amazonaws.com/role-arn" = "arn:aws:iam::${local.aws_account_id}:role/${var.cluster_name}-ebs-csi-role" - } - } - } - "image" = { - "repository" = "${var.csi_ecr_repository_id}.dkr.ecr.${local.aws_region}.amazonaws.com/eks/aws-ebs-csi-driver" - } - }) - ] - - depends_on = [ - module.eks_ebs_csi_irsa, - module.eks, - ] -} - -# Make EBS CSI with gp3 default storage driver -resource "kubernetes_storage_class" "eks_ebs_storage_class" { - metadata { - annotations = { - "storageclass.kubernetes.io/is-default-class" = "true" - } - labels = {} - name = "ebs-sc" - } - - mount_options = [] - parameters = {} - storage_provisioner = "ebs.csi.aws.com" - volume_binding_mode = "WaitForFirstConsumer" - - depends_on = [ - helm_release.aws_ebs_csi_driver, - ] -} - -# Don't want gp2 storageclass set as default. -resource "kubernetes_annotations" "eks_disable_gp2" { - api_version = "storage.k8s.io/v1" - kind = "StorageClass" - metadata { - name = "gp2" - } - annotations = { - "storageclass.kubernetes.io/is-default-class" = "false" - } - force = true - - depends_on = [ - kubernetes_storage_class.eks_ebs_storage_class - ] -} - -## EFS CSI Storage Driver -resource "helm_release" "aws_efs_csi_driver" { - name = "aws-efs-csi-driver" - namespace = "kube-system" - chart = "aws-efs-csi-driver" - repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver" - version = var.efs_csi_driver_version - - values = [ - yamlencode({ - "controller" = { - "serviceAccount" = { - "annotations" = { - "eks.amazonaws.com/role-arn" = "arn:aws:iam::${local.aws_account_id}:role/${var.cluster_name}-efs-csi-controller-role" - } - } - "tags" = var.tags - } - "image" = { - "repository" = "${var.csi_ecr_repository_id}.dkr.ecr.${local.aws_region}.amazonaws.com/eks/aws-efs-csi-driver" - } - "node" = { - "serviceAccount" = { - "annotations" = { - "eks.amazonaws.com/role-arn" = "arn:aws:iam::${local.aws_account_id}:role/${var.cluster_name}-efs-csi-node-role" - } - } - } - }) - ] - - depends_on = [ - module.eks_efs_csi_controller_irsa, - module.eks, - ] -} - -resource "kubernetes_storage_class" "eks_efs_storage_class" { - metadata { - annotations = {} - name = "efs-sc" - labels = {} - } - - mount_options = [] - parameters = { - "provisioningMode" = "efs-ap" - "fileSystemId" = aws_efs_file_system.eks_efs.id - "directoryPerms" = "755" - "uid" = "0" - "gid" = "0" - } - storage_provisioner = "efs.csi.aws.com" - - depends_on = [ - helm_release.aws_efs_csi_driver, - ] -} - -## Nvidia Device Plugin for GPU support -resource "null_resource" "eks_nvidia_device_plugin" { - count = var.nvidia_device_plugin ? 1 : 0 - provisioner "local-exec" { - command = "kubectl --context='${var.cluster_name}' apply --filename='https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v${var.nvidia_device_plugin_version}/nvidia-device-plugin.yml'" - } - depends_on = [ - helm_release.aws_lb_controller, - ] -} - -## cert-manager -module "cert_manager_irsa" { - count = local.cert_manager ? 1 : 0 - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "5.30.0" - - role_name = "${var.cluster_name}-cert-manager-role" - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = [ - "cert-manager:cert-manager", - ] - } - } - tags = var.tags -} - -data "aws_iam_policy_document" "cert_manager" { - count = local.cert_manager ? 1 : 0 - statement { - actions = [ - "route53:GetChange" - ] - resources = ["arn:aws:route53:::change/*"] - } - - statement { - actions = [ - "route53:ChangeResourceRecordSets", - "route53:ListResourceRecordSets", - ] - resources = ["arn:aws:route53:::hostedzone/${var.cert_manager_route53_zone_id}"] - } -} - -resource "aws_iam_policy" "cert_manager" { - count = local.cert_manager ? 1 : 0 - name = "AmazonEKS_Cert_Manager_Policy-${var.cluster_name}" - description = "Provides permissions for cert-manager" - policy = data.aws_iam_policy_document.cert_manager[0].json - tags = var.tags -} - -resource "aws_iam_role_policy_attachment" "cert_manager" { - count = local.cert_manager ? 1 : 0 - role = "${var.cluster_name}-cert-manager-role" - policy_arn = aws_iam_policy.cert_manager[0].arn - depends_on = [ - module.cert_manager_irsa[0] - ] -} - -resource "helm_release" "cert_manager" { - count = local.cert_manager ? 1 : 0 - name = "cert-manager" - namespace = "cert-manager" - create_namespace = true - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = "v${var.cert_manager_version}" - keyring = "${path.module}/cert-manager-keyring.gpg" - verify = var.helm_verify - - # Set up values so CRDs are installed with the chart, the service account has - # correct annotations, and that the pod's security context has permissions - # to read the account token: - # https://cert-manager.io/docs/configuration/acme/dns01/route53/#service-annotation - values = [ - yamlencode({ - "installCRDs" = true - "securityContext" = { - "fsGroup" = 1001 - } - "serviceAccount" = { - "annotations" = { - "eks.amazonaws.com/role-arn" = "arn:aws:iam::${local.aws_account_id}:role/${var.cluster_name}-cert-manager-role" - } - } - }) - ] - - depends_on = [ - module.cert_manager_irsa[0], - module.eks, - ] -} diff --git a/nvidia.tf b/nvidia.tf new file mode 100644 index 0000000..a4c5db7 --- /dev/null +++ b/nvidia.tf @@ -0,0 +1,16 @@ +## NVIDIA GPU Operator + +resource "helm_release" "nvidia_gpu_operator" { + count = var.nvidia_gpu_operator ? 1 : 0 + chart = "gpu-operator" + create_namespace = var.nvidia_gpu_operator_namespace == "kube-system" ? false : true + name = "gpu-operator" + namespace = var.nvidia_gpu_operator_namespace + repository = "https://helm.ngc.nvidia.com/nvidia" + wait = var.nvidia_gpu_operator_wait + version = "v${var.nvidia_gpu_operator_version}" + + depends_on = [ + module.eks, + ] +} diff --git a/outputs.tf b/outputs.tf index febab23..b616292 100644 --- a/outputs.tf +++ b/outputs.tf @@ -13,14 +13,39 @@ output "cluster_oidc_issuer_url" { value = module.eks.cluster_oidc_issuer_url } +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = module.eks.cluster_primary_security_group_id +} + output "eks_managed_node_groups" { description = "Map of attribute maps for all EKS managed node groups created" value = module.eks.eks_managed_node_groups } +output "karpenter_pod_identity_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the Pod Identity IAM role" + value = var.karpenter ? module.karpenter[0].pod_identity_role_arn : null +} + +output "karpenter_pod_identity_role_name" { + description = "The name of the Pod Identity IAM role" + value = var.karpenter ? module.karpenter[0].pod_identity_role_name : null +} + +output "karpenter_role_arn" { + description = "The Amazon Resource Name (ARN) specifying the Karpenter IAM role" + value = var.karpenter ? module.karpenter[0].role_arn : null +} + +output "karpenter_role_name" { + description = "The name of the Karpenter IAM role" + value = var.karpenter ? module.karpenter[0].role_name : null +} + output "kms_key_arn" { description = "The Amazon Resource Name (ARN) of the KMS key for the EKS cluster." - value = aws_kms_key.this.arn + value = var.kms_manage ? aws_kms_key.this[0].arn : module.eks.kms_key_arn } output "node_security_group_arn" { diff --git a/variables.tf b/variables.tf index 43d24c2..d8b077e 100644 --- a/variables.tf +++ b/variables.tf @@ -1,5 +1,29 @@ +variable "aws_auth_roles" { + description = "List of role maps to add to the aws-auth configmap" + type = list(any) + default = [] +} + +variable "cert_manager_namespace" { + default = "cert-manager" + description = "Namespace that cert-manager will use." + type = string +} + +variable "cert_manager_values" { + description = "Additional custom values for the cert-manager Helm chart." + type = map(any) + default = {} +} + +variable "cert_manager_wait" { + description = "Wait for the cert-manager Helm chart installation to complete." + type = bool + default = true +} + variable "cert_manager_version" { - default = "1.13.1" + default = "1.13.3" description = "Version of cert-manager to install." type = string } @@ -10,12 +34,24 @@ variable "cert_manager_route53_zone_id" { type = string } +variable "cluster_addons_coredns" { + description = "Indicates whether to install the CoreDNS cluster addon." + type = bool + default = true +} + variable "cluster_addons_most_recent" { description = "Indicates whether to use the most recent version of cluster addons" type = bool default = true } +variable "cluster_addons_overrides" { + description = "Override parameters for cluster addons." + type = map(any) + default = {} +} + variable "cluster_addons_timeouts" { description = "Create, update, and delete timeout configurations for the cluster addons" type = map(string) @@ -55,6 +91,24 @@ variable "cluster_name" { type = string } +variable "cluster_security_group_additional_rules" { + description = "Additional security group rules to add to the cluster security group created." + type = map(any) + default = {} +} + +variable "create_cluster_security_group" { + description = "Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default" + type = bool + default = true +} + +variable "create_node_security_group" { + description = "Determines whether to create a security group for the node groups or use the existing `node_security_group_id`" + type = bool + default = true +} + # The ECR repository is not the same for every region, in particular # those for govcloud: # https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html @@ -94,21 +148,82 @@ variable "default_max_size" { type = number } +variable "ebs_csi_driver" { + description = "Install and configure the EBS CSI storage driver." + type = bool + default = true +} + +variable "ebs_csi_driver_namespace" { + default = "kube-system" + description = "Namespace that EBS CSI storage driver will use." + type = string +} + +variable "ebs_csi_driver_values" { + description = "Additional custom values for the EBS CSI Driver Helm chart." + type = map(any) + default = {} +} + +variable "ebs_csi_driver_wait" { + description = "Wait for the EBS CSI storage driver Helm chart install to complete." + type = bool + default = true +} + variable "ebs_csi_driver_version" { - default = "2.24.0" + default = "2.25.0" description = "Version of the EFS CSI storage driver to install." type = string } +variable "efs_csi_driver" { + description = "Install and configure the EFS CSI storage driver." + type = bool + default = true +} + +variable "efs_csi_driver_namespace" { + default = "kube-system" + description = "Namespace that EFS CSI storage driver will use." + type = string +} + +variable "efs_csi_driver_values" { + description = "Additional custom values for the EFS CSI Driver Helm chart." + type = map(any) + default = {} +} + variable "efs_csi_driver_version" { - default = "2.5.0" + default = "2.5.2" description = "Version of the EFS CSI storage driver to install." type = string } +variable "efs_csi_driver_wait" { + description = "Wait for the EFS CSI storage driver Helm chart install to complete." + type = bool + default = true +} + variable "eks_managed_node_groups" { - description = "Managed node groups for the EKS cluster." - type = any + description = "Map of managed node groups for the EKS cluster." + type = map(any) + default = {} +} + +variable "fargate_profiles" { + description = "Map of Fargate Profile definitions to create." + type = map(any) + default = {} +} + +variable "fargate_profile_defaults" { + description = "Map of Fargate Profile default configurations." + type = map(any) + default = {} } variable "helm_verify" { @@ -117,28 +232,94 @@ variable "helm_verify" { type = bool } +variable "iam_role_attach_cni_policy" { + default = true + description = "Whether to attach CNI policy to EKS Node groups." + type = bool +} + +variable "karpenter" { + description = "Whether to use Karpenter with the EKS cluster." + type = bool + default = false +} + +variable "karpenter_namespace" { + default = "karpenter" + description = "Namespace that Karpenter will use." + type = string +} + +variable "karpenter_values" { + description = "Additional custom values to use when installing the Karpenter Helm chart." + type = map(any) + default = {} +} + +variable "karpenter_wait" { + description = "Wait for the Karpenter Helm chart installation to complete." + type = bool + default = true +} + +variable "karpenter_version" { + description = "Version of Karpenter Helm chart to install on the EKS cluster." + type = string + default = "0.33.1" +} + +variable "kms_manage" { + default = false + description = "Manage EKS KMS resource instead of the AWS module" + type = bool +} + +variable "kms_key_deletion_window_in_days" { + description = "The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key. If you specify a value, it must be between `7` and `30`, inclusive." + type = number + default = 10 +} + +variable "kms_key_enable_default_policy" { + description = "Specifies whether to enable the default key policy. Defaults to `true` to workaround EFS permissions." + type = bool + default = true +} + variable "kubernetes_version" { default = "1.28" description = "Kubernetes version to use for the EKS cluster." type = string } -variable "iam_role_attach_cni_policy" { - default = true - description = "Whether to attach CNI policy to EKS Node groups." +variable "lb_controller" { + description = "Install and configure the AWS Load Balancer Controller." type = bool + default = true +} + +variable "lb_controller_namespace" { + default = "kube-system" + description = "Namespace that AWS Load Balancer Controller will use." + type = string +} + +variable "lb_controller_values" { + description = "Additional custom values for the AWS Load Balancer Controller Helm chart." + type = map(any) + default = {} } variable "lb_controller_version" { - default = "1.6.1" + default = "1.6.2" description = "Version of the AWS Load Balancer Controller chart to install." type = string } -variable "cluster_security_group_additional_rules" { - description = "Additional security group rules to add to the cluster security group created." - type = any - default = {} +variable "lb_controller_wait" { + description = "Wait for the AWS Load Balancer Controller Helm chart install to complete." + type = bool + default = true } variable "node_security_group_additional_rules" { @@ -170,18 +351,30 @@ variable "node_security_group_tags" { default = {} } -variable "nvidia_device_plugin" { +variable "nvidia_gpu_operator" { default = false - description = "Whether to install the Nvidia device plugin driver" + description = "Whether to install the NVIDIA GPU Operator." type = bool } -variable "nvidia_device_plugin_version" { - default = "0.14.1" - description = "Version of the Nvidia device plugin to install." +variable "nvidia_gpu_operator_namespace" { + default = "nvidia-gpu-operator" + description = "Namespace that NVIDIA GPU Operator will use." type = string } +variable "nvidia_gpu_operator_version" { + default = "23.9.1" + description = "Version of the NVIDIA GPU Operator Helm chart to install." + type = string +} + +variable "nvidia_gpu_operator_wait" { + description = "Wait for the NVIDIA GPU Operator Helm chart installation to complete." + type = bool + default = true +} + variable "private_subnets" { description = "IDs of the private subnets in the EKS cluster VPC." type = list(any)