From 545061a7fd93d0cd2219a1a7d3fcebeff8f87cfb Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Thu, 24 Oct 2024 18:06:09 +0900 Subject: [PATCH 1/6] Enable to attach Karpenter v1 controller policy --- modules/karpenter/controller_iam.tf | 387 ++++++++++++++++++++++++++++ modules/karpenter/variables.tf | 6 + 2 files changed, 393 insertions(+) diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index 7ccdb39..0f97d62 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -388,3 +388,390 @@ data "aws_iam_policy_document" "karpenter_controller_v1_beta" { actions = ["eks:DescribeCluster"] } } + +resource "aws_iam_role_policy" "karpenter_controller_v1" { + count = var.v1 ? 1 : 0 + name = "KarpenterController-v1" + role = aws_iam_role.karpenter_controller.id + policy = data.aws_iam_policy_document.karpenter_controller_v1.json +} + +data "aws_iam_policy_document" "karpenter_controller_v1" { + statement { + sid = "AllowScopedEC2InstanceAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::image/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::snapshot/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:security-group/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:subnet/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + } + + statement { + sid = "AllowScopedEC2LaunchTemplateAccessActions" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedEC2InstanceActionsWithTags" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceCreationTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*", + ] + + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + + values = [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate", + ] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowScopedResourceTagging" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = ["arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*"] + actions = ["ec2:CreateTags"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + + condition { + test = "StringEqualsIfExists" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "ForAllValues:StringEquals" + variable = "aws:TagKeys" + values = ["eks:eks-cluster-name", "karpenter.sh/nodeclaim", "Name"] + } + } + + + statement { + sid = "AllowScopedDeletion" + effect = "Allow" + + # tfsec:ignore:aws-iam-no-policy-wildcards + resources = [ + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*", + ] + + actions = [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.sh/nodepool" + values = ["*"] + } + } + + statement { + sid = "AllowRegionalReadActions" + effect = "Allow" + resources = ["*"] + + actions = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + ] + + condition { + test = "StringEquals" + variable = "aws:RequestedRegion" + values = [data.aws_region.current.name] + } + } + + statement { + sid = "AllowSSMReadActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:ssm:${data.aws_region.current.name}::parameter/aws/service/*"] + actions = ["ssm:GetParameter"] + } + + statement { + sid = "AllowPricingReadActions" + effect = "Allow" + resources = ["*"] + actions = ["pricing:GetProducts"] + } + + statement { + sid = "AllowInterruptionQueueActions" + effect = "Allow" + resources = [aws_sqs_queue.karpenter_interruption.arn] + + actions = [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + } + + statement { + sid = "AllowPassingInstanceRole" + effect = "Allow" + resources = concat([aws_iam_role.karpenter_node.arn], var.additional_node_role_arns) + actions = ["iam:PassRole"] + + condition { + test = "StringEquals" + variable = "iam:PassedToService" + values = ["ec2.amazonaws.com"] + } + } + + statement { + sid = "AllowScopedInstanceProfileCreationActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:CreateInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileTagActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:TagInstanceProfile"] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/eks:eks-cluster-name" + values = [var.cluster_config.name] + } + + condition { + test = "StringEquals" + variable = "aws:RequestTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + + condition { + test = "StringLike" + variable = "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowScopedInstanceProfileActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile", + ] + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}" + values = ["owned"] + } + + condition { + test = "StringEquals" + variable = "aws:ResourceTag/topology.kubernetes.io/region" + values = [data.aws_region.current.name] + } + + condition { + test = "StringLike" + variable = "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass" + values = ["*"] + } + } + + statement { + sid = "AllowInstanceProfileReadActions" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*"] + actions = ["iam:GetInstanceProfile"] + } + + statement { + sid = "AllowAPIServerEndpointDiscovery" + effect = "Allow" + resources = [var.cluster_config.arn] + actions = ["eks:DescribeCluster"] + } +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 9b55d28..a474475 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -17,6 +17,12 @@ variable "oidc_config" { }) } +variable "v1" { + description = "Enable controller policy for v1 resources (Karpenter >= 1.*)" + type = bool + default = true +} + variable "additional_node_role_arns" { description = <<-EOF Additional Node Role ARNS that karpenter should manage From bcf7a28bf75fd38d4ea25a6dd806f9ce7dd13b74 Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Thu, 24 Oct 2024 18:12:33 +0900 Subject: [PATCH 2/6] Enable to detach Karpenter v1beta controller policy --- modules/karpenter/controller_iam.tf | 5 +++-- modules/karpenter/variables.tf | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index 0f97d62..e94c555 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -29,14 +29,15 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 name = "KarpenterController-v1beta" role = aws_iam_role.karpenter_controller.id policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json } moved { - from = aws_iam_role_policy.karpenter_controller_v1_beta[0] - to = aws_iam_role_policy.karpenter_controller_v1_beta + from = aws_iam_role_policy.karpenter_controller_v1_beta + to = aws_iam_role_policy.karpenter_controller_v1_beta[0] } data "aws_iam_policy_document" "karpenter_controller_v1_beta" { diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index a474475..f9c556c 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -17,6 +17,12 @@ variable "oidc_config" { }) } +variable "v1beta" { + description = "Enable controller policy for v1beta resources (Karpenter >= 0.32.*)" + type = bool + default = true +} + variable "v1" { description = "Enable controller policy for v1 resources (Karpenter >= 1.*)" type = bool From 0db1769b80142244b3791dca355b0d24f8d5a141 Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Fri, 25 Oct 2024 12:31:57 +0900 Subject: [PATCH 3/6] Make v1 and v1beta policies mutually exclusive --- modules/karpenter/controller_iam.tf | 2 +- modules/karpenter/variables.tf | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index e94c555..9561ab5 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -29,7 +29,7 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { - count = var.v1beta ? 1 : 0 + count = var.v1 ? 0 : 1 name = "KarpenterController-v1beta" role = aws_iam_role.karpenter_controller.id policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index f9c556c..553edb6 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -17,16 +17,10 @@ variable "oidc_config" { }) } -variable "v1beta" { - description = "Enable controller policy for v1beta resources (Karpenter >= 0.32.*)" - type = bool - default = true -} - variable "v1" { - description = "Enable controller policy for v1 resources (Karpenter >= 1.*)" + description = "Use controller policy for v1 resources (Karpenter >= 1.*)" type = bool - default = true + default = false } variable "additional_node_role_arns" { From c820fe102c9807013e7755942831e6ed254c4f50 Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Fri, 25 Oct 2024 12:56:23 +0900 Subject: [PATCH 4/6] Use Karpenter v1 resources in test --- test/cluster_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/cluster_test.go b/test/cluster_test.go index e788921..d53ce80 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -101,7 +101,7 @@ func installKarpenter(t *testing.T, kubeconfig, clusterName, sgName string) { } const KARPENTER_PROVISIONER = `--- -apiVersion: karpenter.sh/v1beta1 +apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default @@ -109,7 +109,7 @@ spec: template: spec: nodeClassRef: - apiVersion: karpenter.k8s.aws/v1beta1 + group: karpenter.k8s.aws kind: EC2NodeClass name: default requirements: @@ -123,12 +123,13 @@ spec: operator: In values: [small, medium, large] --- -apiVersion: karpenter.k8s.aws/v1beta1 +apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default spec: - amiFamily: Bottlerocket + amiSelectorTerms: + - alias: bottlerocket@latest subnetSelectorTerms: - tags: Name: terraform-aws-eks-test-environment-private* From f4200c1354d9705248ed79802a646342240f8f05 Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Tue, 29 Oct 2024 13:49:17 +0900 Subject: [PATCH 5/6] Revert "Make v1 and v1beta policies mutually exclusive" This reverts commit 0db1769b80142244b3791dca355b0d24f8d5a141. --- modules/karpenter/controller_iam.tf | 2 +- modules/karpenter/variables.tf | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index 9561ab5..e94c555 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -29,7 +29,7 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { - count = var.v1 ? 0 : 1 + count = var.v1beta ? 1 : 0 name = "KarpenterController-v1beta" role = aws_iam_role.karpenter_controller.id policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 553edb6..f9c556c 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -17,10 +17,16 @@ variable "oidc_config" { }) } +variable "v1beta" { + description = "Enable controller policy for v1beta resources (Karpenter >= 0.32.*)" + type = bool + default = true +} + variable "v1" { - description = "Use controller policy for v1 resources (Karpenter >= 1.*)" + description = "Enable controller policy for v1 resources (Karpenter >= 1.*)" type = bool - default = false + default = true } variable "additional_node_role_arns" { From 7257bc8e08b3007616b73c3b950124d2b006024c Mon Sep 17 00:00:00 2001 From: hiromi-ogawa Date: Tue, 29 Oct 2024 14:01:29 +0900 Subject: [PATCH 6/6] Separate policy documents as managed policies --- modules/karpenter/controller_iam.tf | 27 ++++++++++++++++----------- modules/karpenter/variables.tf | 1 + outputs.tf | 1 + variables.tf | 5 +++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf index e94c555..e510e77 100644 --- a/modules/karpenter/controller_iam.tf +++ b/modules/karpenter/controller_iam.tf @@ -28,16 +28,16 @@ data "aws_iam_policy_document" "karpenter_controller_assume_role_policy" { } } -resource "aws_iam_role_policy" "karpenter_controller_v1_beta" { - count = var.v1beta ? 1 : 0 - name = "KarpenterController-v1beta" - role = aws_iam_role.karpenter_controller.id - policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json +resource "aws_iam_role_policy_attachment" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 + role = aws_iam_role.karpenter_controller.id + policy_arn = aws_iam_policy.karpenter_controller_v1_beta[0].arn } -moved { - from = aws_iam_role_policy.karpenter_controller_v1_beta - to = aws_iam_role_policy.karpenter_controller_v1_beta[0] +resource "aws_iam_policy" "karpenter_controller_v1_beta" { + count = var.v1beta ? 1 : 0 + name = "${var.cluster_config.iam_policy_name_prefix}KarpenterController-v1beta-${var.cluster_config.name}" + policy = data.aws_iam_policy_document.karpenter_controller_v1_beta.json } data "aws_iam_policy_document" "karpenter_controller_v1_beta" { @@ -390,10 +390,15 @@ data "aws_iam_policy_document" "karpenter_controller_v1_beta" { } } -resource "aws_iam_role_policy" "karpenter_controller_v1" { +resource "aws_iam_role_policy_attachment" "karpenter_controller_v1" { + count = var.v1 ? 1 : 0 + role = aws_iam_role.karpenter_controller.id + policy_arn = aws_iam_policy.karpenter_controller_v1[0].arn +} + +resource "aws_iam_policy" "karpenter_controller_v1" { count = var.v1 ? 1 : 0 - name = "KarpenterController-v1" - role = aws_iam_role.karpenter_controller.id + name = "${var.cluster_config.iam_policy_name_prefix}KarpenterController-v1-${var.cluster_config.name}" policy = data.aws_iam_policy_document.karpenter_controller_v1.json } diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index f9c556c..71160bf 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -5,6 +5,7 @@ variable "cluster_config" { arn = string private_subnet_ids = map(string) iam_role_name_prefix = string + iam_policy_name_prefix = string fargate_execution_role_arn = string }) } diff --git a/outputs.tf b/outputs.tf index 5664d4b..053cec0 100644 --- a/outputs.tf +++ b/outputs.tf @@ -9,6 +9,7 @@ locals { node_security_group = aws_eks_cluster.control_plane.vpc_config.0.cluster_security_group_id tags = var.tags iam_role_name_prefix = var.iam_role_name_prefix + iam_policy_name_prefix = var.iam_policy_name_prefix fargate_execution_role_arn = aws_iam_role.fargate.arn } } diff --git a/variables.tf b/variables.tf index 500fbe4..7ba2c6f 100644 --- a/variables.tf +++ b/variables.tf @@ -35,6 +35,11 @@ variable "iam_role_name_prefix" { description = "An optional prefix to any IAM Roles created by this module" } +variable "iam_policy_name_prefix" { + default = "" + description = "An optional prefix to any IAM Policies created by this module" +} + variable "cluster_role_arn" { type = string description = "The ARN of IAM role to be used by the cluster, if not specified a role will be created"