Update karpenter IAM policy (#349)

This updates the karpenter controller IAM policy inline with the upstream documentation/cloudformation https://github.com/aws/karpenter/blob/27ad171092b9d448f7ed8f0f6fc9754419d3c12d/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml#L37-L212 The advantage of this policy is that more resources are scoped, so the chance of unxpected opperation impacting other resources e.g. (from another cluster) is reduced.
cookpad · Oct 10, 2023 · 9158e51 · 9158e51
1 parent f4dcdda
commit 9158e51
Show file tree

Hide file tree

Showing 3 changed files with 200 additions and 18 deletions.
diff --git a/modules/karpenter/controller_iam.tf b/modules/karpenter/controller_iam.tf
@@ -36,15 +36,166 @@ resource "aws_iam_role_policy" "karpenter_controller" {
 
 data "aws_iam_policy_document" "karpenter_controller" {
   statement {
+    sid    = "AllowScopedEC2InstanceActions"
+    effect = "Allow"
+
+    # tfsec:ignore:aws-iam-no-policy-wildcards
+    resources = [
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::image/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}::snapshot/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:spot-instances-request/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:security-group/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:subnet/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*",
+    ]
+
     actions = [
-      # Write Operations
+      "ec2:RunInstances",
       "ec2:CreateFleet",
-      "ec2:CreateLaunchTemplate",
-      "ec2:CreateTags",
-      "ec2:DeleteLaunchTemplate",
+    ]
+  }
+
+  statement {
+    sid    = "AllowScopedEC2InstanceActionsWithTags"
+    effect = "Allow"
+
+    # tfsec:ignore:aws-iam-no-policy-wildcards
+    resources = [
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*",
+    ]
+
+    actions = [
       "ec2:RunInstances",
+      "ec2:CreateFleet",
+      "ec2:CreateLaunchTemplate",
+    ]
+
+    condition {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/karpenter.sh/provisioner-name"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    sid    = "AllowScopedResourceCreationTagging"
+    effect = "Allow"
+
+    # tfsec:ignore:aws-iam-no-policy-wildcards
+    resources = [
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:fleet/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:volume/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:network-interface/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*",
+    ]
+
+    actions = ["ec2:CreateTags"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/kubernetes.io/cluster/${var.cluster_config.name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "ec2:CreateAction"
+
+      values = [
+        "RunInstances",
+        "CreateFleet",
+        "CreateLaunchTemplate",
+      ]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/karpenter.sh/provisioner-name"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    sid    = "AllowMachineMigrationTagging"
+    effect = "Allow"
+    # tfsec:ignore:aws-iam-no-policy-wildcards
+    resources = ["arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*"]
+    actions   = ["ec2:CreateTags"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "aws:RequestTag/karpenter.sh/managed-by"
+      values   = [var.cluster_config.name]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:RequestTag/karpenter.sh/provisioner-name"
+      values   = ["*"]
+    }
+
+    condition {
+      test     = "ForAllValues:StringEquals"
+      variable = "aws:TagKeys"
+
+      values = [
+        "karpenter.sh/provisioner-name",
+        "karpenter.sh/managed-by",
+      ]
+    }
+  }
+
+  statement {
+    sid    = "AllowScopedDeletion"
+    effect = "Allow"
+
+    # tfsec:ignore:aws-iam-no-policy-wildcards
+    resources = [
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:instance/*",
+      "arn:${data.aws_partition.current.partition}:ec2:${data.aws_region.current.name}:*:launch-template/*",
+    ]
+
+    actions = [
       "ec2:TerminateInstances",
-      # Read Operations
+      "ec2:DeleteLaunchTemplate",
+    ]
+
+    condition {
+      test     = "StringEquals"
+      variable = "aws:ResourceTag/kubernetes.io/cluster/${var.cluster_config.name}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "aws:ResourceTag/karpenter.sh/provisioner-name"
+      values   = ["*"]
+    }
+  }
+
+  statement {
+    sid       = "AllowRegionalReadActions"
+    effect    = "Allow"
+    resources = ["*"]
+
+    actions = [
       "ec2:DescribeAvailabilityZones",
       "ec2:DescribeImages",
       "ec2:DescribeInstances",
@@ -54,32 +205,59 @@ data "aws_iam_policy_document" "karpenter_controller" {
       "ec2:DescribeSecurityGroups",
       "ec2:DescribeSpotPriceHistory",
       "ec2:DescribeSubnets",
-      "pricing:GetProducts",
-      "ssm:GetParameter",
     ]
 
-    # tfsec:ignore:aws-iam-no-policy-wildcards
-    resources = ["*"]
+    condition {
+      test     = "StringEquals"
+      variable = "aws:RequestedRegion"
+      values   = [data.aws_region.current.name]
+    }
   }
 
   statement {
-    actions   = ["iam:PassRole"]
-    resources = [aws_iam_role.karpenter_node.arn]
+    sid       = "AllowSSMReadActions"
+    effect    = "Allow"
+    resources = ["arn:${data.aws_partition.current.partition}:ssm:${data.aws_region.current.name}::parameter/aws/service/*"]
+    actions   = ["ssm:GetParameter"]
   }
 
   statement {
-    actions   = ["eks:DescribeCluster"]
-    resources = [var.cluster_config.arn]
+    sid       = "AllowPricingReadActions"
+    effect    = "Allow"
+    resources = ["*"]
+    actions   = ["pricing:GetProducts"]
   }
 
   statement {
+    sid       = "AllowInterruptionQueueActions"
+    effect    = "Allow"
+    resources = [aws_sqs_queue.karpenter_interruption.arn]
+
     actions = [
       "sqs:DeleteMessage",
-      "sqs:GetQueueUrl",
       "sqs:GetQueueAttributes",
+      "sqs:GetQueueUrl",
       "sqs:ReceiveMessage",
     ]
+  }
 
-    resources = [aws_sqs_queue.karpenter_interruption.arn]
+  statement {
+    sid       = "AllowPassingInstanceRole"
+    effect    = "Allow"
+    resources = [aws_iam_role.karpenter_node.arn]
+    actions   = ["iam:PassRole"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "iam:PassedToService"
+      values   = ["ec2.amazonaws.com"]
+    }
+  }
+
+  statement {
+    sid       = "AllowAPIServerEndpointDiscovery"
+    effect    = "Allow"
+    resources = [var.cluster_config.arn]
+    actions   = ["eks:DescribeCluster"]
   }
 }
diff --git a/modules/karpenter/data.tf b/modules/karpenter/data.tf
@@ -1,2 +1,3 @@
+data "aws_caller_identity" "current" {}
 data "aws_partition" "current" {}
-
+data "aws_region" "current" {}
diff --git a/modules/karpenter/interruption_queue.tf b/modules/karpenter/interruption_queue.tf
@@ -15,8 +15,11 @@ data "aws_iam_policy_document" "karpenter_interruption_queue_policy" {
     actions   = ["sqs:SendMessage"]
     resources = [aws_sqs_queue.karpenter_interruption.arn]
     principals {
-      type        = "Service"
-      identifiers = ["events.${data.aws_partition.current.dns_suffix}"]
+      type = "Service"
+      identifiers = [
+        "events.amazonaws.com",
+        "sqs.amazonaws.com",
+      ]
     }
   }
 }