[EKS Prow Cluster] Add karpenter terraform module for eks-prow-build cluster

infra/aws/terraform/modules/eks-prow-iam/policy_apply.tf

@@ -60,6 +60,7 @@ data "aws_iam_policy_document" "eks_apply" {
       "ec2:ModifyInstanceAttribute",
       "ec2:TerminateInstances",
       "ec2:ImportKeyPair",
+      "eks:CreateAccessEntry",
       "eks:CreateAddon",
       "eks:CreateCluster",
       "eks:CreateNodegroup",
@@ -69,9 +70,19 @@ data "aws_iam_policy_document" "eks_apply" {
       "eks:UpdateClusterVersion",
       "eks:UpdateNodegroupConfig",
       "eks:UpdateNodegroupVersion",
+      "events:DeleteRule",
+      "events:DescribeRule",
+      "events:ListTagsForResource",
+      "events:ListTargetsByRule",
+      "events:PutRule",
+      "events:PutTargets",
+      "events:RemoveTargets",
+      "events:TagResource",
+      "iam:AttachRolePolicy",
       "iam:CreateOpenIDConnectProvider",
       "iam:CreatePolicy",
       "iam:CreatePolicyVersion",
+      "iam:CreateRole",
       "iam:PassRole",
       "iam:TagOpenIDConnectProvider",
       "iam:TagPolicy",
@@ -90,6 +101,12 @@ data "aws_iam_policy_document" "eks_apply" {
       "logs:PutRetentionPolicy",
       "logs:TagLogGroup",
       "s3:PutObject",
+      "sqs:createqueue",
+      "sqs:deletequeue",
+      "sqs:getqueueattributes",
+      "sqs:listqueuetags",
+      "sqs:setqueueattributes",
+      "sqs:tagqueue",
       # TODO(xmudrii-ubuntu): remove after removing ECR repo
       "ecr-public:*"
     ]

infra/aws/terraform/modules/eks-prow-iam/policy_plan.tf

@@ -31,6 +31,7 @@ data "aws_iam_policy_document" "eks_plan" {
       "acm:DescribeCertificate",
       "acm:ListTagsForCertificate",
       "ec2:DescribeAddresses",
+      "ec2:DescribeAddressesAttribute",
       "ec2:DescribeAvailabilityZones",
       "ec2:DescribeEgressOnlyInternetGateways",
       "ec2:DescribeInternetGateways",
@@ -84,6 +85,7 @@ data "aws_iam_policy_document" "eks_plan" {
       "kms:ListAliases",
       "kms:ListResourceTags",
       "logs:DescribeLogGroups",
+      "logs:ListTagsForResource",
       "logs:ListTagsLogGroup",
       "s3:GetObject",
       "s3:ListBucket"

infra/aws/terraform/prow-build-cluster/hack/flux-update.bash

@@ -91,6 +91,23 @@ flux create hr kubecost \
     --interval=${sync_interval} \
     --export >> ${resources_dir}/kubecost/flux-hr-kubecost.yaml
 
+boilerplate > ${resources_dir}/flux-system/flux-source-helm-karpenter-chart.yaml
+flux create source helm karpenter \
+    --url oci://public.ecr.aws/karpenter/karpenter \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/flux-system/flux-source-helm-karpenter-chart.yaml
+
+boilerplate > ${resources_dir}/karpenter/flux-hr-karpenter.yaml
+flux create hr karpenter \
+    --source HelmRepository/karpenter.flux-system \
+    --namespace=karpenter \
+    --chart karpenter \
+    --chart-version 0.36.2 \
+    --values ${resources_dir}/karpenter/${PROW_ENV}-cluster-values \
+    --interval=${sync_interval} \
+    --export >> ${resources_dir}/karpenter/flux-hr-karpenter.yaml
+
+
 # This list contains names of folders inside ./resources directory
 # that are used for generating FluxCD kustomizations.
 kustomizations=(
@@ -104,6 +121,7 @@ kustomizations=(
     external-secrets
     kubecost
     cluster-autoscaler
+    karpenter
 )
 
 # Code below is used to figure out a relative path of

infra/aws/terraform/prow-build-cluster/karpenter.tf

@@ -0,0 +1,36 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+module "karpenter" {
+  source = "terraform-aws-modules/eks/aws//modules/karpenter"
+
+  cluster_name = module.eks.cluster_name
+  create_access_entry = false
+
+  enable_irsa = true
+  irsa_namespace_service_accounts = ["karpenter:karpenter"]
+  irsa_oidc_provider_arn = module.eks.oidc_provider_arn
+
+  # Attach additional IAM policies to the Karpenter node IAM role
+  node_iam_role_additional_policies = {
+    AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+  }
+
+  tags = {
+    Environment = var.cluster_name == "prow-canary-cluster" ? "canary" : "prod"
+    Terraform   = "true"
+  }
+}

infra/aws/terraform/prow-build-cluster/locals.tf

@@ -37,6 +37,7 @@ locals {
   auto_scaling_tags = {
     "k8s.io/cluster-autoscaler/${var.cluster_name}" = "owned"
     "k8s.io/cluster-autoscaler/enabled"             = true
+    "karpenter.sh/discovery"                        = var.cluster_name
   }
 
   azs = slice(data.aws_availability_zones.available.names, 0, 3)
@@ -81,6 +82,27 @@ locals {
     }
   ]
 
+  karpenter_roles = [
+    {
+      rolearn  = module.karpenter.node_iam_role_arn
+      username = "system:node:{{EC2PrivateDNSName}}"
+      groups = [
+        "system:bootstrappers",
+        "system:nodes"
+      ]
+    }
+  ]
+
+  sso_roles = [
+    {
+      rolearn  = "arn:aws:iam::468814281478:role/AWSReservedSSO_AdministratorAccess_abaef4db15a2c055"
+      username = "sso-admins"
+      groups   = [
+        "eks-cluster-admin"
+      ]
+    }
+  ]
+
   aws_auth_roles = flatten([
     local.configure_prow ? [
       # Allow access to the Prow-EKS-Admin IAM role (used by Prow directly).
@@ -93,6 +115,8 @@ locals {
       }
     ] : [],
     local.cluster_admin_roles,
-    local.cluster_viewer_roles
+    local.cluster_viewer_roles,
+    local.karpenter_roles,
+    local.sso_roles
   ])
 }

infra/aws/terraform/prow-build-cluster/resources/flux-system/flux-ks-karpenter.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: karpenter
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  path: ./infra/aws/terraform/prow-build-cluster/resources/karpenter
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-io
+    namespace: flux-system

infra/aws/terraform/prow-build-cluster/resources/karpenter/canary-cluster-values

@@ -0,0 +1,23 @@
+settings:
+  clusterName: prow-canary-cluster
+  interruptionQueue: Karpenter-prow-canary-cluster
+serviceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: "arn:aws:iam::054318140392:role/KarpenterController-20240527081538529900000002"
+
+controller:
+  resources:
+    requests:
+      cpu: 1
+      memory: 1Gi
+    limits:
+      cpu: 1
+      memory: 1Gi
+
+# default affinity rule works well in our case
+# https://github.com/aws/karpenter-provider-aws/blob/main/charts/karpenter/values.yaml#L70
+tolerations:
+  - effect: NoSchedule
+    key: node-group
+    operator: Equal
+    value: stable

infra/aws/terraform/prow-build-cluster/resources/karpenter/flux-hr-karpenter.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: karpenter
+  namespace: flux-system
+spec:
+  chart:
+    spec:
+      chart: karpenter
+      reconcileStrategy: ChartVersion
+      sourceRef:
+        kind: HelmRepository
+        name: karpenter
+        namespace: flux-system
+      version: 0.36.2
+  install:
+    createNamespace: true
+  interval: 1m0s
+  releaseName: karpenter
+  targetNamespace: karpenter
+  values:
+    controller:
+      resources:
+        limits:
+          cpu: 1
+          memory: 1Gi
+        requests:
+          cpu: 1
+          memory: 1Gi
+    serviceAccount:
+      annotations:
+        # terraform state show module.karpenter.aws_iam_role.controller\[0\] | grep " arn "
+        eks.amazonaws.com/role-arn: arn:aws:iam::468814281478:role/KarpenterController-20240527081538529900000002
+    settings:
+      clusterName: prow-build-cluster
+      interruptionQueue: Karpenter-prow-build-cluster
+    tolerations:
+    - effect: NoSchedule
+      key: node-group
+      operator: Equal
+      value: stable

infra/aws/terraform/prow-build-cluster/resources/karpenter/prod-cluster-values

@@ -0,0 +1,23 @@
+settings:
+  clusterName: prow-build-cluster
+  interruptionQueue: Karpenter-prow-build-cluster
+serviceAccount:
+  annotations:
+    eks.amazonaws.com/role-arn: "arn:aws:iam::468814281478:role/KarpenterController-20240527081538529900000002"
+
+controller:
+  resources:
+    requests:
+      cpu: 1
+      memory: 1Gi
+    limits:
+      cpu: 1
+      memory: 1Gi
+
+# default affinity rule works well in our case
+# https://github.com/aws/karpenter-provider-aws/blob/main/charts/karpenter/values.yaml#L70
+tolerations:
+  - effect: NoSchedule
+    key: node-group
+    operator: Equal
+    value: stable

infra/aws/terraform/prow-build-cluster/vpc.tf

@@ -24,7 +24,6 @@ limitations under the License.
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "~> 5.1"
-
   name = "${var.cluster_name}-vpc"
 
   cidr                  = var.vpc_cidr
@@ -65,6 +64,7 @@ module "vpc" {
   public_subnet_tags = {
     "kubernetes.io/role/elb"                    = 1
     "kubernetes.io/cluster/${var.cluster_name}" = "owned"
+    "karpenter.sh/discovery"                    = var.cluster_name
   }
 
   private_subnet_tags = {