diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 973bced534..473b1b49c6 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -18,8 +18,11 @@ Once the cluster is up and running, you can check that Karpenter is functioning # First, make sure you have updated your local kubeconfig aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter -# Second, scale the example deployment -kubectl scale deployment inflate --replicas 5 +# Second, deploy the Karpenter NodeClass/NodePool +kubectl apply -f karpenter.yaml + +# Second, deploy the example deployment +kubectl apply -f inflate.yaml # You can watch Karpenter's controller logs with kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller @@ -32,10 +35,10 @@ kubectl get nodes -L karpenter.sh/registered ``` ```text -NAME STATUS ROLES AGE VERSION REGISTERED -ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true -ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a -ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a +NAME STATUS ROLES AGE VERSION REGISTERED +ip-10-0-13-51.eu-west-1.compute.internal Ready 29s v1.31.1-eks-1b3e656 true +ip-10-0-41-242.eu-west-1.compute.internal Ready 35m v1.31.1-eks-1b3e656 +ip-10-0-8-151.eu-west-1.compute.internal Ready 35m v1.31.1-eks-1b3e656 ``` ```sh @@ -44,24 +47,27 @@ kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName ```text NAME NODE -inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal -inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal -aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal -aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal -aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal -coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal -coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal -eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal -eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal -eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal -karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal -karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal -kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal -kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal -kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal +inflate-67cd5bb766-hvqfn ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-jnsdp ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-k4gwf ip-10-0-41-242.eu-west-1.compute.internal +inflate-67cd5bb766-m49f6 ip-10-0-13-51.eu-west-1.compute.internal +inflate-67cd5bb766-pgzx9 ip-10-0-8-151.eu-west-1.compute.internal +aws-node-58m4v ip-10-0-3-57.eu-west-1.compute.internal +aws-node-pj2gc ip-10-0-8-151.eu-west-1.compute.internal +aws-node-thffj ip-10-0-41-242.eu-west-1.compute.internal +aws-node-vh66d ip-10-0-13-51.eu-west-1.compute.internal +coredns-844dbb9f6f-9g9lg ip-10-0-41-242.eu-west-1.compute.internal +coredns-844dbb9f6f-fmzfq ip-10-0-41-242.eu-west-1.compute.internal +eks-pod-identity-agent-jr2ns ip-10-0-8-151.eu-west-1.compute.internal +eks-pod-identity-agent-mpjkq ip-10-0-13-51.eu-west-1.compute.internal +eks-pod-identity-agent-q4tjc ip-10-0-3-57.eu-west-1.compute.internal +eks-pod-identity-agent-zzfdj ip-10-0-41-242.eu-west-1.compute.internal +karpenter-5b8965dc9b-rx9bx ip-10-0-8-151.eu-west-1.compute.internal +karpenter-5b8965dc9b-xrfnx ip-10-0-41-242.eu-west-1.compute.internal +kube-proxy-2xf42 ip-10-0-41-242.eu-west-1.compute.internal +kube-proxy-kbfc8 ip-10-0-8-151.eu-west-1.compute.internal +kube-proxy-kt8zn ip-10-0-13-51.eu-west-1.compute.internal +kube-proxy-sl6bz ip-10-0-3-57.eu-west-1.compute.internal ``` ### Tear Down & Clean-Up @@ -72,7 +78,6 @@ Because Karpenter manages the state of node resources outside of Terraform, Karp ```bash kubectl delete deployment inflate -kubectl delete node -l karpenter.sh/provisioner-name=default ``` 2. Remove the resources created by Terraform @@ -91,7 +96,6 @@ Note that this example may create resources which cost money. Run `terraform des | [terraform](#requirement\_terraform) | >= 1.3.2 | | [aws](#requirement\_aws) | >= 5.81 | | [helm](#requirement\_helm) | >= 2.7 | -| [kubectl](#requirement\_kubectl) | >= 2.0 | ## Providers @@ -100,7 +104,6 @@ Note that this example may create resources which cost money. Run `terraform des | [aws](#provider\_aws) | >= 5.81 | | [aws.virginia](#provider\_aws.virginia) | >= 5.81 | | [helm](#provider\_helm) | >= 2.7 | -| [kubectl](#provider\_kubectl) | >= 2.0 | ## Modules @@ -116,9 +119,6 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/alekc/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | diff --git a/examples/karpenter/inflate.yaml b/examples/karpenter/inflate.yaml new file mode 100644 index 0000000000..b70cfd8509 --- /dev/null +++ b/examples/karpenter/inflate.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 5 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 diff --git a/examples/karpenter/karpenter.yaml b/examples/karpenter/karpenter.yaml new file mode 100644 index 0000000000..3d3107c06f --- /dev/null +++ b/examples/karpenter/karpenter.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiSelectorTerms: + - alias: bottlerocket@latest + role: ex-karpenter + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + tags: + karpenter.sh/discovery: ex-karpenter +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index a7aee5ba7a..9961063c95 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -21,20 +21,6 @@ provider "helm" { } } -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - data "aws_availability_zones" "available" { # Exclude local zones filter { @@ -89,21 +75,20 @@ module "eks" { eks_managed_node_groups = { karpenter = { - ami_type = "AL2023_x86_64_STANDARD" + ami_type = "BOTTLEROCKET_x86_64" instance_types = ["m5.large"] min_size = 2 max_size = 3 desired_size = 2 + + labels = { + # Used to ensure Karpenter runs on nodes that it does not manage + "karpenter.sh/controller" = "true" + } } } - # cluster_tags = merge(local.tags, { - # NOTE - only use this option if you are using "attach_cluster_primary_security_group" - # and you know what you're doing. In this case, you can remove the "node_security_group_tags" below. - # "karpenter.sh/discovery" = local.name - # }) - node_security_group_tags = merge(local.tags, { # NOTE - if creating multiple security groups with this module, only tag the # security group that Karpenter should utilize with the following tag @@ -121,11 +106,12 @@ module "eks" { module "karpenter" { source = "../../modules/karpenter" - cluster_name = module.eks.cluster_name - + cluster_name = module.eks.cluster_name enable_v1_permissions = true - enable_pod_identity = true + # Name needs to match role name passed to the EC2NodeClass + node_iam_role_use_name_prefix = false + node_iam_role_name = local.name create_pod_identity_association = true # Used to attach additional IAM policies to the Karpenter node IAM role @@ -154,11 +140,13 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "1.1.0" + version = "1.1.1" wait = false values = [ <<-EOT + nodeSelector: + karpenter.sh/controller: 'true' dnsPolicy: Default settings: clusterName: ${module.eks.cluster_name} @@ -170,98 +158,6 @@ resource "helm_release" "karpenter" { ] } -resource "kubectl_manifest" "karpenter_node_class" { - yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1beta1 - kind: EC2NodeClass - metadata: - name: default - spec: - amiFamily: AL2023 - role: ${module.karpenter.node_iam_role_name} - subnetSelectorTerms: - - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelectorTerms: - - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - YAML - - depends_on = [ - helm_release.karpenter - ] -} - -resource "kubectl_manifest" "karpenter_node_pool" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1beta1 - kind: NodePool - metadata: - name: default - spec: - template: - spec: - nodeClassRef: - name: default - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m", "r"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["4", "8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "karpenter.k8s.aws/instance-generation" - operator: Gt - values: ["5"] - limits: - cpu: 1000 - disruption: - consolidationPolicy: WhenEmpty - consolidateAfter: 30s - YAML - - depends_on = [ - kubectl_manifest.karpenter_node_class - ] -} - -# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) -# and starts with zero replicas -resource "kubectl_manifest" "karpenter_example_deployment" { - yaml_body = <<-YAML - apiVersion: apps/v1 - kind: Deployment - metadata: - name: inflate - spec: - replicas: 0 - selector: - matchLabels: - app: inflate - template: - metadata: - labels: - app: inflate - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 - YAML - - depends_on = [ - helm_release.karpenter - ] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/examples/karpenter/versions.tf b/examples/karpenter/versions.tf index 6c8b905a55..08067c5339 100644 --- a/examples/karpenter/versions.tf +++ b/examples/karpenter/versions.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/helm" version = ">= 2.7" } - kubectl = { - source = "alekc/kubectl" - version = ">= 2.0" - } } }