From 43a9ba0b4e64df9b278985c306783f428c73629f Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Thu, 21 Mar 2024 09:41:21 -0700 Subject: [PATCH] docs: add AMI requirement note (#5904) --- .../content/en/docs/concepts/nodeclasses.md | 1 + .../en/preview/concepts/nodeclasses.md | 1 + .../content/en/v0.32/concepts/nodeclasses.md | 21 ++++----- .../content/en/v0.33/concepts/nodeclasses.md | 21 ++++----- .../content/en/v0.34/concepts/nodeclasses.md | 43 ++++++++++--------- .../content/en/v0.35/concepts/nodeclasses.md | 1 + 6 files changed, 47 insertions(+), 41 deletions(-) diff --git a/website/content/en/docs/concepts/nodeclasses.md b/website/content/en/docs/concepts/nodeclasses.md index e95f576cef11..d426f579e7e6 100644 --- a/website/content/en/docs/concepts/nodeclasses.md +++ b/website/content/en/docs/concepts/nodeclasses.md @@ -457,6 +457,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index e95f576cef11..d426f579e7e6 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -457,6 +457,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} diff --git a/website/content/en/v0.32/concepts/nodeclasses.md b/website/content/en/v0.32/concepts/nodeclasses.md index 0e72e039b974..a5fbbd0636cb 100644 --- a/website/content/en/v0.32/concepts/nodeclasses.md +++ b/website/content/en/v0.32/concepts/nodeclasses.md @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,8 +70,8 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -246,7 +246,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -313,8 +313,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -400,8 +400,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -422,6 +422,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} diff --git a/website/content/en/v0.33/concepts/nodeclasses.md b/website/content/en/v0.33/concepts/nodeclasses.md index 0e72e039b974..a5fbbd0636cb 100644 --- a/website/content/en/v0.33/concepts/nodeclasses.md +++ b/website/content/en/v0.33/concepts/nodeclasses.md @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,8 +70,8 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -246,7 +246,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -313,8 +313,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -400,8 +400,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -422,6 +422,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} diff --git a/website/content/en/v0.34/concepts/nodeclasses.md b/website/content/en/v0.34/concepts/nodeclasses.md index bc139771dcb7..bc2cac8545ad 100644 --- a/website/content/en/v0.34/concepts/nodeclasses.md +++ b/website/content/en/v0.34/concepts/nodeclasses.md @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,15 +70,15 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test - name: my-ami - id: ami-123 - + # Optional, use instance-store volumes for node ephemeral-storage instanceStorePolicy: RAID0 @@ -248,7 +248,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -315,8 +315,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -402,8 +402,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -424,6 +424,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} @@ -628,25 +629,25 @@ The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.a If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: -```yaml -spec: +```yaml +spec: instanceStorePolicy: RAID0 ``` -This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). -The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: -#### AL2 +#### AL2 -On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. -#### Others +#### Others -For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. -{{% alert title="Tip" color="secondary" %}} -Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. {{% /alert %}} ## spec.userData diff --git a/website/content/en/v0.35/concepts/nodeclasses.md b/website/content/en/v0.35/concepts/nodeclasses.md index e95f576cef11..d426f579e7e6 100644 --- a/website/content/en/v0.35/concepts/nodeclasses.md +++ b/website/content/en/v0.35/concepts/nodeclasses.md @@ -457,6 +457,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}}