feat: implement v1 AMI selection (#6450)

Co-authored-by: Nick Tran <nichotr@amazon.com>

hack/docs/instancetypes_gen/main.go

@@ -105,7 +105,9 @@ func main() {
 	// Fake a NodeClass so we can use it to get InstanceTypes
 	nodeClass := &v1.EC2NodeClass{
 		Spec: v1.EC2NodeClassSpec{
-			AMIFamily: &v1.AMIFamilyAL2023,
+			AMISelectorTerms: []v1.AMISelectorTerm{{
+				Alias: "al2023@latest",
+			}},
 			SubnetSelectorTerms: []v1.SubnetSelectorTerm{
 				{
 					Tags: map[string]string{

pkg/apis/apis.go

@@ -27,7 +27,7 @@ import (
 //go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds
 var (
 	Group              = "karpenter.k8s.aws"
-	CompatabilityGroup = "compatibility." + Group
+	CompatibilityGroup = "compatibility." + Group
 	//go:embed crds/karpenter.k8s.aws_ec2nodeclasses.yaml
 	EC2NodeClassCRD []byte
 	CRDs            = append(apis.CRDs,

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -57,24 +57,28 @@ spec:
                 EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider.
                 This will contain configuration necessary to launch instances in AWS.
               properties:
-                amiFamily:
-                  description: AMIFamily is the AMI family that instances use.
-                  enum:
-                    - AL2
-                    - AL2023
-                    - Bottlerocket
-                    - Ubuntu
-                    - Custom
-                    - Windows2019
-                    - Windows2022
-                  type: string
                 amiSelectorTerms:
                   description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed.
                   items:
                     description: |-
                       AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes.
                       If multiple fields are used for selection, the requirements are ANDed.
                     properties:
+                      alias:
+                        description: |-
+                          Alias specifies which EKS optimized AMI to select.
+                          Each alias consists of a family and an AMI version, specified as "family@version".
+                          Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022.
+                          The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0").
+                          The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments.
+                          Note: The Windows families do **not** support version pinning, and only latest may be used.
+                        maxLength: 30
+                        type: string
+                        x-kubernetes-validations:
+                          - message: '''alias'' is improperly formatted, must match the format ''family@version'''
+                            rule: self.matches('^[a-zA-Z0-9]*@.*$')
+                          - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022'''
+                            rule: self.find('^[^@]+') in ['al2','al2023','bottlerocket','windows2019','windows2022']
                       id:
                         description: ID is the ami id in EC2
                         pattern: ami-[0-9a-z]+
@@ -102,12 +106,17 @@ spec:
                             rule: self.all(k, k != '' && self[k] != '')
                     type: object
                   maxItems: 30
+                  minItems: 1
                   type: array
                   x-kubernetes-validations:
-                    - message: expected at least one, got none, ['tags', 'id', 'name']
-                      rule: self.all(x, has(x.tags) || has(x.id) || has(x.name))
+                    - message: expected at least one, got none, ['tags', 'id', 'name', 'alias']
+                      rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias))
                     - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms'
-                      rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))'
+                      rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))'
+                    - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms'
+                      rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))'
+                    - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms'
+                      rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)'
                 associatePublicIPAddress:
                   description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass.
                   type: boolean
@@ -549,13 +558,10 @@ spec:
                     this UserData to ensure nodes are being provisioned with the correct configuration.
                   type: string
               required:
-                - amiFamily
                 - securityGroupSelectorTerms
                 - subnetSelectorTerms
               type: object
               x-kubernetes-validations:
-                - message: amiSelectorTerms is required when amiFamily == 'Custom'
-                  rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != 0 : true'
                 - message: must specify exactly one of ['role', 'instanceProfile']
                   rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))
                 - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.

pkg/apis/v1/ec2nodeclass.go

@@ -16,6 +16,8 @@ package v1
 
 import (
 	"fmt"
+	"log"
+	"strings"
 
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/samber/lo"
@@ -46,15 +48,14 @@ type EC2NodeClassSpec struct {
 	// +optional
 	AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"`
 	// AMISelectorTerms is a list of or ami selector terms. The terms are ORed.
-	// +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))"
-	// +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))"
+	// +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias))"
+	// +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))"
+	// +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))"
+	// +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)"
+	// +kubebuilder:validation:MinItems:=1
 	// +kubebuilder:validation:MaxItems:=30
 	// +optional
-	AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"`
-	// AMIFamily is the AMI family that instances use.
-	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
-	// +required
-	AMIFamily *string `json:"amiFamily"`
+	AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -163,6 +164,17 @@ type SecurityGroupSelectorTerm struct {
 // AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes.
 // If multiple fields are used for selection, the requirements are ANDed.
 type AMISelectorTerm struct {
+	// Alias specifies which EKS optimized AMI to select.
+	// Each alias consists of a family and an AMI version, specified as "family@version".
+	// Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022.
+	// The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0").
+	// The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments.
+	// Note: The Windows families do **not** support version pinning, and only latest may be used.
+	// +kubebuilder:validation:XValidation:message="'alias' is improperly formatted, must match the format 'family@version'",rule="self.matches('^[a-zA-Z0-9]*@.*$')"
+	// +kubebuilder:validation:XValidation:message="family is not supported, must be one of the following: 'al2', 'al2023', 'bottlerocket', 'windows2019', 'windows2022'",rule="self.find('^[^@]+') in ['al2','al2023','bottlerocket','windows2019','windows2022']"
+	// +kubebuilder:validation:MaxLength=30
+	// +optional
+	Alias string `json:"alias,omitempty"`
 	// Tags is a map of key/value tags used to select subnets
 	// Specifying '*' for a value selects all values for a given tag key.
 	// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
@@ -405,7 +417,6 @@ type EC2NodeClass struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	// +kubebuilder:validation:XValidation:message="amiSelectorTerms is required when amiFamily == 'Custom'",rule="self.amiFamily == 'Custom' ? self.amiSelectorTerms.size() != 0 : true"
 	// +kubebuilder:validation:XValidation:message="must specify exactly one of ['role', 'instanceProfile']",rule="(has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))"
 	// +kubebuilder:validation:XValidation:message="changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.",rule="(has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile))"
 	Spec   EC2NodeClassSpec   `json:"spec,omitempty"`
@@ -442,6 +453,45 @@ func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]strin
 	})
 }
 
+func (in *EC2NodeClass) AMIFamily() string {
+	if family, ok := in.Annotations[AnnotationAMIFamilyCompatibility]; ok {
+		return family
+	}
+	if term, ok := lo.Find(in.Spec.AMISelectorTerms, func(t AMISelectorTerm) bool {
+		return t.Alias != ""
+	}); ok {
+		switch strings.Split(term.Alias, "@")[0] {
+		case "al2":
+			return AMIFamilyAL2
+		case "al2023":
+			return AMIFamilyAL2023
+		case "bottlerocket":
+			return AMIFamilyBottlerocket
+		case "windows2019":
+			return AMIFamilyWindows2019
+		case "windows2022":
+			return AMIFamilyWindows2022
+		}
+	}
+	return AMIFamilyCustom
+}
+
+func (in *EC2NodeClass) AMIVersion() string {
+	if _, ok := in.Annotations[AnnotationAMIFamilyCompatibility]; ok {
+		return "latest"
+	}
+	if term, ok := lo.Find(in.Spec.AMISelectorTerms, func(t AMISelectorTerm) bool {
+		return t.Alias != ""
+	}); ok {
+		parts := strings.Split(term.Alias, "@")
+		if len(parts) != 2 {
+			log.Fatalf("failed to parse AMI alias %q, invalid format", term.Alias)
+		}
+		return parts[1]
+	}
+	return "latest"
+}
+
 // EC2NodeClassList contains a list of EC2NodeClass
 // +kubebuilder:object:root=true
 type EC2NodeClassList struct {

pkg/apis/v1/ec2nodeclass_conversion.go

@@ -16,6 +16,8 @@ package v1
 
 import (
 	"context"
+	"fmt"
+	"strings"
 
 	"github.com/samber/lo"
 	"knative.dev/pkg/apis"
@@ -27,6 +29,7 @@ func (in *EC2NodeClass) ConvertTo(ctx context.Context, to apis.Convertible) erro
 	v1beta1enc := to.(*v1beta1.EC2NodeClass)
 	v1beta1enc.ObjectMeta = in.ObjectMeta
 
+	v1beta1enc.Spec.AMIFamily = lo.ToPtr(in.AMIFamily())
 	in.Spec.convertTo(&v1beta1enc.Spec)
 	in.Status.convertTo((&v1beta1enc.Status))
 	return nil
@@ -54,7 +57,6 @@ func (in *EC2NodeClassSpec) convertTo(v1beta1enc *v1beta1.EC2NodeClassSpec) {
 			Tags:  ami.Tags,
 		}
 	})
-	v1beta1enc.AMIFamily = in.AMIFamily
 	v1beta1enc.AssociatePublicIPAddress = in.AssociatePublicIPAddress
 	v1beta1enc.Context = in.Context
 	v1beta1enc.DetailedMonitoring = in.DetailedMonitoring
@@ -102,6 +104,19 @@ func (in *EC2NodeClass) ConvertFrom(ctx context.Context, from apis.Convertible)
 	v1beta1enc := from.(*v1beta1.EC2NodeClass)
 	in.ObjectMeta = v1beta1enc.ObjectMeta
 
+	// If the v1beta1 AMI family is supported on v1, construct an alias. Otherwise, use the compatibility annotation.
+	// In practice, this is only used to support the Ubuntu AMI family during conversion.
+	switch lo.FromPtr(v1beta1enc.Spec.AMIFamily) {
+	case AMIFamilyAL2, AMIFamilyAL2023, AMIFamilyBottlerocket, Windows2019, Windows2022:
+		in.Spec.AMISelectorTerms = []AMISelectorTerm{{
+			Alias: fmt.Sprintf("%s@latest", strings.ToLower(lo.FromPtr(v1beta1enc.Spec.AMIFamily))),
+		}}
+	default:
+		in.Annotations = lo.Assign(in.Annotations, map[string]string{
+			AnnotationAMIFamilyCompatibility: lo.FromPtr(v1beta1enc.Spec.AMIFamily),
+		})
+	}
+
 	in.Spec.convertFrom(&v1beta1enc.Spec)
 	in.Status.convertFrom((&v1beta1enc.Status))
 	return nil
@@ -121,15 +136,14 @@ func (in *EC2NodeClassSpec) convertFrom(v1beta1enc *v1beta1.EC2NodeClassSpec) {
 			Tags: sg.Tags,
 		}
 	})
-	in.AMISelectorTerms = lo.Map(v1beta1enc.AMISelectorTerms, func(ami v1beta1.AMISelectorTerm, _ int) AMISelectorTerm {
+	in.AMISelectorTerms = append(in.AMISelectorTerms, lo.Map(v1beta1enc.AMISelectorTerms, func(ami v1beta1.AMISelectorTerm, _ int) AMISelectorTerm {
 		return AMISelectorTerm{
 			ID:    ami.ID,
 			Name:  ami.Name,
 			Owner: ami.Owner,
 			Tags:  ami.Tags,
 		}
-	})
-	in.AMIFamily = v1beta1enc.AMIFamily
+	})...)
 	in.AssociatePublicIPAddress = v1beta1enc.AssociatePublicIPAddress
 	in.Context = v1beta1enc.Context
 	in.DetailedMonitoring = v1beta1enc.DetailedMonitoring

pkg/apis/v1/ec2nodeclass_conversion_test.go

@@ -22,6 +22,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	"sigs.k8s.io/karpenter/pkg/test"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	. "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 )
@@ -38,8 +40,10 @@ var _ = Describe("Convert v1 to v1beta1 EC2NodeClass API", func() {
 	})
 
 	It("should convert v1 ec2nodeclass metadata", func() {
-		v1ec2nodeclass.ObjectMeta = test.ObjectMeta()
-		Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed())
+		v1ec2nodeclass.ObjectMeta = test.ObjectMeta(metav1.ObjectMeta{
+			Annotations: map[string]string{"foo": "bar"},
+		})
+		Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed())
 		Expect(v1beta1ec2nodeclass.ObjectMeta).To(BeEquivalentTo(v1ec2nodeclass.ObjectMeta))
 	})
 	Context("EC2NodeClass Spec", func() {
@@ -108,10 +112,17 @@ var _ = Describe("Convert v1 to v1beta1 EC2NodeClass API", func() {
 			Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed())
 			Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AssociatePublicIPAddress)).To(BeTrue())
 		})
-		It("should convert v1 ec2nodeclass ami family", func() {
-			v1ec2nodeclass.Spec.AMIFamily = &AMIFamilyUbuntu
+		It("should convert v1 ec2nodeclass alias", func() {
+			v1ec2nodeclass.Spec.AMISelectorTerms = []AMISelectorTerm{{Alias: "al2023@latest"}}
+			Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed())
+			Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyAL2023))
+		})
+		It("should convert v1 ec2nodeclass with AMIFamily compat annotation", func() {
+			v1ec2nodeclass.Annotations = lo.Assign(v1ec2nodeclass.Annotations, map[string]string{
+				AnnotationAMIFamilyCompatibility: v1beta1.AMIFamilyAL2023,
+			})
 			Expect(v1ec2nodeclass.ConvertTo(ctx, v1beta1ec2nodeclass)).To(Succeed())
-			Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyUbuntu))
+			Expect(lo.FromPtr(v1beta1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyAL2023))
 		})
 		It("should convert v1 ec2nodeclass user data", func() {
 			v1ec2nodeclass.Spec.UserData = lo.ToPtr("test user data")
@@ -269,8 +280,18 @@ var _ = Describe("Convert v1beta1 to v1 EC2NodeClass API", func() {
 	})
 
 	It("should convert v1beta1 ec2nodeclass metadata", func() {
-		v1beta1ec2nodeclass.ObjectMeta = test.ObjectMeta()
+		v1beta1ec2nodeclass.ObjectMeta = test.ObjectMeta(metav1.ObjectMeta{
+			Annotations: map[string]string{"foo": "bar"},
+		})
 		Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed())
+
+		// Remove the compatibility annotations from the EC2NodeClass
+		v1ec2nodeclass.ObjectMeta.Annotations = lo.OmitByKeys(v1ec2nodeclass.ObjectMeta.Annotations, []string{
+			AnnotationAMIFamilyCompatibility,
+		})
+		if len(v1ec2nodeclass.ObjectMeta.Annotations) == 0 {
+			v1ec2nodeclass.ObjectMeta.Annotations = nil
+		}
 		Expect(v1ec2nodeclass.ObjectMeta).To(BeEquivalentTo(v1beta1ec2nodeclass.ObjectMeta))
 	})
 	Context("EC2NodeClass Spec", func() {
@@ -340,9 +361,14 @@ var _ = Describe("Convert v1beta1 to v1 EC2NodeClass API", func() {
 			Expect(lo.FromPtr(v1ec2nodeclass.Spec.AssociatePublicIPAddress)).To(BeTrue())
 		})
 		It("should convert v1beta1 ec2nodeclass ami family", func() {
-			v1beta1ec2nodeclass.Spec.AMIFamily = &AMIFamilyUbuntu
+			v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023
+			Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed())
+			Expect(v1ec2nodeclass.Spec.AMISelectorTerms).To(ContainElement(AMISelectorTerm{Alias: "al2023@latest"}))
+		})
+		It("should convert v1beta1 ec2nodeclass ami family (ubuntu compat)", func() {
+			v1beta1ec2nodeclass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu
 			Expect(v1ec2nodeclass.ConvertFrom(ctx, v1beta1ec2nodeclass)).To(Succeed())
-			Expect(lo.FromPtr(v1ec2nodeclass.Spec.AMIFamily)).To(Equal(v1beta1.AMIFamilyUbuntu))
+			Expect(v1ec2nodeclass.Annotations).To(HaveKeyWithValue(AnnotationAMIFamilyCompatibility, "Ubuntu"))
 		})
 		It("should convert v1beta1 ec2nodeclass user data", func() {
 			v1beta1ec2nodeclass.Spec.UserData = lo.ToPtr("test user data")