Skip to content

Commit

Permalink
feat: reimplement amiFamily (#6569)
Browse files Browse the repository at this point in the history 
Co-authored-by: Jonathan Innis <joinnis@amazon.com>
Co-authored-by: Nick Tran <nichotr@amazon.com>
  • Loading branch information
@jmdeal @jonathan-innis @njtran
3 people authored Jul 30, 2024
1 parent fc46de2 commit 094b57c
Show file tree
Hide file tree
Showing 28 changed files with 515 additions and 231 deletions.
29 changes: 29 additions & 0 deletions pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,22 @@ spec:
EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider.
This will contain configuration necessary to launch instances in AWS.
properties:
amiFamily:
description: |-
AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates.
This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias'
family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no
alias is specified, this field is required.
NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using
the AMIFamily() helper function
enum:
- AL2
- AL2023
- Bottlerocket
- Custom
- Windows2019
- Windows2022
type: string
amiSelectorTerms:
description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed.
items:
Expand Down Expand Up @@ -558,6 +574,7 @@ spec:
this UserData to ensure nodes are being provisioned with the correct configuration.
type: string
required:
- amiSelectorTerms
- securityGroupSelectorTerms
- subnetSelectorTerms
type: object
Expand All @@ -566,6 +583,18 @@ spec:
rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))
- message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.
rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile))
- message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias
rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)'
- message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias
rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)'
- message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias
rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)'
- message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias
rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)'
- message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias
rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)'
- message: must specify amiFamily if amiSelectorTerms does not contain an alias
rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)'
status:
description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass
properties:
Expand Down
84 changes: 56 additions & 28 deletions pkg/apis/v1/ec2nodeclass.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,17 @@ type EC2NodeClassSpec struct {
// +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)"
// +kubebuilder:validation:MinItems:=1
// +kubebuilder:validation:MaxItems:=30
// +optional
// +required
AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms" hash:"ignore"`
// AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates.
// This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias'
// family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no
// alias is specified, this field is required.
// NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using
// the AMIFamily() helper function
// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022}
// +optional
AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"`
// UserData to be applied to the provisioned nodes.
// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
// this UserData to ensure nodes are being provisioned with the correct configuration.
Expand Down Expand Up @@ -419,6 +428,12 @@ type EC2NodeClass struct {

// +kubebuilder:validation:XValidation:message="must specify exactly one of ['role', 'instanceProfile']",rule="(has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile))"
// +kubebuilder:validation:XValidation:message="changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this.",rule="(has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile))"
// +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'al2') ? (self.amiFamily == 'Custom' || self.amiFamily == 'AL2') : true)"
// +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'al2023') ? (self.amiFamily == 'Custom' || self.amiFamily == 'AL2023') : true)"
// +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'bottlerocket') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Bottlerocket') : true)"
// +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'windows2019') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Windows2019') : true)"
// +kubebuilder:validation:XValidation:message="if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias",rule="!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find('^[^@]+') == 'windows2022') ? (self.amiFamily == 'Custom' || self.amiFamily == 'Windows2022') : true)"
// +kubebuilder:validation:XValidation:message="must specify amiFamily if amiSelectorTerms does not contain an alias",rule="self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)"
Spec EC2NodeClassSpec `json:"spec,omitempty"`
Status EC2NodeClassStatus `json:"status,omitempty"`
}
Expand All @@ -430,7 +445,13 @@ type EC2NodeClass struct {
const EC2NodeClassHashVersion = "v3"

func (in *EC2NodeClass) Hash() string {
return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{
return fmt.Sprint(lo.Must(hashstructure.Hash([]interface{}{
in.Spec,
// AMIFamily should be hashed using the dynamically resolved value rather than the literal value of the field.
// This ensures that scenarios such as changing the field from nil to AL2023 with the alias "al2023@latest"
// doesn't trigger drift.
in.AMIFamily(),
}, hashstructure.FormatV2, &hashstructure.HashOptions{
SlicesAsSets: true,
IgnoreZeroValue: true,
ZeroNil: true,
Expand All @@ -453,43 +474,50 @@ func (in *EC2NodeClass) InstanceProfileTags(clusterName string) map[string]strin
})
}

// AMIFamily returns the family for a NodePool based on the following items, in order of precdence:
// - ec2nodeclass.spec.amiFamily
// - ec2nodeclass.spec.amiSelectorTerms[].alias
//
// If an alias is specified, ec2nodeclass.spec.amiFamily must match that alias, or be 'Custom' (enforced via validation).
func (in *EC2NodeClass) AMIFamily() string {
if family, ok := in.Annotations[AnnotationAMIFamilyCompatibility]; ok {
return family
if in.Spec.AMIFamily != nil {
return *in.Spec.AMIFamily
}
if term, ok := lo.Find(in.Spec.AMISelectorTerms, func(t AMISelectorTerm) bool {
return t.Alias != ""
}); ok {
switch strings.Split(term.Alias, "@")[0] {
case "al2":
return AMIFamilyAL2
case "al2023":
return AMIFamilyAL2023
case "bottlerocket":
return AMIFamilyBottlerocket
case "windows2019":
return AMIFamilyWindows2019
case "windows2022":
return AMIFamilyWindows2022
}
return AMIFamilyFromAlias(term.Alias)
}
// Unreachable: validation enforces that one of the above conditions must be met
return AMIFamilyCustom
}

func (in *EC2NodeClass) AMIVersion() string {
if _, ok := in.Annotations[AnnotationAMIFamilyCompatibility]; ok {
return "latest"
func AMIFamilyFromAlias(alias string) string {
components := strings.Split(alias, "@")
if len(components) != 2 {
log.Fatalf("failed to parse AMI alias %q, invalid format", alias)
}
if term, ok := lo.Find(in.Spec.AMISelectorTerms, func(t AMISelectorTerm) bool {
return t.Alias != ""
}); ok {
parts := strings.Split(term.Alias, "@")
if len(parts) != 2 {
log.Fatalf("failed to parse AMI alias %q, invalid format", term.Alias)
}
return parts[1]
family, ok := lo.Find([]string{
AMIFamilyAL2,
AMIFamilyAL2023,
AMIFamilyBottlerocket,
AMIFamilyWindows2019,
AMIFamilyWindows2022,
}, func(family string) bool {
return strings.ToLower(family) == components[0]
})
if !ok {
log.Fatalf("%q is an invalid alias family", components[0])
}
return family
}

func AMIVersionFromAlias(alias string) string {
components := strings.Split(alias, "@")
if len(components) != 2 {
log.Fatalf("failed to parse AMI alias %q, invalid format", alias)
}
return "latest"
return components[1]
}

// EC2NodeClassList contains a list of EC2NodeClass
Expand Down
130 changes: 91 additions & 39 deletions pkg/apis/v1/ec2nodeclass_conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,32 @@ import (
"strings"

"github.com/samber/lo"
"k8s.io/apimachinery/pkg/api/resource"
"knative.dev/pkg/apis"

"github.com/aws/aws-sdk-go/service/ec2"

"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
)

func (in *EC2NodeClass) ConvertTo(ctx context.Context, to apis.Convertible) error {
v1beta1enc := to.(*v1beta1.EC2NodeClass)
v1beta1enc.ObjectMeta = in.ObjectMeta
v1beta1enc.Annotations = lo.OmitByKeys(v1beta1enc.Annotations, []string{AnnotationUbuntuCompatibilityKey})

if value, ok := in.Annotations[AnnotationUbuntuCompatibilityKey]; ok {
compatSpecifiers := strings.Split(value, ",")
// The only blockDeviceMappings present on the v1 EC2NodeClass are those that we injected during conversion.
// These should be dropped.
if lo.Contains(compatSpecifiers, AnnotationUbuntuCompatibilityBlockDeviceMappings) {
in.Spec.BlockDeviceMappings = nil
}
// We don't need to explicitly check for the AMIFamily compat specifier, the presence of the annotation implies its existence
v1beta1enc.Spec.AMIFamily = lo.ToPtr(AMIFamilyUbuntu)
} else {
v1beta1enc.Spec.AMIFamily = lo.ToPtr(in.AMIFamily())
}

v1beta1enc.Spec.AMIFamily = lo.ToPtr(in.AMIFamily())
in.Spec.convertTo(&v1beta1enc.Spec)
in.Status.convertTo((&v1beta1enc.Status))
return nil
Expand All @@ -49,13 +65,16 @@ func (in *EC2NodeClassSpec) convertTo(v1beta1enc *v1beta1.EC2NodeClassSpec) {
Tags: sg.Tags,
}
})
v1beta1enc.AMISelectorTerms = lo.Map(in.AMISelectorTerms, func(ami AMISelectorTerm, _ int) v1beta1.AMISelectorTerm {
return v1beta1.AMISelectorTerm{
ID: ami.ID,
Name: ami.Name,
Owner: ami.Owner,
Tags: ami.Tags,
v1beta1enc.AMISelectorTerms = lo.FilterMap(in.AMISelectorTerms, func(term AMISelectorTerm, _ int) (v1beta1.AMISelectorTerm, bool) {
if term.Alias != "" {
return v1beta1.AMISelectorTerm{}, false
}
return v1beta1.AMISelectorTerm{
ID: term.ID,
Name: term.Name,
Owner: term.Owner,
Tags: term.Tags,
}, true
})
v1beta1enc.AssociatePublicIPAddress = in.AssociatePublicIPAddress
v1beta1enc.Context = in.Context
Expand Down Expand Up @@ -104,33 +123,81 @@ func (in *EC2NodeClass) ConvertFrom(ctx context.Context, from apis.Convertible)
v1beta1enc := from.(*v1beta1.EC2NodeClass)
in.ObjectMeta = v1beta1enc.ObjectMeta

// TODO: jmdeal@ remove before v1
// Temporarily fail closed when trying to convert EC2NodeClasses with the Ubuntu AMI family since compatibility support isn't yet integrated.
// This check can be removed once it's added.
if lo.FromPtr(v1beta1enc.Spec.AMIFamily) == v1beta1.AMIFamilyUbuntu {
return fmt.Errorf("failed to convert v1beta1 EC2NodeClass to v1, conversion for Ubuntu AMIFamily is currently unsupported")
}
switch lo.FromPtr(v1beta1enc.Spec.AMIFamily) {
case AMIFamilyAL2, AMIFamilyAL2023, AMIFamilyBottlerocket, AMIFamilyWindows2019, AMIFamilyWindows2022:
// If no amiSelectorTerms are specified, we can create an alias and don't need to specify amiFamily. Otherwise,
// we'll carry over the amiSelectorTerms and amiFamily.
if len(v1beta1enc.Spec.AMISelectorTerms) == 0 {
in.Spec.AMIFamily = nil
in.Spec.AMISelectorTerms = []AMISelectorTerm{{
Alias: fmt.Sprintf("%s@latest", strings.ToLower(lo.FromPtr(v1beta1enc.Spec.AMIFamily))),
}}
} else {
in.Spec.AMIFamily = v1beta1enc.Spec.AMIFamily
}
case AMIFamilyUbuntu:
// If there are no AMISelectorTerms specified, we will fail closed when converting the NodeClass. Users must
// pin their AMIs **before** upgrading to Karpenter v1.0.0 if they were using the Ubuntu AMIFamily.
// TODO: jmdeal@ verify doc link to the upgrade guide once available
if len(v1beta1enc.Spec.AMISelectorTerms) == 0 {
return fmt.Errorf("converting EC2NodeClass %q from v1beta1 to v1, automatic Ubuntu AMI discovery is not supported (https://karpenter.sh/v1.0/upgrading/upgrade-guide/)", v1beta1enc.Name)
}

// If the AMIFamily is still supported by the v1 APIs, and there are no AMISelectorTerms defined, create an alias.
// Otherwise, don't modify the AMISelectorTerms and add the compatibility annotation.
if lo.Contains([]string{
AMIFamilyAL2, AMIFamilyAL2023, AMIFamilyBottlerocket, AMIFamilyWindows2019, AMIFamilyWindows2022,
}, lo.FromPtr(v1beta1enc.Spec.AMIFamily)) && len(v1beta1enc.Spec.AMISelectorTerms) == 0 {
in.Spec.AMISelectorTerms = []AMISelectorTerm{{
Alias: fmt.Sprintf("%s@latest", strings.ToLower(lo.FromPtr(v1beta1enc.Spec.AMIFamily))),
}}
} else {
// If AMISelectorTerms were specified, we can continue to use them to discover Ubuntu AMIs and use the AL2 AMI
// family for bootstrapping. AL2 and Ubuntu have an identical UserData format, but do have different default
// BlockDeviceMappings. We'll set the BlockDeviceMappings to Ubuntu's default if no user specified
// BlockDeviceMappings are present.
compatSpecifiers := []string{AnnotationUbuntuCompatibilityAMIFamily}
in.Spec.AMIFamily = lo.ToPtr(AMIFamilyAL2)
if v1beta1enc.Spec.BlockDeviceMappings == nil {
compatSpecifiers = append(compatSpecifiers, AnnotationUbuntuCompatibilityBlockDeviceMappings)
in.Spec.BlockDeviceMappings = []*BlockDeviceMapping{{
DeviceName: lo.ToPtr("/dev/sda1"),
RootVolume: true,
EBS: &BlockDevice{
Encrypted: lo.ToPtr(true),
VolumeType: lo.ToPtr(ec2.VolumeTypeGp3),
VolumeSize: lo.ToPtr(resource.MustParse("20Gi")),
},
}}
}
// This compatibility annotation will be used to determine if the amiFamily was mutated from Ubuntu to AL2, and
// if we needed to inject any blockDeviceMappings. This is required to enable a round-trip conversion.
in.Annotations = lo.Assign(in.Annotations, map[string]string{
AnnotationAMIFamilyCompatibility: lo.FromPtr(v1beta1enc.Spec.AMIFamily),
AnnotationUbuntuCompatibilityKey: strings.Join(compatSpecifiers, ","),
})
default:
// The amiFamily is custom or undefined (shouldn't be possible via validation). We'll treat it as custom
// regardless.
in.Spec.AMIFamily = lo.ToPtr(AMIFamilyCustom)
}

in.Spec.convertFrom(&v1beta1enc.Spec)
in.Status.convertFrom((&v1beta1enc.Status))
in.Status.convertFrom(&v1beta1enc.Status)
return nil
}

func (in *EC2NodeClassSpec) convertFrom(v1beta1enc *v1beta1.EC2NodeClassSpec) {
if in.AMISelectorTerms == nil {
in.AMISelectorTerms = lo.Map(v1beta1enc.AMISelectorTerms, func(ami v1beta1.AMISelectorTerm, _ int) AMISelectorTerm {
return AMISelectorTerm{
ID: ami.ID,
Name: ami.Name,
Owner: ami.Owner,
Tags: ami.Tags,
}
})
}
if in.BlockDeviceMappings == nil {
in.BlockDeviceMappings = lo.Map(v1beta1enc.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping, _ int) *BlockDeviceMapping {
return &BlockDeviceMapping{
DeviceName: bdm.DeviceName,
RootVolume: bdm.RootVolume,
EBS: (*BlockDevice)(bdm.EBS),
}
})
}

in.SubnetSelectorTerms = lo.Map(v1beta1enc.SubnetSelectorTerms, func(subnet v1beta1.SubnetSelectorTerm, _ int) SubnetSelectorTerm {
return SubnetSelectorTerm{
ID: subnet.ID,
Expand All @@ -144,14 +211,6 @@ func (in *EC2NodeClassSpec) convertFrom(v1beta1enc *v1beta1.EC2NodeClassSpec) {
Tags: sg.Tags,
}
})
in.AMISelectorTerms = append(in.AMISelectorTerms, lo.Map(v1beta1enc.AMISelectorTerms, func(ami v1beta1.AMISelectorTerm, _ int) AMISelectorTerm {
return AMISelectorTerm{
ID: ami.ID,
Name: ami.Name,
Owner: ami.Owner,
Tags: ami.Tags,
}
})...)
in.AssociatePublicIPAddress = v1beta1enc.AssociatePublicIPAddress
in.Context = v1beta1enc.Context
in.DetailedMonitoring = v1beta1enc.DetailedMonitoring
Expand All @@ -161,13 +220,6 @@ func (in *EC2NodeClassSpec) convertFrom(v1beta1enc *v1beta1.EC2NodeClassSpec) {
in.Tags = v1beta1enc.Tags
in.UserData = v1beta1enc.UserData
in.MetadataOptions = (*MetadataOptions)(v1beta1enc.MetadataOptions)
in.BlockDeviceMappings = lo.Map(v1beta1enc.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping, _ int) *BlockDeviceMapping {
return &BlockDeviceMapping{
DeviceName: bdm.DeviceName,
RootVolume: bdm.RootVolume,
EBS: (*BlockDevice)(bdm.EBS),
}
})
}

func (in *EC2NodeClassStatus) convertFrom(v1beta1enc *v1beta1.EC2NodeClassStatus) {
Expand Down
Loading

0 comments on commit 094b57c

Please sign in to comment.