Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: RFC Implementation Supporting ODCR #6198

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: RFC Implementation Supporting ODCR #6198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/controller/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func main() {
op.AMIProvider,
op.LaunchTemplateProvider,
op.InstanceTypesProvider,
op.CapacityReservationProvider,
)...).
WithWebhooks(ctx, webhooks.NewWebhooks()...).
Start(ctx)
Expand Down
270 changes: 270 additions & 0 deletions pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml
View file
Open in desktop

Large diffs are not rendered by default.

42 changes: 42 additions & 0 deletions pkg/apis/v1/ec2nodeclass.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ type EC2NodeClassSpec struct {
// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
// +required
AMIFamily *string `json:"amiFamily"`
// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
// +kubebuilder:validation:MaxItems:=30
// +optional
CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
// UserData to be applied to the provisioned nodes.
// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
// this UserData to ensure nodes are being provisioned with the correct configuration.
Expand Down Expand Up @@ -175,6 +179,44 @@ type AMISelectorTerm struct {
Owner string `json:"owner,omitempty"`
}

// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
// If multiple fields are used for selection, the requirements are ANDed.
type CapacityReservationSelectorTerm struct {
// The Availability Zone of the Capacity Reservation
// +optional
AvailabilityZone string `json:"availabilityZone,omitempty"`
// The platform of operating system for which the Capacity Reservation reserves capacity
// +optional
ID string `json:"id,omitempty"`
// The type of operating system for which the Capacity Reservation reserves capacity
// +optional
InstanceType string `json:"instanceType,omitempty"`
// Tags is a map of key/value tags used to select subnets
// Specifying '*' for a value selects all values for a given tag key.
// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
// +kubebuilder:validation:MaxProperties:=20
// +optional
Tags map[string]string `json:"tags,omitempty"`
// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
// - open:
// The Capacity Reservation accepts all instances that have
// matching attributes (instance type, platform, and Availability
// Zone). Instances that have matching attributes launch into the
// Capacity Reservation automatically without specifying any
// additional parameters.
// - targeted:
// The Capacity Reservation only accepts instances that
// have matching attributes (instance type, platform, and
// Availability Zone), and explicitly target the Capacity
// Reservation. This ensures that only permitted instances can use
// the reserved capacity.
// +optional
Type string `json:"type,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be an enum and we validate through kubebuilder OpenAPI that this is only one of the valid values. See https://github.com/kubernetes-sigs/karpenter/blob/main/pkg/apis/v1beta1/nodepool.go#L76 for an example of this

// The ID of the Amazon Web Services account that owns the Capacity Reservation
// +optional
OwnerID string `json:"ownerId,omitempty"`
}

// MetadataOptions contains parameters for specifying the exposure of the
// Instance Metadata Service to provisioned EC2 nodes.
type MetadataOptions struct {
Expand Down
63 changes: 63 additions & 0 deletions pkg/apis/v1/ec2nodeclass_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,71 @@ type AMI struct {
Requirements []v1.NodeSelectorRequirement `json:"requirements"`
}

// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
type CapacityReservation struct {
// ID of the Capacity Reservation
// +required
ID string `json:"id"`
// AvailabilityZone of the Capacity Reservation
// +required
AvailabilityZone string `json:"availabilityZone"`
// Available Instance Count of the Capacity Reservation
// +required
AvailableInstanceCount int `json:"availableInstanceCount"`
// The date and time at which the Capacity Reservation expires. When a Capacity
// Reservation expires, the reserved capacity is released and you can no longer
// launch instances into it. The Capacity Reservation's state changes to expired
// when it reaches its end date and time.
// +optional
EndDate *string `json:"endDate,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
EndDate *string `json:"endDate,omitempty"`
EndDate *time.Time `json:"endDate,omitempty"`

Can this map to the actual go type?

// Indicates the way in which the Capacity Reservation ends. A Capacity Reservation
// can have one of the following end types:
//
// * unlimited - The Capacity Reservation remains active until you explicitly
// cancel it.
//
// * limited - The Capacity Reservation expires automatically at a specified
// date and time.
// +required
EndDateType string `json:"endDateType"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this just be implied from the presence or lack of the end date time?

// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
// - open:
// The Capacity Reservation accepts all instances that have
// matching attributes (instance type, platform, and Availability
// Zone). Instances that have matching attributes launch into the
// Capacity Reservation automatically without specifying any
// additional parameters.
// - targeted:
// The Capacity Reservation only accepts instances that
// have matching attributes (instance type, platform, and
// Availability Zone), and explicitly target the Capacity
// Reservation. This ensures that only permitted instances can use
// the reserved capacity.
// +required
InstanceMatchCriteria string `json:"instanceMatchCriteria"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're calling this type above, we should probably match that here

// Instance Platform of the Capacity Reservation
// +required
InstancePlatform string `json:"instancePlatform"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we think that discovering this information is interesting?

// Instance Type of the Capacity Reservation
// +required
InstanceType string `json:"instanceType"`
// Owner Id of the Capacity Reservation
// +required
OwnerID string `json:"ownerId"`
// The date and time at which the Capacity Reservation was started.
// +required
StartDate string `json:"startDate"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this relevant right now? Is it possible for you to delay the start of an ODCR?

// Total Instance Count of the Capacity Reservation
// +required
TotalInstanceCount int `json:"totalInstanceCount"`
}

// EC2NodeClassStatus contains the resolved state of the EC2NodeClass
type EC2NodeClassStatus struct {
// CapacityReservations contains the current Capacity Reservations values that are available to the
// cluster under the CapacityReservations selectors.
// +optional
CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
// Subnets contains the current Subnet values that are available to the
// cluster under the subnet selectors.
// +optional
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/v1/labels.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
func init() {
v1beta1.RestrictedLabelDomains = v1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...)
v1beta1.WellKnownLabels = v1beta1.WellKnownLabels.Insert(
LabelCapactiyReservationID,
LabelInstanceHypervisor,
LabelInstanceEncryptionInTransitSupported,
LabelInstanceCategory,
Expand Down Expand Up @@ -119,6 +120,7 @@ var (
AnnotationEC2NodeClassHash = Group + "/ec2nodeclass-hash"
AnnotationEC2NodeClassHashVersion = Group + "/ec2nodeclass-hash-version"
AnnotationInstanceTagged = Group + "/tagged"
LabelCapactiyReservationID = Group + "/capacity-reservation-id"

TagNodeClaim = v1beta1.Group + "/nodeclaim"
TagManagedLaunchTemplate = Group + "/cluster"
Expand Down
56 changes: 56 additions & 0 deletions pkg/apis/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 42 additions & 0 deletions pkg/apis/v1beta1/ec2nodeclass.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ type EC2NodeClassSpec struct {
// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I get that you need this in v1beta1 right now for testing, but I think that we should think about dropping this from the v1beta1 API since I suspect that this will just come after v1

// +required
AMIFamily *string `json:"amiFamily"`
// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
// +kubebuilder:validation:MaxItems:=30
// +optional
CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
// UserData to be applied to the provisioned nodes.
// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
// this UserData to ensure nodes are being provisioned with the correct configuration.
Expand Down Expand Up @@ -175,6 +179,44 @@ type AMISelectorTerm struct {
Owner string `json:"owner,omitempty"`
}

// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
// If multiple fields are used for selection, the requirements are ANDed.
type CapacityReservationSelectorTerm struct {
jonathan-innis marked this conversation as resolved.
Show resolved Hide resolved
// The Availability Zone of the Capacity Reservation
// +optional
AvailabilityZone string `json:"availabilityZone,omitempty"`
// The platform of operating system for which the Capacity Reservation reserves capacity
// +optional
ID string `json:"id,omitempty"`
// The type of operating system for which the Capacity Reservation reserves capacity
// +optional
InstanceType string `json:"instanceType,omitempty"`
// Tags is a map of key/value tags used to select subnets
// Specifying '*' for a value selects all values for a given tag key.
// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
// +kubebuilder:validation:MaxProperties:=20
// +optional
Tags map[string]string `json:"tags,omitempty"`
// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
// - open:
// The Capacity Reservation accepts all instances that have
// matching attributes (instance type, platform, and Availability
// Zone). Instances that have matching attributes launch into the
// Capacity Reservation automatically without specifying any
// additional parameters.
// - targeted:
// The Capacity Reservation only accepts instances that
// have matching attributes (instance type, platform, and
// Availability Zone), and explicitly target the Capacity
// Reservation. This ensures that only permitted instances can use
// the reserved capacity.
// +optional
Type string `json:"type,omitempty"`
// The ID of the Amazon Web Services account that owns the Capacity Reservation
// +optional
OwnerID string `json:"ownerId,omitempty"`
}

// MetadataOptions contains parameters for specifying the exposure of the
// Instance Metadata Service to provisioned EC2 nodes.
type MetadataOptions struct {
Expand Down
63 changes: 63 additions & 0 deletions pkg/apis/v1beta1/ec2nodeclass_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,65 @@ type SecurityGroup struct {
Name string `json:"name,omitempty"`
}

// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
type CapacityReservation struct {
// ID of the Capacity Reservation
// +required
ID string `json:"id"`
// AvailabilityZone of the Capacity Reservation
// +required
AvailabilityZone string `json:"availabilityZone"`
// Available Instance Count of the Capacity Reservation
// +required
AvailableInstanceCount int `json:"availableInstanceCount"`
// The date and time at which the Capacity Reservation expires. When a Capacity
// Reservation expires, the reserved capacity is released and you can no longer
// launch instances into it. The Capacity Reservation's state changes to expired
// when it reaches its end date and time.
// +optional
EndDate *string `json:"endDate,omitempty"`
// Indicates the way in which the Capacity Reservation ends. A Capacity Reservation
// can have one of the following end types:
//
// * unlimited - The Capacity Reservation remains active until you explicitly
// cancel it.
//
// * limited - The Capacity Reservation expires automatically at a specified
// date and time.
// +required
EndDateType string `json:"endDateType"`
// Indicates the type of instance launches that the Capacity Reservation accepts. The options include:
// - open:
// The Capacity Reservation accepts all instances that have
// matching attributes (instance type, platform, and Availability
// Zone). Instances that have matching attributes launch into the
// Capacity Reservation automatically without specifying any
// additional parameters.
// - targeted:
// The Capacity Reservation only accepts instances that
// have matching attributes (instance type, platform, and
// Availability Zone), and explicitly target the Capacity
// Reservation. This ensures that only permitted instances can use
// the reserved capacity.
// +required
InstanceMatchCriteria string `json:"instanceMatchCriteria"`
tvonhacht-apple marked this conversation as resolved.
Show resolved Hide resolved
// Instance Platform of the Capacity Reservation
// +required
InstancePlatform string `json:"instancePlatform"`
tvonhacht-apple marked this conversation as resolved.
Show resolved Hide resolved
// Instance Type of the Capacity Reservation
// +required
InstanceType string `json:"instanceType"`
// Owner Id of the Capacity Reservation
// +required
OwnerID string `json:"ownerId"`
tvonhacht-apple marked this conversation as resolved.
Show resolved Hide resolved
// The date and time at which the Capacity Reservation was started.
// +required
StartDate string `json:"startDate"`
// Total Instance Count of the Capacity Reservation
// +required
TotalInstanceCount int `json:"totalInstanceCount"`
}

// AMI contains resolved AMI selector values utilized for node launch
type AMI struct {
// ID of the AMI
Expand All @@ -57,6 +116,10 @@ type AMI struct {

// EC2NodeClassStatus contains the resolved state of the EC2NodeClass
type EC2NodeClassStatus struct {
// CapacityReservations contains the current Capacity Reservations values that are available to the
// cluster under the CapacityReservations selectors.
// +optional
CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
// Subnets contains the current Subnet values that are available to the
// cluster under the subnet selectors.
// +optional
Expand Down