Add initial implementation to add capacityReservationSelectorTerms an…

…d update status when found

cmd/controller/main.go

@@ -62,6 +62,7 @@ func main() {
 			cloudProvider,
 			op.SubnetProvider,
 			op.SecurityGroupProvider,
+			op.CapacityReservationProvider,
 			op.InstanceProfileProvider,
 			op.InstanceProvider,
 			op.PricingProvider,

pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -216,6 +216,43 @@ spec:
                 - message: must have only one blockDeviceMappings with rootVolume
                   rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size()
                     <= 1
+              capacityReservationSelectorTerms:
+                description: CapacityReservationSelectorTerms is a list of or Capacity
+                  Reservation selector terms. The terms are ORed.
+                items:
+                  description: |-
+                    CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+                    If multiple fields are used for selection, the requirements are ANDed.
+                  properties:
+                    availabilityZone:
+                      description: The Availability Zone of the Capacity Reservation
+                      type: string
+                    id:
+                      description: The platform of operating system for which the
+                        Capacity Reservation reserves capacity
+                      type: string
+                    instanceType:
+                      description: The type of operating system for which the Capacity
+                        Reservation reserves capacity
+                      type: string
+                    ownerId:
+                      description: The ID of the Amazon Web Services account that
+                        owns the Capacity Reservation
+                      type: string
+                    tags:
+                      additionalProperties:
+                        type: string
+                      description: |-
+                        Tags is a map of key/value tags used to select subnets
+                        Specifying '*' for a value selects all values for a given tag key.
+                      maxProperties: 20
+                      type: object
+                      x-kubernetes-validations:
+                      - message: empty tag keys or values aren't supported
+                        rule: self.all(k, k != '' && self[k] != '')
+                  type: object
+                maxItems: 30
+                type: array
               context:
                 description: |-
                   Context is a Reserved field in EC2 APIs
@@ -516,6 +553,75 @@ spec:
                   - requirements
                   type: object
                 type: array
+              capacityReservations:
+                description: |-
+                  CapacityReservations contains the current Capacity Reservations values that are available to the
+                  cluster under the CapacityReservations selectors.
+                items:
+                  description: CapacityReservation contains resolved Capacity Reservation
+                    selector values utilized for node launch
+                  properties:
+                    availabilityZone:
+                      description: AvailabilityZone of the Capacity Reservation
+                      type: string
+                    availableInstanceCount:
+                      description: Available Instance Count of the Capacity Reservation
+                      type: integer
+                    endDate:
+                      description: |-
+                        The date and time at which the Capacity Reservation expires. When a Capacity
+                        Reservation expires, the reserved capacity is released and you can no longer
+                        launch instances into it. The Capacity Reservation's state changes to expired
+                        when it reaches its end date and time.
+                      type: string
+                    endDateType:
+                      description: |-
+                        Indicates the way in which the Capacity Reservation ends. A Capacity Reservation
+                        can have one of the following end types:
+
+
+                           * unlimited - The Capacity Reservation remains active until you explicitly
+                           cancel it.
+
+
+                           * limited - The Capacity Reservation expires automatically at a specified
+                           date and time.
+                      type: string
+                    id:
+                      description: ID of the Capacity Reservation
+                      type: string
+                    instanceMatchCriteria:
+                      description: Instance Match Criteria of the Capacity Reservation
+                      type: string
+                    instancePlatform:
+                      description: Instance Platform of the Capacity Reservation
+                      type: string
+                    instanceType:
+                      description: Instance Type of the Capacity Reservation
+                      type: string
+                    ownerId:
+                      description: Owner Id of the Capacity Reservation
+                      type: string
+                    startDate:
+                      description: The date and time at which the Capacity Reservation
+                        was started.
+                      type: string
+                    totalInstanceCount:
+                      description: Total Instance Count of the Capacity Reservation
+                      type: integer
+                  required:
+                  - availabilityZone
+                  - availableInstanceCount
+                  - endDateType
+                  - id
+                  - instanceMatchCriteria
+                  - instancePlatform
+                  - instanceType
+                  - ownerId
+                  - startDate
+                  - totalInstanceCount
+                  type: object
+                type: array
               instanceProfile:
                 description: InstanceProfile contains the resolved instance profile
                   for the role

pkg/apis/v1beta1/ec2nodeclass.go

@@ -55,6 +55,10 @@ type EC2NodeClassSpec struct {
 	// +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022}
 	// +required
 	AMIFamily *string `json:"amiFamily"`
+	// CapacityReservationSelectorTerms is a list of or Capacity Reservation selector terms. The terms are ORed.
+	// +kubebuilder:validation:MaxItems:=30
+	// +required
+	CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms,omitempty" hash:"ignore"`
 	// UserData to be applied to the provisioned nodes.
 	// It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into
 	// this UserData to ensure nodes are being provisioned with the correct configuration.
@@ -175,6 +179,29 @@ type AMISelectorTerm struct {
 	Owner string `json:"owner,omitempty"`
 }
 
+// CapacityReservationSelectorTerm defines selection logic for a Capacity Reservation used by Karpenter to launch nodes.
+// If multiple fields are used for selection, the requirements are ANDed.
+type CapacityReservationSelectorTerm struct {
+	// The Availability Zone of the Capacity Reservation
+	// +optional
+	AvailabilityZone string `json:"availabilityZone,omitempty"`
+	// The platform of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	ID string `json:"id,omitempty"`
+	// Tags is a map of key/value tags used to select subnets
+	// Specifying '*' for a value selects all values for a given tag key.
+	// +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')"
+	// +kubebuilder:validation:MaxProperties:=20
+	// +optional
+	Tags map[string]string `json:"tags,omitempty"`
+	// The type of operating system for which the Capacity Reservation reserves capacity
+	// +optional
+	InstanceType string `json:"instanceType,omitempty"`
+	// The ID of the Amazon Web Services account that owns the Capacity Reservation
+	// +optional
+	OwnerID string `json:"ownerId,omitempty"`
+}
+
 // MetadataOptions contains parameters for specifying the exposure of the
 // Instance Metadata Service to provisioned EC2 nodes.
 type MetadataOptions struct {

pkg/apis/v1beta1/ec2nodeclass_status.go

@@ -38,6 +38,53 @@ type SecurityGroup struct {
 	Name string `json:"name,omitempty"`
 }
 
+// CapacityReservation contains resolved Capacity Reservation selector values utilized for node launch
+type CapacityReservation struct {
+	// ID of the Capacity Reservation
+	// +required
+	ID string `json:"id"`
+	// AvailabilityZone of the Capacity Reservation
+	// +required
+	AvailabilityZone string `json:"availabilityZone"`
+	// Available Instance Count of the Capacity Reservation
+	// +required
+	AvailableInstanceCount int `json:"availableInstanceCount"`
+	// The date and time at which the Capacity Reservation expires. When a Capacity
+	// Reservation expires, the reserved capacity is released and you can no longer
+	// launch instances into it. The Capacity Reservation's state changes to expired
+	// when it reaches its end date and time.
+	// +optional
+	EndDate *string `json:"endDate,omitempty"`
+	// Indicates the way in which the Capacity Reservation ends. A Capacity Reservation
+	// can have one of the following end types:
+	//
+	//    * unlimited - The Capacity Reservation remains active until you explicitly
+	//    cancel it.
+	//
+	//    * limited - The Capacity Reservation expires automatically at a specified
+	//    date and time.
+	// +required
+	EndDateType string `json:"endDateType"`
+	// Instance Match Criteria of the Capacity Reservation
+	// +required
+	InstanceMatchCriteria string `json:"instanceMatchCriteria"`
+	// Instance Platform of the Capacity Reservation
+	// +required
+	InstancePlatform string `json:"instancePlatform"`
+	// Instance Type of the Capacity Reservation
+	// +required
+	InstanceType string `json:"instanceType"`
+	// Owner Id of the Capacity Reservation
+	// +required
+	OwnerID string `json:"ownerId"`
+	// The date and time at which the Capacity Reservation was started.
+	// +required
+	StartDate string `json:"startDate"`
+	// Total Instance Count of the Capacity Reservation
+	// +required
+	TotalInstanceCount int `json:"totalInstanceCount"`
+}
+
 // AMI contains resolved AMI selector values utilized for node launch
 type AMI struct {
 	// ID of the AMI
@@ -53,6 +100,10 @@ type AMI struct {
 
 // EC2NodeClassStatus contains the resolved state of the EC2NodeClass
 type EC2NodeClassStatus struct {
+	// CapacityReservations contains the current Capacity Reservations values that are available to the
+	// cluster under the CapacityReservations selectors.
+	// +optional
+	CapacityReservations []CapacityReservation `json:"capacityReservations,omitempty"`
 	// Subnets contains the current Subnet values that are available to the
 	// cluster under the subnet selectors.
 	// +optional

pkg/cloudprovider/cloudprovider.go

@@ -96,6 +96,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC
 	}
 	instance, err := c.instanceProvider.Create(ctx, nodeClass, nodeClaim, instanceTypes)
 	if err != nil {
+		if cloudprovider.IsInsufficientCapacityError(err) {
+			return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("creating instance, %w", err))
+		}
 		return nil, fmt.Errorf("creating instance, %w", err)
 	}
 	instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool {

pkg/controllers/controllers.go

@@ -24,6 +24,7 @@ import (
 	nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination"
 	controllersinstancetype "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/instancetype"
 	controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/providers/pricing"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -52,12 +53,12 @@ import (
 
 func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder,
 	unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider subnet.Provider,
-	securityGroupProvider securitygroup.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
+	securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider, instanceProfileProvider instanceprofile.Provider, instanceProvider instance.Provider,
 	pricingProvider pricing.Provider, amiProvider amifamily.Provider, launchTemplateProvider launchtemplate.Provider, instanceTypeProvider instancetype.Provider) []controller.Controller {
 
 	controllers := []controller.Controller{
 		nodeclasshash.NewController(kubeClient),
-		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
+		nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, capacityReservationProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
 		nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider),
 		nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
 		nodeclaimtagging.NewController(kubeClient, instanceProvider),

pkg/controllers/nodeclass/status/controller.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/capacityreservation"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/securitygroup"
@@ -49,23 +50,25 @@ type nodeClassStatusReconciler interface {
 type Controller struct {
 	kubeClient client.Client
 
-	ami             *AMI
-	instanceprofile *InstanceProfile
-	subnet          *Subnet
-	securitygroup   *SecurityGroup
-	launchtemplate  *LaunchTemplate
+	ami                 *AMI
+	instanceprofile     *InstanceProfile
+	subnet              *Subnet
+	securitygroup       *SecurityGroup
+	capacityreservation *CapacityReservation
+	launchtemplate      *LaunchTemplate
 }
 
-func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider,
+func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider, capacityReservationProvider capacityreservation.Provider,
 	amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) corecontroller.Controller {
 	return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{
 		kubeClient: kubeClient,
 
-		ami:             &AMI{amiProvider: amiProvider},
-		subnet:          &Subnet{subnetProvider: subnetProvider},
-		securitygroup:   &SecurityGroup{securityGroupProvider: securityGroupProvider},
-		instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
-		launchtemplate:  &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
+		ami:                 &AMI{amiProvider: amiProvider},
+		subnet:              &Subnet{subnetProvider: subnetProvider},
+		securitygroup:       &SecurityGroup{securityGroupProvider: securityGroupProvider},
+		capacityreservation: &CapacityReservation{capacityReservationProvider: capacityReservationProvider},
+		instanceprofile:     &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
+		launchtemplate:      &LaunchTemplate{launchTemplateProvider: launchTemplateProvider},
 	})
 }
 
@@ -87,6 +90,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeCl
 		c.securitygroup,
 		c.instanceprofile,
 		c.launchtemplate,
+		c.capacityreservation,
 	} {
 		res, err := reconciler.Reconcile(ctx, nodeClass)
 		errs = multierr.Append(errs, err)