From 071154e1725a5b789951be38003a404637097931 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 31 Jul 2024 03:44:38 -0700 Subject: [PATCH] Add Conversion webhooks --- kwok/cloudprovider/cloudprovider.go | 13 + kwok/main.go | 2 +- pkg/apis/v1/nodeclaim_conversion.go | 190 +++++++ pkg/apis/v1/nodeclaim_conversion_test.go | 536 ++++++++++++++++++ pkg/apis/v1/nodepool_conversion.go | 202 +++++++ pkg/apis/v1/nodepool_conversion_test.go | 536 ++++++++++++++++++ pkg/apis/v1beta1/nodeclaim_conversion.go | 29 + pkg/apis/v1beta1/nodepool_conversion.go | 29 + pkg/cloudprovider/fake/cloudprovider.go | 25 +- pkg/cloudprovider/types.go | 4 + pkg/operator/injection/injection.go | 24 + pkg/operator/operator.go | 9 +- .../karpenter.test.sh_testnodeclasses.yaml | 109 ++++ pkg/test/v1alpha1/doc.go | 39 ++ pkg/test/v1alpha1/testnodeclass.go | 52 ++ pkg/test/v1alpha1/testnodeclass_status.go | 39 ++ pkg/test/v1alpha1/zz_generated.deepcopy.go | 106 ++++ pkg/webhooks/webhooks.go | 38 ++ 18 files changed, 1975 insertions(+), 7 deletions(-) create mode 100644 pkg/apis/v1/nodeclaim_conversion.go create mode 100644 pkg/apis/v1/nodeclaim_conversion_test.go create mode 100644 pkg/apis/v1/nodepool_conversion.go create mode 100644 pkg/apis/v1/nodepool_conversion_test.go create mode 100644 pkg/apis/v1beta1/nodeclaim_conversion.go create mode 100644 pkg/apis/v1beta1/nodepool_conversion.go create mode 100644 pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml create mode 100644 pkg/test/v1alpha1/doc.go create mode 100644 pkg/test/v1alpha1/testnodeclass.go create mode 100644 pkg/test/v1alpha1/testnodeclass_status.go create mode 100644 pkg/test/v1alpha1/zz_generated.deepcopy.go diff --git a/kwok/cloudprovider/cloudprovider.go b/kwok/cloudprovider/cloudprovider.go index 47d70bf947..ce4f0720d7 100644 --- a/kwok/cloudprovider/cloudprovider.go +++ b/kwok/cloudprovider/cloudprovider.go @@ -24,17 +24,20 @@ import ( "math/rand" "strings" + "github.com/awslabs/operatorpkg/object" "github.com/docker/docker/pkg/namesgenerator" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" ) func NewCloudProvider(ctx context.Context, kubeClient client.Client, instanceTypes []*cloudprovider.InstanceType) *CloudProvider { @@ -118,6 +121,16 @@ func (c CloudProvider) Name() string { return "kwok" } +func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + { + Group: object.GVK(&v1alpha1.TestNodeClass{}).Group, + Version: object.GVK(&v1alpha1.TestNodeClass{}).Version, + Kind: object.GVK(&v1alpha1.TestNodeClass{}).Kind, + }, + } +} + func (c CloudProvider) getInstanceType(instanceTypeName string) (*cloudprovider.InstanceType, error) { it, found := lo.Find(c.instanceTypes, func(it *cloudprovider.InstanceType) bool { return it.Name == instanceTypeName diff --git a/kwok/main.go b/kwok/main.go index 0e8969974a..c03faa6439 100644 --- a/kwok/main.go +++ b/kwok/main.go @@ -45,5 +45,5 @@ func main() { state.NewCluster(op.Clock, op.GetClient(), cloudProvider), op.EventRecorder, cloudProvider, - )...).Start(ctx) + )...).Start(ctx, cloudProvider) } diff --git a/pkg/apis/v1/nodeclaim_conversion.go b/pkg/apis/v1/nodeclaim_conversion.go new file mode 100644 index 0000000000..c343e15f78 --- /dev/null +++ b/pkg/apis/v1/nodeclaim_conversion.go @@ -0,0 +1,190 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/awslabs/operatorpkg/status" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "knative.dev/pkg/apis" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" +) + +// convert v1 to v1beta1 +func (in *NodeClaim) ConvertTo(ctx context.Context, to apis.Convertible) error { + v1beta1NC := to.(*v1beta1.NodeClaim) + v1beta1NC.ObjectMeta = in.ObjectMeta + + in.Status.convertTo((&v1beta1NC.Status)) + return in.Spec.convertTo(ctx, &v1beta1NC.Spec, in.Annotations[KubeletCompatibilityAnnotationKey]) +} + +func (in *NodeClaimSpec) convertTo(ctx context.Context, v1beta1nc *v1beta1.NodeClaimSpec, kubeletAnnotation string) error { + v1beta1nc.Taints = in.Taints + v1beta1nc.StartupTaints = in.StartupTaints + v1beta1nc.Resources = v1beta1.ResourceRequirements(in.Resources) + v1beta1nc.Requirements = lo.Map(in.Requirements, func(v1Requirements NodeSelectorRequirementWithMinValues, _ int) v1beta1.NodeSelectorRequirementWithMinValues { + return v1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1Requirements.Key, + Operator: v1Requirements.Operator, + Values: v1Requirements.Values, + }, + MinValues: v1Requirements.MinValues, + } + }) + + if in.NodeClassRef != nil { + nodeclass, found := lo.Find(injection.GetNodeClasses(ctx), func(nc schema.GroupVersionKind) bool { + return nc.Kind == in.NodeClassRef.Kind && nc.Group == in.NodeClassRef.Group + }) + v1beta1nc.NodeClassRef = &v1beta1.NodeClassReference{ + Kind: in.NodeClassRef.Kind, + Name: in.NodeClassRef.Name, + APIVersion: lo.Ternary(found, nodeclass.GroupVersion().String(), ""), + } + } + + if kubeletAnnotation != "" { + v1beta1kubelet := &v1beta1.KubeletConfiguration{} + err := json.Unmarshal([]byte(kubeletAnnotation), v1beta1kubelet) + if err != nil { + return fmt.Errorf("unmarshaling kubelet config annotation, %w", err) + } + v1beta1nc.Kubelet = v1beta1kubelet + } + return nil +} + +func (in *NodeClaimStatus) convertTo(v1beta1nc *v1beta1.NodeClaimStatus) { + v1beta1nc.NodeName = in.NodeName + v1beta1nc.ProviderID = in.ProviderID + v1beta1nc.ImageID = in.ImageID + v1beta1nc.Capacity = in.Capacity + v1beta1nc.Allocatable = in.Allocatable + v1beta1nc.Conditions = lo.Map(in.Conditions, func(v1status status.Condition, _ int) apis.Condition { + return apis.Condition{ + Type: apis.ConditionType(v1status.Type), + Reason: v1status.Reason, + Status: v1.ConditionStatus(v1status.Status), + Message: v1status.Message, + LastTransitionTime: apis.VolatileTime{Inner: v1status.LastTransitionTime}, + } + }) +} + +// convert v1beta1 to v1 +func (in *NodeClaim) ConvertFrom(ctx context.Context, from apis.Convertible) error { + v1beta1NC := from.(*v1beta1.NodeClaim) + in.ObjectMeta = v1beta1NC.ObjectMeta + + in.Status.convertFrom((&v1beta1NC.Status)) + kubeletAnnotation, err := in.Spec.convertFrom(ctx, &v1beta1NC.Spec) + if err != nil { + return err + } + if kubeletAnnotation == "" { + in.Annotations = lo.OmitByKeys(in.Annotations, []string{KubeletCompatibilityAnnotationKey}) + } else { + in.Annotations = lo.Assign(in.Annotations, map[string]string{KubeletCompatibilityAnnotationKey: kubeletAnnotation}) + } + return in.setExpireAfter(ctx, v1beta1NC) +} + +// only need to set expireAfter for v1beta1 to v1 +func (in *NodeClaim) setExpireAfter(ctx context.Context, v1beta1nc *v1beta1.NodeClaim) error { + kubeClient := injection.GetClient(ctx) + nodePoolName, ok := v1beta1nc.Labels[NodePoolLabelKey] + if !ok { + // If we don't have a nodepool for this nodeclaim, there's nothing to look up + return nil + } + nodePool := &NodePool{} + if err := kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { + if errors.IsNotFound(err) { + // If the nodepool doesn't exist, fallback to no expiry, and use the CRD default + return nil + } + return fmt.Errorf("getting nodepool, %w", err) + } + in.Spec.ExpireAfter = nodePool.Spec.Template.Spec.ExpireAfter + return nil +} + +func (in *NodeClaimSpec) convertFrom(ctx context.Context, v1beta1nc *v1beta1.NodeClaimSpec) (string, error) { + in.Taints = v1beta1nc.Taints + in.StartupTaints = v1beta1nc.StartupTaints + in.Resources = ResourceRequirements(v1beta1nc.Resources) + in.Requirements = lo.Map(v1beta1nc.Requirements, func(v1beta1Requirements v1beta1.NodeSelectorRequirementWithMinValues, _ int) NodeSelectorRequirementWithMinValues { + return NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1Requirements.Key, + Operator: v1beta1Requirements.Operator, + Values: v1beta1Requirements.Values, + }, + MinValues: v1beta1Requirements.MinValues, + } + }) + + defaultNodeClassGVK := injection.GetNodeClasses(ctx)[0] + nodeclassGroupVersion, err := schema.ParseGroupVersion(v1beta1nc.NodeClassRef.APIVersion) + if err != nil { + return "", err + } + in.NodeClassRef = &NodeClassReference{ + Name: v1beta1nc.NodeClassRef.Name, + Kind: lo.Ternary(v1beta1nc.NodeClassRef.Kind == "", defaultNodeClassGVK.Kind, v1beta1nc.NodeClassRef.Kind), + Group: lo.Ternary(v1beta1nc.NodeClassRef.APIVersion == "", defaultNodeClassGVK.Group, nodeclassGroupVersion.Group), + } + + if v1beta1nc.Kubelet != nil { + kubelet, err := json.Marshal(v1beta1nc.Kubelet) + if err != nil { + return "", fmt.Errorf("marshaling kubelet config annotation, %w", err) + } + return string(kubelet), nil + } + return "", nil +} + +func (in *NodeClaimStatus) convertFrom(v1beta1nc *v1beta1.NodeClaimStatus) { + in.NodeName = v1beta1nc.NodeName + in.ProviderID = v1beta1nc.ProviderID + in.ImageID = v1beta1nc.ImageID + in.Capacity = v1beta1nc.Capacity + in.Allocatable = v1beta1nc.Allocatable + in.Conditions = lo.Map(v1beta1nc.Conditions, func(v1beta1status apis.Condition, _ int) status.Condition { + return status.Condition{ + Type: string(v1beta1status.Type), + Reason: v1beta1status.Reason, + Status: metav1.ConditionStatus(v1beta1status.Status), + Message: v1beta1status.Message, + LastTransitionTime: v1beta1status.LastTransitionTime.Inner, + } + }) +} diff --git a/pkg/apis/v1/nodeclaim_conversion_test.go b/pkg/apis/v1/nodeclaim_conversion_test.go new file mode 100644 index 0000000000..9321d48015 --- /dev/null +++ b/pkg/apis/v1/nodeclaim_conversion_test.go @@ -0,0 +1,536 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "encoding/json" + "time" + + "github.com/awslabs/operatorpkg/object" + "github.com/awslabs/operatorpkg/status" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + . "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" + "sigs.k8s.io/karpenter/pkg/test" +) + +var _ = Describe("Convert v1 to v1beta1 NodeClaim API", func() { + var ( + v1nodepool *NodePool + v1nodeclaim *NodeClaim + v1beta1nodeclaim *v1beta1.NodeClaim + ) + + BeforeEach(func() { + v1nodepool = &NodePool{ + ObjectMeta: test.ObjectMeta(), + Spec: NodePoolSpec{ + Template: NodeClaimTemplate{ + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "default", + }, + Requirements: []NodeSelectorRequirementWithMinValues{}, + }, + }, + }, + } + v1nodeclaim = &NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NodePoolLabelKey: v1nodepool.Name, + }, + }, + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "test", + Kind: "test-kind", + Group: "test-group", + }, + }, + } + v1beta1nodeclaim = &v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NodePoolLabelKey: v1nodepool.Name, + }, + }, + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test", + Kind: "test-kind", + APIVersion: "test-group/test-version", + }, + }, + } + Expect(env.Client.Create(ctx, v1nodepool)).To(Succeed()) + cloudProvider.NodeClassGroupVersionKind = cloudProvider.GetSupportedNodeClasses() + ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses()) + ctx = injection.WithClient(ctx, env.Client) + }) + It("should convert v1 nodeclaim metadata", func() { + v1nodeclaim.ObjectMeta = test.ObjectMeta() + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + v1beta1nodeclaim.Annotations = nil + Expect(v1beta1nodeclaim.ObjectMeta).To(BeEquivalentTo(v1nodeclaim.ObjectMeta)) + }) + Context("NodeClaim Spec", func() { + It("should convert v1 nodeclaim taints", func() { + v1nodeclaim.Spec.Taints = []v1.Taint{ + { + Key: "test-key-1", + Value: "test-value-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-2", + Value: "test-value-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1nodeclaim.Spec.Taints { + Expect(v1beta1nodeclaim.Spec.Taints[i].Key).To(Equal(v1nodeclaim.Spec.Taints[i].Key)) + Expect(v1beta1nodeclaim.Spec.Taints[i].Value).To(Equal(v1nodeclaim.Spec.Taints[i].Value)) + Expect(v1beta1nodeclaim.Spec.Taints[i].Effect).To(Equal(v1nodeclaim.Spec.Taints[i].Effect)) + } + }) + It("should convert v1 nodeclaim startup taints", func() { + v1nodeclaim.Spec.StartupTaints = []v1.Taint{ + { + Key: "test-key-startup-1", + Value: "test-value-startup-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-startup-2", + Value: "test-value-startup-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1nodeclaim.Spec.StartupTaints { + Expect(v1beta1nodeclaim.Spec.StartupTaints[i].Key).To(Equal(v1nodeclaim.Spec.StartupTaints[i].Key)) + Expect(v1beta1nodeclaim.Spec.StartupTaints[i].Value).To(Equal(v1nodeclaim.Spec.StartupTaints[i].Value)) + Expect(v1beta1nodeclaim.Spec.StartupTaints[i].Effect).To(Equal(v1nodeclaim.Spec.StartupTaints[i].Effect)) + } + }) + It("should convert v1 nodeclaim requirements", func() { + v1nodeclaim.Spec.Requirements = []NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(451613), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{CapacityTypeSpot}, + }, + MinValues: lo.ToPtr(9787513), + }, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1nodeclaim.Spec.Requirements { + Expect(v1beta1nodeclaim.Spec.Requirements[i].Key).To(Equal(v1nodeclaim.Spec.Requirements[i].Key)) + Expect(v1beta1nodeclaim.Spec.Requirements[i].Operator).To(Equal(v1nodeclaim.Spec.Requirements[i].Operator)) + Expect(v1beta1nodeclaim.Spec.Requirements[i].Values).To(Equal(v1nodeclaim.Spec.Requirements[i].Values)) + Expect(v1beta1nodeclaim.Spec.Requirements[i].MinValues).To(Equal(v1nodeclaim.Spec.Requirements[i].MinValues)) + } + }) + It("should convert v1 nodeclaim resources", func() { + v1nodeclaim.Spec.Resources = ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("134G"), + }, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + for key := range v1nodeclaim.Spec.Resources.Requests { + Expect(v1nodeclaim.Spec.Resources.Requests[key]).To(Equal(v1beta1nodeclaim.Spec.Resources.Requests[key])) + } + }) + Context("NodeClassRef", func() { + It("should convert v1 nodeclaim template nodeClassRef", func() { + v1nodeclaim.Spec.NodeClassRef = &NodeClassReference{ + Kind: object.GVK(&v1alpha1.TestNodeClass{}).Kind, + Name: "nodeclass-test", + Group: object.GVK(&v1alpha1.TestNodeClass{}).Group, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.Kind).To(Equal(v1nodeclaim.Spec.NodeClassRef.Kind)) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.Name).To(Equal(v1nodeclaim.Spec.NodeClassRef.Name)) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.APIVersion).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].GroupVersion().String())) + }) + It("should not include APIVersion for v1beta1 if Group and Kind is not in the supported nodeclass", func() { + v1nodeclaim.Spec.NodeClassRef = &NodeClassReference{ + Kind: "test-kind", + Name: "nodeclass-test", + Group: "testgroup.sh", + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.Kind).To(Equal(v1nodeclaim.Spec.NodeClassRef.Kind)) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.Name).To(Equal(v1nodeclaim.Spec.NodeClassRef.Name)) + Expect(v1beta1nodeclaim.Spec.NodeClassRef.APIVersion).To(Equal("")) + }) + }) + }) + Context("NodeClaim Status", func() { + It("should convert v1 nodeclaim nodename", func() { + v1nodeclaim.Status.NodeName = "test-node-name" + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Status.NodeName).To(Equal(v1beta1nodeclaim.Status.NodeName)) + }) + It("should convert v1 nodeclaim provider id", func() { + v1nodeclaim.Status.ProviderID = "test-provider-id" + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Status.ProviderID).To(Equal(v1beta1nodeclaim.Status.ProviderID)) + }) + It("should convert v1 nodeclaim image id", func() { + v1nodeclaim.Status.ImageID = "test-image-id" + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Status.ImageID).To(Equal(v1beta1nodeclaim.Status.ImageID)) + }) + It("should convert v1 nodeclaim capacity", func() { + v1nodeclaim.Status.Capacity = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("13432"), + v1.ResourceMemory: resource.MustParse("1332G"), + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Status.Capacity).To(Equal(v1beta1nodeclaim.Status.Capacity)) + }) + It("should convert v1 nodeclaim allocatable", func() { + v1nodeclaim.Status.Allocatable = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("13432"), + v1.ResourceMemory: resource.MustParse("1332G"), + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Status.Allocatable).To(Equal(v1beta1nodeclaim.Status.Allocatable)) + }) + It("should convert v1 nodeclaim conditions", func() { + v1nodeclaim.Status.Conditions = []status.Condition{ + { + Status: status.ConditionReady, + Reason: "test-reason", + }, + { + Status: ConditionTypeDrifted, + Reason: "test-reason", + }, + } + Expect(v1nodeclaim.ConvertTo(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1nodeclaim.Status.Conditions { + Expect(v1nodeclaim.Status.Conditions[i].Type).To(Equal(string(v1beta1nodeclaim.Status.Conditions[i].Type))) + Expect(string(v1nodeclaim.Status.Conditions[i].Status)).To(Equal(string(v1beta1nodeclaim.Status.Conditions[i].Status))) + Expect(v1nodeclaim.Status.Conditions[i].Message).To(Equal(v1beta1nodeclaim.Status.Conditions[i].Message)) + Expect(v1nodeclaim.Status.Conditions[i].Reason).To(Equal(v1beta1nodeclaim.Status.Conditions[i].Reason)) + Expect(v1nodeclaim.Status.Conditions[i].LastTransitionTime).To(Equal(v1beta1nodeclaim.Status.Conditions[i].LastTransitionTime.Inner)) + } + }) + }) +}) + +var _ = Describe("Convert V1beta1 to V1 NodeClaim API", func() { + var ( + v1nodePool *NodePool + v1nodeclaim *NodeClaim + v1beta1nodeclaim *v1beta1.NodeClaim + ) + + BeforeEach(func() { + v1nodePool = &NodePool{ + ObjectMeta: test.ObjectMeta(), + Spec: NodePoolSpec{ + Template: NodeClaimTemplate{ + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "default", + }, + Requirements: []NodeSelectorRequirementWithMinValues{}, + }, + }, + }, + } + v1nodePool.Spec.Template.Spec.ExpireAfter = NillableDuration{Duration: lo.ToPtr(30 * time.Minute)} + v1nodeclaim = &NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NodePoolLabelKey: v1nodePool.Name, + }, + }, + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "test", + Kind: "test-kind", + Group: "test-group", + }, + }, + } + v1beta1nodeclaim = &v1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + NodePoolLabelKey: v1nodePool.Name, + }, + }, + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test", + Kind: "test-kind", + APIVersion: "test-group/test-version", + }, + }, + } + Expect(env.Client.Create(ctx, v1nodePool)).To(Succeed()) + + cloudProvider.NodeClassGroupVersionKind = cloudProvider.GetSupportedNodeClasses() + ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses()) + ctx = injection.WithClient(ctx, env.Client) + }) + Context("ExpireAfter", func() { + It("should convert v1beta1 expireAfter to v1", func() { + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(lo.FromPtr(v1nodeclaim.Spec.ExpireAfter.Duration)).To(Equal(30 * time.Minute)) + }) + It("should default the v1beta1 expireAfter to v1 when the nodepool doesn't exist", func() { + Expect(env.Client.Delete(ctx, v1nodePool)).To(Succeed()) + v1nodePool.Spec.Template.Spec.ExpireAfter = NillableDuration{Duration: lo.ToPtr(30 * time.Minute)} + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Spec.ExpireAfter.Duration).To(BeNil()) + }) + It("should default the v1beta1 expireAfter to v1 when the nodepool label doesn't exist", func() { + delete(v1beta1nodeclaim.Labels, v1beta1.NodePoolLabelKey) + v1nodePool.Spec.Template.Spec.ExpireAfter = NillableDuration{Duration: lo.ToPtr(30 * time.Minute)} + Expect(env.Client.Update(ctx, v1nodePool)).To(Succeed()) + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Spec.ExpireAfter.Duration).To(BeNil()) + }) + }) + It("should convert v1beta1 nodeclaim metadata", func() { + v1beta1nodeclaim.ObjectMeta = test.ObjectMeta() + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + v1beta1nodeclaim.Annotations = map[string]string{} + Expect(v1nodeclaim.ObjectMeta).To(BeEquivalentTo(v1beta1nodeclaim.ObjectMeta)) + }) + Context("NodeClaim Spec", func() { + It("should convert v1beta1 nodeclaim taints", func() { + v1beta1nodeclaim.Spec.Taints = []v1.Taint{ + { + Key: "test-key-1", + Value: "test-value-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-2", + Value: "test-value-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1beta1nodeclaim.Spec.Taints { + Expect(v1nodeclaim.Spec.Taints[i].Key).To(Equal(v1beta1nodeclaim.Spec.Taints[i].Key)) + Expect(v1nodeclaim.Spec.Taints[i].Value).To(Equal(v1beta1nodeclaim.Spec.Taints[i].Value)) + Expect(v1nodeclaim.Spec.Taints[i].Effect).To(Equal(v1beta1nodeclaim.Spec.Taints[i].Effect)) + } + }) + It("should convert v1beta1 nodeclaim startup taints", func() { + v1beta1nodeclaim.Spec.StartupTaints = []v1.Taint{ + { + Key: "test-key-startup-1", + Value: "test-value-startup-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-startup-2", + Value: "test-value-startup-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1beta1nodeclaim.Spec.StartupTaints { + Expect(v1nodeclaim.Spec.StartupTaints[i].Key).To(Equal(v1beta1nodeclaim.Spec.StartupTaints[i].Key)) + Expect(v1nodeclaim.Spec.StartupTaints[i].Value).To(Equal(v1beta1nodeclaim.Spec.StartupTaints[i].Value)) + Expect(v1nodeclaim.Spec.StartupTaints[i].Effect).To(Equal(v1beta1nodeclaim.Spec.StartupTaints[i].Effect)) + } + }) + It("should convert v1beta1 nodeclaim requirements", func() { + v1beta1nodeclaim.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(4189133), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{CapacityTypeSpot}, + }, + MinValues: lo.ToPtr(7716191), + }, + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1beta1nodeclaim.Spec.Requirements { + Expect(v1nodeclaim.Spec.Requirements[i].Key).To(Equal(v1beta1nodeclaim.Spec.Requirements[i].Key)) + Expect(v1nodeclaim.Spec.Requirements[i].Operator).To(Equal(v1beta1nodeclaim.Spec.Requirements[i].Operator)) + Expect(v1nodeclaim.Spec.Requirements[i].Values).To(Equal(v1beta1nodeclaim.Spec.Requirements[i].Values)) + Expect(v1nodeclaim.Spec.Requirements[i].MinValues).To(Equal(v1beta1nodeclaim.Spec.Requirements[i].MinValues)) + } + }) + It("should convert v1beta1 nodeclaim resources", func() { + v1beta1nodeclaim.Spec.Resources = v1beta1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("134G"), + }, + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + for key := range v1beta1nodeclaim.Spec.Resources.Requests { + Expect(v1beta1nodeclaim.Spec.Resources.Requests[key]).To(Equal(v1nodeclaim.Spec.Resources.Requests[key])) + } + }) + It("should convert v1 nodeclaim template kubelet", func() { + v1beta1nodeclaim.Spec.Kubelet = &v1beta1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + SystemReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + KubeReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + EvictionHard: map[string]string{"eviction-key": "eviction"}, + EvictionSoft: map[string]string{"eviction-key": "eviction"}, + EvictionSoftGracePeriod: map[string]metav1.Duration{"test-soft-grace": {Duration: time.Hour}}, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(382902)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(382902)), + CPUCFSQuota: lo.ToPtr(false), + } + Expect(v1nodeclaim.Annotations).To(BeNil()) + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + kubelet := &v1beta1.KubeletConfiguration{} + kubeletString, found := v1nodeclaim.Annotations[KubeletCompatibilityAnnotationKey] + Expect(found).To(BeTrue()) + err := json.Unmarshal([]byte(kubeletString), kubelet) + Expect(err).To(BeNil()) + Expect(kubelet.ClusterDNS).To(Equal(v1beta1nodeclaim.Spec.Kubelet.ClusterDNS)) + Expect(lo.FromPtr(kubelet.MaxPods)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.MaxPods))) + Expect(lo.FromPtr(kubelet.PodsPerCore)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.PodsPerCore))) + Expect(lo.FromPtr(kubelet.EvictionMaxPodGracePeriod)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.EvictionMaxPodGracePeriod))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(kubelet.SystemReserved).To(Equal(v1beta1nodeclaim.Spec.Kubelet.SystemReserved)) + Expect(kubelet.KubeReserved).To(Equal(v1beta1nodeclaim.Spec.Kubelet.KubeReserved)) + Expect(kubelet.EvictionHard).To(Equal(v1beta1nodeclaim.Spec.Kubelet.EvictionHard)) + Expect(kubelet.EvictionSoft).To(Equal(v1beta1nodeclaim.Spec.Kubelet.EvictionSoft)) + Expect(kubelet.EvictionSoftGracePeriod).To(Equal(v1beta1nodeclaim.Spec.Kubelet.EvictionSoftGracePeriod)) + Expect(lo.FromPtr(kubelet.CPUCFSQuota)).To(Equal(lo.FromPtr(v1beta1nodeclaim.Spec.Kubelet.CPUCFSQuota))) + }) + Context("NodeClassRef", func() { + It("should convert v1beta1 nodeclaim template nodeClassRef", func() { + v1beta1nodeclaim.Spec.NodeClassRef = &v1beta1.NodeClassReference{ + Kind: "test-kind", + Name: "nodeclass-test", + APIVersion: "testgroup.sh/testversion", + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Spec.NodeClassRef.Kind).To(Equal(v1beta1nodeclaim.Spec.NodeClassRef.Kind)) + Expect(v1nodeclaim.Spec.NodeClassRef.Name).To(Equal(v1beta1nodeclaim.Spec.NodeClassRef.Name)) + Expect(v1nodeclaim.Spec.NodeClassRef.Group).To(Equal("testgroup.sh")) + }) + It("should set default nodeclass group and kind on v1beta1 nodeclassRef", func() { + v1beta1nodeclaim.Spec.NodeClassRef = &v1beta1.NodeClassReference{ + Name: "nodeclass-test", + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1nodeclaim.Spec.NodeClassRef.Kind).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].Kind)) + Expect(v1nodeclaim.Spec.NodeClassRef.Name).To(Equal(v1beta1nodeclaim.Spec.NodeClassRef.Name)) + Expect(v1nodeclaim.Spec.NodeClassRef.Group).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].Group)) + }) + }) + }) + Context("NodeClaim Status", func() { + It("should convert v1beta1 nodeclaim nodename", func() { + v1beta1nodeclaim.Status.NodeName = "test-node-name" + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Status.NodeName).To(Equal(v1nodeclaim.Status.NodeName)) + }) + It("should convert v1beta1 nodeclaim provider id", func() { + v1beta1nodeclaim.Status.ProviderID = "test-provider-id" + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Status.ProviderID).To(Equal(v1nodeclaim.Status.ProviderID)) + }) + It("should convert v1beta1 nodeclaim image id", func() { + v1beta1nodeclaim.Status.ImageID = "test-image-id" + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Status.ImageID).To(Equal(v1nodeclaim.Status.ImageID)) + }) + It("should convert v1beta1 nodeclaim capacity", func() { + v1beta1nodeclaim.Status.Capacity = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("13432"), + v1.ResourceMemory: resource.MustParse("1332G"), + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Status.Capacity).To(Equal(v1nodeclaim.Status.Capacity)) + }) + It("should convert v1beta1 nodeclaim allocatable", func() { + v1beta1nodeclaim.Status.Allocatable = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("13432"), + v1.ResourceMemory: resource.MustParse("1332G"), + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + Expect(v1beta1nodeclaim.Status.Allocatable).To(Equal(v1nodeclaim.Status.Allocatable)) + }) + It("should convert v1beta1 nodeclaim conditions", func() { + v1beta1nodeclaim.Status.Conditions = []apis.Condition{ + { + Status: status.ConditionReady, + Reason: "test-reason", + }, + { + Status: ConditionTypeDrifted, + Reason: "test-reason", + }, + } + Expect(v1nodeclaim.ConvertFrom(ctx, v1beta1nodeclaim)).To(Succeed()) + for i := range v1beta1nodeclaim.Status.Conditions { + Expect(string(v1beta1nodeclaim.Status.Conditions[i].Type)).To(Equal(v1nodeclaim.Status.Conditions[i].Type)) + Expect(string(v1beta1nodeclaim.Status.Conditions[i].Status)).To(Equal(string(v1nodeclaim.Status.Conditions[i].Status))) + Expect(v1beta1nodeclaim.Status.Conditions[i].Message).To(Equal(v1nodeclaim.Status.Conditions[i].Message)) + Expect(v1beta1nodeclaim.Status.Conditions[i].Reason).To(Equal(v1nodeclaim.Status.Conditions[i].Reason)) + Expect(v1beta1nodeclaim.Status.Conditions[i].LastTransitionTime.Inner).To(Equal(v1nodeclaim.Status.Conditions[i].LastTransitionTime)) + } + }) + }) +}) diff --git a/pkg/apis/v1/nodepool_conversion.go b/pkg/apis/v1/nodepool_conversion.go new file mode 100644 index 0000000000..8d941d445b --- /dev/null +++ b/pkg/apis/v1/nodepool_conversion.go @@ -0,0 +1,202 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "knative.dev/pkg/apis" + + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" +) + +// Convert v1 NodePool to v1beta1 NodePool +func (in *NodePool) ConvertTo(ctx context.Context, to apis.Convertible) error { + v1beta1NP := to.(*v1beta1.NodePool) + v1beta1NP.ObjectMeta = in.ObjectMeta + + // Convert v1 status + v1beta1NP.Status.Resources = in.Status.Resources + return in.Spec.convertTo(ctx, &v1beta1NP.Spec, in.Annotations[KubeletCompatibilityAnnotationKey]) +} + +func (in *NodePoolSpec) convertTo(ctx context.Context, v1beta1np *v1beta1.NodePoolSpec, kubeletAnnotation string) error { + v1beta1np.Weight = in.Weight + v1beta1np.Limits = v1beta1.Limits(in.Limits) + in.Disruption.convertTo(&v1beta1np.Disruption) + // Set the expireAfter to the nodeclaim template's expireAfter. + // Don't convert terminationGracePeriod, as this is only included in v1. + v1beta1np.Disruption.ExpireAfter = v1beta1.NillableDuration(in.Template.Spec.ExpireAfter) + return in.Template.convertTo(ctx, &v1beta1np.Template, kubeletAnnotation) +} + +func (in *Disruption) convertTo(v1beta1np *v1beta1.Disruption) { + v1beta1np.ConsolidationPolicy = v1beta1.ConsolidationPolicy(in.ConsolidationPolicy) + // If the v1 nodepool is WhenUnderutilized, the v1beta1 nodepool should have an unset consolidateAfter + v1beta1np.ConsolidateAfter = lo.Ternary(in.ConsolidationPolicy == ConsolidationPolicyWhenUnderutilized, + nil, (*v1beta1.NillableDuration)(lo.ToPtr(in.ConsolidateAfter))) + v1beta1np.Budgets = lo.Map(in.Budgets, func(v1budget Budget, _ int) v1beta1.Budget { + return v1beta1.Budget{ + Nodes: v1budget.Nodes, + Schedule: v1budget.Schedule, + Duration: v1budget.Duration, + } + }) +} + +func (in *NodeClaimTemplate) convertTo(ctx context.Context, v1beta1np *v1beta1.NodeClaimTemplate, kubeletAnnotation string) error { + v1beta1np.ObjectMeta = v1beta1.ObjectMeta(in.ObjectMeta) + v1beta1np.Spec.Taints = in.Spec.Taints + v1beta1np.Spec.StartupTaints = in.Spec.StartupTaints + v1beta1np.Spec.Requirements = lo.Map(in.Spec.Requirements, func(v1Requirements NodeSelectorRequirementWithMinValues, _ int) v1beta1.NodeSelectorRequirementWithMinValues { + return v1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1Requirements.Key, + Values: v1Requirements.Values, + Operator: v1Requirements.Operator, + }, + MinValues: v1Requirements.MinValues, + } + }) + + nodeClasses := injection.GetNodeClasses(ctx) + // We are sorting the supported nodeclass, so that we are able to consistently find the same GVK, + // if multiple version of a nodeclass are supported + sort.Slice(nodeClasses, func(i int, j int) bool { + if nodeClasses[i].Group != nodeClasses[j].Group { + return nodeClasses[i].Group < nodeClasses[j].Group + } + if nodeClasses[i].Version != nodeClasses[j].Version { + return nodeClasses[i].Version < nodeClasses[j].Version + } + return nodeClasses[i].Kind < nodeClasses[j].Kind + }) + matchingNodeClass, found := lo.Find(nodeClasses, func(nc schema.GroupVersionKind) bool { + return nc.Kind == in.Spec.NodeClassRef.Kind && nc.Group == in.Spec.NodeClassRef.Group + }) + v1beta1np.Spec.NodeClassRef = &v1beta1.NodeClassReference{ + Kind: in.Spec.NodeClassRef.Kind, + Name: in.Spec.NodeClassRef.Name, + APIVersion: lo.Ternary(found, matchingNodeClass.GroupVersion().String(), ""), + } + + if kubeletAnnotation != "" { + v1beta1kubelet := &v1beta1.KubeletConfiguration{} + err := json.Unmarshal([]byte(kubeletAnnotation), v1beta1kubelet) + if err != nil { + return fmt.Errorf("unmarshaling kubelet config annotation, %w", err) + + } + v1beta1np.Spec.Kubelet = v1beta1kubelet + } + return nil +} + +// Convert v1beta1 NodePool to V1 NodePool +func (in *NodePool) ConvertFrom(ctx context.Context, v1beta1np apis.Convertible) error { + v1beta1NP := v1beta1np.(*v1beta1.NodePool) + in.ObjectMeta = v1beta1NP.ObjectMeta + + // Convert v1beta1 status + in.Status.Resources = v1beta1NP.Status.Resources + + kubeletAnnotation, err := in.Spec.convertFrom(ctx, &v1beta1NP.Spec) + if err != nil { + return err + } + if kubeletAnnotation == "" { + in.Annotations = lo.OmitByKeys(in.Annotations, []string{KubeletCompatibilityAnnotationKey}) + } else { + in.Annotations = lo.Assign(in.Annotations, map[string]string{KubeletCompatibilityAnnotationKey: kubeletAnnotation}) + } + return nil +} + +func (in *NodePoolSpec) convertFrom(ctx context.Context, v1beta1np *v1beta1.NodePoolSpec) (string, error) { + in.Weight = v1beta1np.Weight + in.Limits = Limits(v1beta1np.Limits) + in.Template.Spec.ExpireAfter = NillableDuration(v1beta1np.Disruption.ExpireAfter) + in.Disruption.convertFrom(&v1beta1np.Disruption) + return in.Template.convertFrom(ctx, &v1beta1np.Template) +} + +func (in *Disruption) convertFrom(v1beta1np *v1beta1.Disruption) { + // if consolidationPolicy is WhenUnderutilized, set the v1 duration to 0, otherwise, set to the value of consolidateAfter. + in.ConsolidateAfter = lo.Ternary(v1beta1np.ConsolidationPolicy == v1beta1.ConsolidationPolicyWhenUnderutilized, + NillableDuration{Duration: lo.ToPtr(time.Duration(0))}, (NillableDuration)(lo.FromPtr(v1beta1np.ConsolidateAfter))) + in.ConsolidationPolicy = ConsolidationPolicy(v1beta1np.ConsolidationPolicy) + in.Budgets = lo.Map(v1beta1np.Budgets, func(v1beta1budget v1beta1.Budget, _ int) Budget { + return Budget{ + Nodes: v1beta1budget.Nodes, + Schedule: v1beta1budget.Schedule, + Duration: v1beta1budget.Duration, + } + }) +} + +func (in *NodeClaimTemplate) convertFrom(ctx context.Context, v1beta1np *v1beta1.NodeClaimTemplate) (string, error) { + in.ObjectMeta = ObjectMeta(v1beta1np.ObjectMeta) + in.Spec.Taints = v1beta1np.Spec.Taints + in.Spec.StartupTaints = v1beta1np.Spec.StartupTaints + in.Spec.Requirements = lo.Map(v1beta1np.Spec.Requirements, func(v1beta1Requirements v1beta1.NodeSelectorRequirementWithMinValues, _ int) NodeSelectorRequirementWithMinValues { + return NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1Requirements.Key, + Values: v1beta1Requirements.Values, + Operator: v1beta1Requirements.Operator, + }, + MinValues: v1beta1Requirements.MinValues, + } + }) + + nodeclasses := injection.GetNodeClasses(ctx) + in.Spec.NodeClassRef = &NodeClassReference{ + Name: v1beta1np.Spec.NodeClassRef.Name, + Kind: lo.Ternary(v1beta1np.Spec.NodeClassRef.Kind == "", nodeclasses[0].Kind, v1beta1np.Spec.NodeClassRef.Kind), + Group: lo.Ternary(v1beta1np.Spec.NodeClassRef.APIVersion == "", nodeclasses[0].Group, strings.Split(v1beta1np.Spec.NodeClassRef.APIVersion, "/")[0]), + } + + defaultNodeClassGVK := injection.GetNodeClasses(ctx)[0] + nodeclassGroupVersion, err := schema.ParseGroupVersion(v1beta1np.Spec.NodeClassRef.APIVersion) + if err != nil { + return "", err + } + in.Spec.NodeClassRef = &NodeClassReference{ + Name: v1beta1np.Spec.NodeClassRef.Name, + Kind: lo.Ternary(v1beta1np.Spec.NodeClassRef.Kind == "", defaultNodeClassGVK.Kind, v1beta1np.Spec.NodeClassRef.Kind), + Group: lo.Ternary(v1beta1np.Spec.NodeClassRef.APIVersion == "", defaultNodeClassGVK.Group, nodeclassGroupVersion.Group), + } + + if v1beta1np.Spec.Kubelet != nil { + kubelet, err := json.Marshal(v1beta1np.Spec.Kubelet) + if err != nil { + return "", fmt.Errorf("marshaling kubelet config annotation, %w", err) + } + return string(kubelet), nil + } + + return "", nil +} diff --git a/pkg/apis/v1/nodepool_conversion_test.go b/pkg/apis/v1/nodepool_conversion_test.go new file mode 100644 index 0000000000..e5e125c131 --- /dev/null +++ b/pkg/apis/v1/nodepool_conversion_test.go @@ -0,0 +1,536 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1_test + +import ( + "encoding/json" + "time" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + "github.com/awslabs/operatorpkg/object" + + "sigs.k8s.io/karpenter/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" +) + +var _ = Describe("Convert V1 to V1beta1 NodePool API", func() { + var v1nodepool *NodePool + var v1beta1nodepool *v1beta1.NodePool + + BeforeEach(func() { + v1nodepool = &NodePool{ + Spec: NodePoolSpec{ + Template: NodeClaimTemplate{ + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "test", + Kind: "test", + Group: "test", + }, + }, + }, + }, + } + v1beta1nodepool = &v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test", + Kind: "test", + APIVersion: "group/test", + }, + }, + }, + }, + } + cloudProvider.NodeClassGroupVersionKind = cloudProvider.GetSupportedNodeClasses() + ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses()) + }) + + It("should convert v1 nodepool metadata", func() { + v1nodepool.ObjectMeta = test.ObjectMeta() + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.ObjectMeta).To(BeEquivalentTo(v1nodepool.ObjectMeta)) + }) + Context("NodePool Spec", func() { + It("should convert v1 nodepool weights", func() { + v1nodepool.Spec.Weight = lo.ToPtr(int32(62)) + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(lo.FromPtr(v1beta1nodepool.Spec.Weight)).To(Equal(int32(62))) + }) + It("should convert v1 nodepool limits", func() { + v1nodepool.Spec.Limits = Limits{ + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("14145G"), + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for _, resource := range lo.Keys(v1nodepool.Spec.Limits) { + Expect(v1beta1nodepool.Spec.Limits[resource]).To(Equal(v1nodepool.Spec.Limits[resource])) + } + }) + Context("NodeClaimTemplate", func() { + It("should convert v1 nodepool metadata", func() { + v1nodepool.Spec.Template.ObjectMeta = ObjectMeta{ + Labels: map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + }, + Annotations: map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + }, + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.Spec.Template.ObjectMeta).To(BeEquivalentTo(v1nodepool.Spec.Template.ObjectMeta)) + }) + It("should convert v1 nodepool template taints", func() { + v1nodepool.Spec.Template.Spec.Taints = []v1.Taint{ + { + Key: "test-key-1", + Value: "test-value-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-2", + Value: "test-value-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Template.Spec.Taints { + Expect(v1beta1nodepool.Spec.Template.Spec.Taints[i].Key).To(Equal(v1nodepool.Spec.Template.Spec.Taints[i].Key)) + Expect(v1beta1nodepool.Spec.Template.Spec.Taints[i].Value).To(Equal(v1nodepool.Spec.Template.Spec.Taints[i].Value)) + Expect(v1beta1nodepool.Spec.Template.Spec.Taints[i].Effect).To(Equal(v1nodepool.Spec.Template.Spec.Taints[i].Effect)) + } + }) + It("should convert v1 nodepool template startup taints", func() { + v1nodepool.Spec.Template.Spec.StartupTaints = []v1.Taint{ + { + Key: "test-key-startup-1", + Value: "test-value-startup-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-startup-2", + Value: "test-value-startup-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Template.Spec.StartupTaints { + Expect(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Key).To(Equal(v1nodepool.Spec.Template.Spec.StartupTaints[i].Key)) + Expect(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Value).To(Equal(v1nodepool.Spec.Template.Spec.StartupTaints[i].Value)) + Expect(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Effect).To(Equal(v1nodepool.Spec.Template.Spec.StartupTaints[i].Effect)) + } + }) + It("should convert v1 nodepool template requirements", func() { + v1nodepool.Spec.Template.Spec.Requirements = []NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(433234), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{CapacityTypeSpot}, + }, + MinValues: lo.ToPtr(65765), + }, + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Template.Spec.Requirements { + Expect(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Key).To(Equal(v1nodepool.Spec.Template.Spec.Requirements[i].Key)) + Expect(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Operator).To(Equal(v1nodepool.Spec.Template.Spec.Requirements[i].Operator)) + Expect(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Values).To(Equal(v1nodepool.Spec.Template.Spec.Requirements[i].Values)) + Expect(v1beta1nodepool.Spec.Template.Spec.Requirements[i].MinValues).To(Equal(v1nodepool.Spec.Template.Spec.Requirements[i].MinValues)) + } + }) + Context("NodeClassRef", func() { + It("should convert v1 nodepool template nodeClassRef", func() { + v1nodepool.Spec.Template.Spec.NodeClassRef = &NodeClassReference{ + Kind: object.GVK(&v1alpha1.TestNodeClass{}).Kind, + Name: "nodeclass-test", + Group: object.GVK(&v1alpha1.TestNodeClass{}).Group, + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(v1nodepool.Spec.Template.Spec.NodeClassRef.Kind)) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Name).To(Equal(v1nodepool.Spec.Template.Spec.NodeClassRef.Name)) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.APIVersion).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].GroupVersion().String())) + }) + It("should not include APIVersion for v1beta1 if Group and Kind is not in the supported nodeclass", func() { + v1nodepool.Spec.Template.Spec.NodeClassRef = &NodeClassReference{ + Kind: "test-kind", + Name: "nodeclass-test", + Group: "testgroup.sh", + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(v1nodepool.Spec.Template.Spec.NodeClassRef.Kind)) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Name).To(Equal(v1nodepool.Spec.Template.Spec.NodeClassRef.Name)) + Expect(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.APIVersion).To(Equal("")) + }) + }) + }) + Context("Disruption", func() { + It("should convert v1 nodepool consolidateAfter to nil with WhenUnderutilized", func() { + v1nodepool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenUnderutilized + v1nodepool.Spec.Disruption.ConsolidateAfter = NillableDuration{Duration: lo.ToPtr(time.Second * 2121)} + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.Spec.Disruption.ConsolidateAfter).To(BeNil()) + }) + It("should convert v1 nodepool consolidateAfter with WhenEmpty", func() { + v1nodepool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmpty + v1nodepool.Spec.Disruption.ConsolidateAfter = NillableDuration{Duration: lo.ToPtr(time.Second * 2121)} + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(lo.FromPtr(v1beta1nodepool.Spec.Disruption.ConsolidateAfter.Duration)).To(Equal(lo.FromPtr(v1nodepool.Spec.Disruption.ConsolidateAfter.Duration))) + }) + It("should convert v1 nodepool consolidatePolicy", func() { + v1nodepool.Spec.Disruption.ConsolidationPolicy = ConsolidationPolicyWhenEmpty + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(string(v1beta1nodepool.Spec.Disruption.ConsolidationPolicy)).To(Equal(string(v1nodepool.Spec.Disruption.ConsolidationPolicy))) + }) + It("should convert v1 nodepool ExpireAfter", func() { + v1nodepool.Spec.Template.Spec.ExpireAfter = NillableDuration{Duration: lo.ToPtr(time.Second * 2121)} + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1beta1nodepool.Spec.Disruption.ExpireAfter.Duration).To(Equal(v1nodepool.Spec.Template.Spec.ExpireAfter.Duration)) + }) + Context("Budgets", func() { + It("should convert v1 nodepool nodes", func() { + v1nodepool.Spec.Disruption.Budgets = append(v1nodepool.Spec.Disruption.Budgets, Budget{ + Nodes: "1545", + }) + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Disruption.Budgets { + Expect(v1beta1nodepool.Spec.Disruption.Budgets[i].Nodes).To(Equal(v1nodepool.Spec.Disruption.Budgets[i].Nodes)) + } + }) + It("should convert v1 nodepool schedule", func() { + v1nodepool.Spec.Disruption.Budgets = append(v1nodepool.Spec.Disruption.Budgets, Budget{ + Schedule: lo.ToPtr("1545"), + }) + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Disruption.Budgets { + Expect(v1beta1nodepool.Spec.Disruption.Budgets[i].Schedule).To(Equal(v1nodepool.Spec.Disruption.Budgets[i].Schedule)) + } + }) + It("should convert v1 nodepool duration", func() { + v1nodepool.Spec.Disruption.Budgets = append(v1nodepool.Spec.Disruption.Budgets, Budget{ + Duration: &metav1.Duration{Duration: time.Second * 2121}, + }) + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1nodepool.Spec.Disruption.Budgets { + Expect(v1beta1nodepool.Spec.Disruption.Budgets[i].Duration.Duration).To(Equal(v1nodepool.Spec.Disruption.Budgets[i].Duration.Duration)) + } + }) + }) + }) + }) + It("should convert v1 nodepool status", func() { + v1nodepool.Status.Resources = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("14145G"), + } + Expect(v1nodepool.ConvertTo(ctx, v1beta1nodepool)).To(Succeed()) + for _, resource := range lo.Keys(v1nodepool.Status.Resources) { + Expect(v1beta1nodepool.Status.Resources[resource]).To(Equal(v1nodepool.Status.Resources[resource])) + } + }) +}) + +var _ = Describe("Convert V1beta1 to V1 NodePool API", func() { + var ( + v1nodepool *NodePool + v1beta1nodepool *v1beta1.NodePool + ) + + BeforeEach(func() { + v1nodepool = &NodePool{ + Spec: NodePoolSpec{ + Template: NodeClaimTemplate{ + Spec: NodeClaimSpec{ + NodeClassRef: &NodeClassReference{ + Name: "test", + Kind: "test", + Group: "test", + }, + }, + }, + }, + } + v1beta1nodepool = &v1beta1.NodePool{ + Spec: v1beta1.NodePoolSpec{ + Template: v1beta1.NodeClaimTemplate{ + Spec: v1beta1.NodeClaimSpec{ + NodeClassRef: &v1beta1.NodeClassReference{ + Name: "test", + Kind: "test", + APIVersion: "group/test", + }, + }, + }, + }, + } + cloudProvider.NodeClassGroupVersionKind = cloudProvider.GetSupportedNodeClasses() + ctx = injection.WithNodeClasses(ctx, cloudProvider.GetSupportedNodeClasses()) + }) + + It("should convert v1beta1 nodepool metadata", func() { + v1beta1nodepool.ObjectMeta = test.ObjectMeta() + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + v1nodepool.Annotations = nil + Expect(v1nodepool.ObjectMeta).To(BeEquivalentTo(v1beta1nodepool.ObjectMeta)) + }) + Context("NodePool Spec", func() { + It("should convert v1beta1 nodepool weights", func() { + v1beta1nodepool.Spec.Weight = lo.ToPtr(int32(62)) + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Weight).To(Equal(v1beta1nodepool.Spec.Weight)) + }) + It("should convert v1beta1 nodepool limits", func() { + v1beta1nodepool.Spec.Limits = v1beta1.Limits{ + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("14145G"), + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for _, resource := range lo.Keys(v1beta1nodepool.Spec.Limits) { + Expect(v1nodepool.Spec.Limits[resource]).To(Equal(v1beta1nodepool.Spec.Limits[resource])) + } + }) + Context("NodeClaimTemplate", func() { + It("should convert v1beta1 nodepool metadata", func() { + v1beta1nodepool.Spec.Template.ObjectMeta = v1beta1.ObjectMeta{ + Labels: map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + }, + Annotations: map[string]string{ + "test-key-1": "test-value-1", + "test-key-2": "test-value-2", + }, + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Template.ObjectMeta).To(BeEquivalentTo(v1beta1nodepool.Spec.Template.ObjectMeta)) + }) + It("should convert v1beta1 nodepool template taints", func() { + v1beta1nodepool.Spec.Template.Spec.Taints = []v1.Taint{ + { + Key: "test-key-1", + Value: "test-value-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-2", + Value: "test-value-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Template.Spec.Taints { + Expect(v1nodepool.Spec.Template.Spec.Taints[i].Key).To(Equal(v1beta1nodepool.Spec.Template.Spec.Taints[i].Key)) + Expect(v1nodepool.Spec.Template.Spec.Taints[i].Value).To(Equal(v1beta1nodepool.Spec.Template.Spec.Taints[i].Value)) + Expect(v1nodepool.Spec.Template.Spec.Taints[i].Effect).To(Equal(v1beta1nodepool.Spec.Template.Spec.Taints[i].Effect)) + } + }) + It("should convert v1beta1 nodepool template startup taints", func() { + v1beta1nodepool.Spec.Template.Spec.StartupTaints = []v1.Taint{ + { + Key: "test-key-startup-1", + Value: "test-value-startup-1", + Effect: v1.TaintEffectNoExecute, + }, + { + Key: "test-key-startup-2", + Value: "test-value-startup-2", + Effect: v1.TaintEffectNoSchedule, + }, + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Template.Spec.StartupTaints { + Expect(v1nodepool.Spec.Template.Spec.StartupTaints[i].Key).To(Equal(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Key)) + Expect(v1nodepool.Spec.Template.Spec.StartupTaints[i].Value).To(Equal(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Value)) + Expect(v1nodepool.Spec.Template.Spec.StartupTaints[i].Effect).To(Equal(v1beta1nodepool.Spec.Template.Spec.StartupTaints[i].Effect)) + } + }) + It("should convert v1beta1 nodepool template requirements", func() { + v1beta1nodepool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(98946513), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{CapacityTypeSpot}, + }, + MinValues: lo.ToPtr(513164), + }, + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Template.Spec.Requirements { + Expect(v1nodepool.Spec.Template.Spec.Requirements[i].Key).To(Equal(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Key)) + Expect(v1nodepool.Spec.Template.Spec.Requirements[i].Operator).To(Equal(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Operator)) + Expect(v1nodepool.Spec.Template.Spec.Requirements[i].Values).To(Equal(v1beta1nodepool.Spec.Template.Spec.Requirements[i].Values)) + Expect(v1nodepool.Spec.Template.Spec.Requirements[i].MinValues).To(Equal(v1beta1nodepool.Spec.Template.Spec.Requirements[i].MinValues)) + } + }) + It("should convert v1 nodepool template kubelet", func() { + v1beta1nodepool.Spec.Template.Spec.Kubelet = &v1beta1.KubeletConfiguration{ + ClusterDNS: []string{"test-cluster-dns"}, + MaxPods: lo.ToPtr(int32(9383)), + PodsPerCore: lo.ToPtr(int32(9334283)), + SystemReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + KubeReserved: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + EvictionHard: map[string]string{"eviction-key": "eviction"}, + EvictionSoft: map[string]string{"eviction-key": "eviction"}, + EvictionSoftGracePeriod: map[string]metav1.Duration{"test-soft-grace": {Duration: time.Hour}}, + EvictionMaxPodGracePeriod: lo.ToPtr(int32(382902)), + ImageGCHighThresholdPercent: lo.ToPtr(int32(382902)), + CPUCFSQuota: lo.ToPtr(false), + } + Expect(v1nodepool.Annotations).To(BeNil()) + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + kubelet := &v1beta1.KubeletConfiguration{} + kubeletString, found := v1nodepool.Annotations[KubeletCompatibilityAnnotationKey] + Expect(found).To(BeTrue()) + err := json.Unmarshal([]byte(kubeletString), kubelet) + Expect(err).To(BeNil()) + Expect(kubelet.ClusterDNS).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.ClusterDNS)) + Expect(lo.FromPtr(kubelet.MaxPods)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.MaxPods))) + Expect(lo.FromPtr(kubelet.PodsPerCore)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.PodsPerCore))) + Expect(lo.FromPtr(kubelet.EvictionMaxPodGracePeriod)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.EvictionMaxPodGracePeriod))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(lo.FromPtr(kubelet.ImageGCHighThresholdPercent)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.ImageGCHighThresholdPercent))) + Expect(kubelet.SystemReserved).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.SystemReserved)) + Expect(kubelet.KubeReserved).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.KubeReserved)) + Expect(kubelet.EvictionHard).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.EvictionHard)) + Expect(kubelet.EvictionSoft).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.EvictionSoft)) + Expect(kubelet.EvictionSoftGracePeriod).To(Equal(v1beta1nodepool.Spec.Template.Spec.Kubelet.EvictionSoftGracePeriod)) + Expect(lo.FromPtr(kubelet.CPUCFSQuota)).To(Equal(lo.FromPtr(v1beta1nodepool.Spec.Template.Spec.Kubelet.CPUCFSQuota))) + }) + Context("NodeClassRef", func() { + It("should convert v1beta1 nodepool template nodeClassRef", func() { + v1beta1nodepool.Spec.Template.Spec.NodeClassRef = &v1beta1.NodeClassReference{ + Kind: "test-kind", + Name: "nodeclass-test", + APIVersion: "testgroup.sh/testversion", + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Kind)) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Name).To(Equal(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Name)) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Group).To(Equal("testgroup.sh")) + }) + It("should set default nodeclass group and kind on v1beta1 nodeclassRef", func() { + v1beta1nodepool.Spec.Template.Spec.NodeClassRef = &v1beta1.NodeClassReference{ + Name: "nodeclass-test", + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Kind).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].Kind)) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Name).To(Equal(v1beta1nodepool.Spec.Template.Spec.NodeClassRef.Name)) + Expect(v1nodepool.Spec.Template.Spec.NodeClassRef.Group).To(Equal(cloudProvider.NodeClassGroupVersionKind[0].Group)) + }) + }) + }) + Context("Disruption", func() { + It("should convert v1beta1 nodepool consolidateAfter to 0 for WhenUnderutilized", func() { + v1beta1nodepool.Spec.Disruption.ConsolidationPolicy = v1beta1.ConsolidationPolicyWhenUnderutilized + v1beta1nodepool.Spec.Disruption.ConsolidateAfter = nil + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(lo.FromPtr(v1nodepool.Spec.Disruption.ConsolidateAfter.Duration)).To(BeEquivalentTo(0)) + }) + It("should convert v1beta1 nodepool consolidateAfter for WhenEmpty", func() { + v1beta1nodepool.Spec.Disruption.ConsolidationPolicy = v1beta1.ConsolidationPolicyWhenEmpty + v1beta1nodepool.Spec.Disruption.ConsolidateAfter = &v1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 2121)} + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Disruption.ConsolidateAfter.Duration).To(Equal(v1beta1nodepool.Spec.Disruption.ConsolidateAfter.Duration)) + }) + It("should convert v1beta1 nodepool consolidatePolicy", func() { + v1beta1nodepool.Spec.Disruption.ConsolidationPolicy = v1beta1.ConsolidationPolicyWhenEmpty + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(string(v1nodepool.Spec.Disruption.ConsolidationPolicy)).To(Equal(string(v1beta1nodepool.Spec.Disruption.ConsolidationPolicy))) + }) + It("should convert v1beta1 nodepool ExpireAfter", func() { + v1beta1nodepool.Spec.Disruption.ExpireAfter = v1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 2121)} + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + Expect(v1nodepool.Spec.Template.Spec.ExpireAfter.Duration).To(Equal(v1beta1nodepool.Spec.Disruption.ExpireAfter.Duration)) + }) + Context("Budgets", func() { + It("should convert v1beta1 nodepool nodes", func() { + v1beta1nodepool.Spec.Disruption.Budgets = append(v1beta1nodepool.Spec.Disruption.Budgets, v1beta1.Budget{ + Nodes: "1545", + }) + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Disruption.Budgets { + Expect(v1nodepool.Spec.Disruption.Budgets[i].Nodes).To(Equal(v1beta1nodepool.Spec.Disruption.Budgets[i].Nodes)) + } + }) + It("should convert v1beta1 nodepool schedule", func() { + v1beta1nodepool.Spec.Disruption.Budgets = append(v1beta1nodepool.Spec.Disruption.Budgets, v1beta1.Budget{ + Schedule: lo.ToPtr("1545"), + }) + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Disruption.Budgets { + Expect(v1nodepool.Spec.Disruption.Budgets[i].Schedule).To(Equal(v1beta1nodepool.Spec.Disruption.Budgets[i].Schedule)) + } + }) + It("should convert v1beta1 nodepool duration", func() { + v1beta1nodepool.Spec.Disruption.Budgets = append(v1beta1nodepool.Spec.Disruption.Budgets, v1beta1.Budget{ + Duration: &metav1.Duration{Duration: time.Second * 2121}, + }) + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for i := range v1beta1nodepool.Spec.Disruption.Budgets { + Expect(v1nodepool.Spec.Disruption.Budgets[i].Duration.Duration).To(Equal(v1beta1nodepool.Spec.Disruption.Budgets[i].Duration.Duration)) + } + }) + }) + }) + }) + It("should convert v1beta1 nodepool status", func() { + v1beta1nodepool.Status.Resources = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("5"), + v1.ResourceMemory: resource.MustParse("14145G"), + } + Expect(v1nodepool.ConvertFrom(ctx, v1beta1nodepool)).To(Succeed()) + for _, resource := range lo.Keys(v1beta1nodepool.Status.Resources) { + Expect(v1beta1nodepool.Status.Resources[resource]).To(Equal(v1nodepool.Status.Resources[resource])) + } + }) +}) diff --git a/pkg/apis/v1beta1/nodeclaim_conversion.go b/pkg/apis/v1beta1/nodeclaim_conversion.go new file mode 100644 index 0000000000..d69c557055 --- /dev/null +++ b/pkg/apis/v1beta1/nodeclaim_conversion.go @@ -0,0 +1,29 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + + "knative.dev/pkg/apis" +) + +// Since v1 is the hub conversion version, We will only need to implement conversion webhooks for v1 + +func (in *NodeClaim) ConvertTo(_ context.Context, _ apis.Convertible) error { return nil } + +func (in *NodeClaim) ConvertFrom(_ context.Context, _ apis.Convertible) error { return nil } diff --git a/pkg/apis/v1beta1/nodepool_conversion.go b/pkg/apis/v1beta1/nodepool_conversion.go new file mode 100644 index 0000000000..ed50b62f56 --- /dev/null +++ b/pkg/apis/v1beta1/nodepool_conversion.go @@ -0,0 +1,29 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + + "knative.dev/pkg/apis" +) + +// Since v1 is the hub conversion version, We will only need to implement conversion webhooks for v1 + +func (in *NodePool) ConvertTo(_ context.Context, _ apis.Convertible) error { return nil } + +func (in *NodePool) ConvertFrom(_ context.Context, _ apis.Convertible) error { return nil } diff --git a/pkg/cloudprovider/fake/cloudprovider.go b/pkg/cloudprovider/fake/cloudprovider.go index 9f81588317..646edebc05 100644 --- a/pkg/cloudprovider/fake/cloudprovider.go +++ b/pkg/cloudprovider/fake/cloudprovider.go @@ -23,16 +23,19 @@ import ( "sort" "sync" + "github.com/awslabs/operatorpkg/object" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/test" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" "sigs.k8s.io/karpenter/pkg/utils/functional" "sigs.k8s.io/karpenter/pkg/utils/resources" ) @@ -51,8 +54,9 @@ type CloudProvider struct { NextCreateErr error DeleteCalls []*v1beta1.NodeClaim - CreatedNodeClaims map[string]*v1beta1.NodeClaim - Drifted cloudprovider.DriftReason + CreatedNodeClaims map[string]*v1beta1.NodeClaim + Drifted cloudprovider.DriftReason + NodeClassGroupVersionKind []schema.GroupVersionKind } func NewCloudProvider() *CloudProvider { @@ -77,6 +81,13 @@ func (c *CloudProvider) Reset() { c.NextCreateErr = nil c.DeleteCalls = []*v1beta1.NodeClaim{} c.Drifted = "drifted" + c.NodeClassGroupVersionKind = []schema.GroupVersionKind{ + { + Group: "", + Version: "", + Kind: "", + }, + } } func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (*v1beta1.NodeClaim, error) { @@ -237,3 +248,13 @@ func (c *CloudProvider) IsDrifted(context.Context, *v1beta1.NodeClaim) (cloudpro func (c *CloudProvider) Name() string { return "fake" } + +func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + { + Group: object.GVK(&v1alpha1.TestNodeClass{}).Group, + Version: object.GVK(&v1alpha1.TestNodeClass{}).Version, + Kind: object.GVK(&v1alpha1.TestNodeClass{}).Kind, + }, + } +} diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index dde363ca32..cdd37f8b50 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -26,6 +26,7 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -55,6 +56,9 @@ type CloudProvider interface { IsDrifted(context.Context, *v1beta1.NodeClaim) (DriftReason, error) // Name returns the CloudProvider implementation name. Name() string + // GetSupportedNodeClasses returns CloudProvider NodeClass that implements status.Object + // NOTE: It returns a list where the first element should be the default NodeClass + GetSupportedNodeClasses() []schema.GroupVersionKind } // InstanceType describes the properties of a potential node (either concrete attributes of an instance of this type diff --git a/pkg/operator/injection/injection.go b/pkg/operator/injection/injection.go index 43eec1e1b6..316ad5bccb 100644 --- a/pkg/operator/injection/injection.go +++ b/pkg/operator/injection/injection.go @@ -22,13 +22,19 @@ import ( "os" "github.com/samber/lo" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/operator/options" ) type controllerNameKeyType struct{} +type clientKeyType struct{} +type nodeClassType struct{} var controllerNameKey = controllerNameKeyType{} +var clientKey = clientKeyType{} +var nodeClassKey = nodeClassType{} func WithControllerName(ctx context.Context, name string) context.Context { return context.WithValue(ctx, controllerNameKey, name) @@ -57,3 +63,21 @@ func WithOptionsOrDie(ctx context.Context, opts ...options.Injectable) context.C } return ctx } + +func WithClient(ctx context.Context, client client.Client) context.Context { + return context.WithValue(ctx, clientKey, client) +} + +func GetClient(ctx context.Context) client.Client { + c := ctx.Value(clientKey) + return c.(client.Client) +} + +func WithNodeClasses(ctx context.Context, opts []schema.GroupVersionKind) context.Context { + return context.WithValue(ctx, nodeClassKey, opts) +} + +func GetNodeClasses(ctx context.Context) []schema.GroupVersionKind { + retval := ctx.Value(nodeClassKey) + return retval.([]schema.GroupVersionKind) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 64a1320c68..82a600d676 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -31,6 +31,7 @@ import ( "knative.dev/pkg/changeset" crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/metrics" "github.com/go-logr/zapr" @@ -191,9 +192,6 @@ func NewOperator() (context.Context, *Operator) { return []string{o.(*v1beta1.NodeClaim).Status.ProviderID} }), "failed to setup nodeclaim provider id indexer") - lo.Must0(mgr.AddReadyzCheck("manager", func(req *http.Request) error { - return lo.Ternary(mgr.GetCache().WaitForCacheSync(req.Context()), nil, fmt.Errorf("failed to sync caches")) - })) lo.Must0(mgr.AddHealthzCheck("healthz", healthz.Ping)) lo.Must0(mgr.AddReadyzCheck("readyz", healthz.Ping)) @@ -221,7 +219,7 @@ func (o *Operator) WithWebhooks(ctx context.Context, ctors ...knativeinjection.C return o } -func (o *Operator) Start(ctx context.Context) { +func (o *Operator) Start(ctx context.Context, cp cloudprovider.CloudProvider) { wg := &sync.WaitGroup{} wg.Add(1) go func() { @@ -234,6 +232,9 @@ func (o *Operator) Start(ctx context.Context) { wg.Add(1) go func() { defer wg.Done() + // Taking the first supported NodeClass to be the default NodeClass + ctx = injection.WithNodeClasses(ctx, cp.GetSupportedNodeClasses()) + ctx = injection.WithClient(ctx, o.GetClient()) webhooks.Start(ctx, o.GetConfig(), o.webhooks...) }() } diff --git a/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml new file mode 100644 index 0000000000..3907c2cafd --- /dev/null +++ b/pkg/test/v1alpha1/crds/karpenter.test.sh_testnodeclasses.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: testnodeclasses.karpenter.test.sh +spec: + group: karpenter.test.sh + names: + kind: TestNodeClass + listKind: TestNodeClassList + plural: testnodeclasses + singular: testnodeclass + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: TestNodeClass is the Schema for the TestNodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: TestNodeClassStatus contains the resolved state of the TestNodeClass + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/test/v1alpha1/doc.go b/pkg/test/v1alpha1/doc.go new file mode 100644 index 0000000000..1c148526a5 --- /dev/null +++ b/pkg/test/v1alpha1/doc.go @@ -0,0 +1,39 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=karpenter.test.sh +package v1alpha1 // doc.go is discovered by codegen +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +const ( + Group = "karpenter.test.sh" +) + +func init() { + gv := schema.GroupVersion{Group: Group, Version: "v1alpha1"} + v1.AddToGroupVersion(scheme.Scheme, gv) + scheme.Scheme.AddKnownTypes(gv, + &TestNodeClass{}, + &TestNodeClassList{}, + ) +} diff --git a/pkg/test/v1alpha1/testnodeclass.go b/pkg/test/v1alpha1/testnodeclass.go new file mode 100644 index 0000000000..778376408b --- /dev/null +++ b/pkg/test/v1alpha1/testnodeclass.go @@ -0,0 +1,52 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + _ "embed" + + "github.com/awslabs/operatorpkg/object" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +//go:generate controller-gen crd object:headerFile="../../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds +var ( + //go:embed crds/karpenter.test.sh_testnodeclasses.yaml + TestNodeClassCRD []byte + CRDs = []*v1.CustomResourceDefinition{ + object.Unmarshal[v1.CustomResourceDefinition](TestNodeClassCRD), + } +) + +// TestNodeClass is the Schema for the TestNodeClass API +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=testnodeclasses,scope=Cluster +// +kubebuilder:subresource:status +type TestNodeClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Status TestNodeClassStatus `json:"status,omitempty"` +} + +// TestNodeClassList contains a list of TestNodeClass +// +kubebuilder:object:root=true +type TestNodeClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []TestNodeClass `json:"items"` +} diff --git a/pkg/test/v1alpha1/testnodeclass_status.go b/pkg/test/v1alpha1/testnodeclass_status.go new file mode 100644 index 0000000000..c3f48ec3dd --- /dev/null +++ b/pkg/test/v1alpha1/testnodeclass_status.go @@ -0,0 +1,39 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "github.com/awslabs/operatorpkg/status" +) + +// TestNodeClassStatus contains the resolved state of the TestNodeClass +type TestNodeClassStatus struct { + // Conditions contains signals for health and readiness + Conditions []status.Condition `json:"conditions,omitempty"` +} + +func (t *TestNodeClass) StatusConditions() status.ConditionSet { + return status.NewReadyConditions().For(t) +} + +func (t *TestNodeClass) GetConditions() []status.Condition { + return t.Status.Conditions +} + +func (t *TestNodeClass) SetConditions(conditions []status.Condition) { + t.Status.Conditions = conditions +} diff --git a/pkg/test/v1alpha1/zz_generated.deepcopy.go b/pkg/test/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000000..d38c0fb9aa --- /dev/null +++ b/pkg/test/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,106 @@ +//go:build !ignore_autogenerated + +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "github.com/awslabs/operatorpkg/status" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestNodeClass) DeepCopyInto(out *TestNodeClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestNodeClass. +func (in *TestNodeClass) DeepCopy() *TestNodeClass { + if in == nil { + return nil + } + out := new(TestNodeClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TestNodeClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestNodeClassList) DeepCopyInto(out *TestNodeClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]TestNodeClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestNodeClassList. +func (in *TestNodeClassList) DeepCopy() *TestNodeClassList { + if in == nil { + return nil + } + out := new(TestNodeClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TestNodeClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestNodeClassStatus) DeepCopyInto(out *TestNodeClassStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]status.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestNodeClassStatus. +func (in *TestNodeClassStatus) DeepCopy() *TestNodeClassStatus { + if in == nil { + return nil + } + out := new(TestNodeClassStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index 341ffbbe9b..e689617abf 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -39,16 +39,40 @@ import ( "knative.dev/pkg/webhook/certificates" "knative.dev/pkg/webhook/configmaps" "knative.dev/pkg/webhook/resourcesemantics" + "knative.dev/pkg/webhook/resourcesemantics/conversion" "knative.dev/pkg/webhook/resourcesemantics/validation" "sigs.k8s.io/controller-runtime/pkg/healthz" + v1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" "sigs.k8s.io/karpenter/pkg/operator/logging" "sigs.k8s.io/karpenter/pkg/operator/options" ) const component = "webhook" +var ( + ConversionResource = map[schema.GroupKind]conversion.GroupKindConversion{ + v1beta1.SchemeGroupVersion.WithKind("NodePool").GroupKind(): { + DefinitionName: "nodepools.karpenter.sh", + HubVersion: "v1", + Zygotes: map[string]conversion.ConvertibleObject{ + "v1": &v1.NodePool{}, + "v1beta1": &v1beta1.NodePool{}, + }, + }, + v1beta1.SchemeGroupVersion.WithKind("NodeClaim").GroupKind(): { + DefinitionName: "nodeclaims.karpenter.sh", + HubVersion: "v1", + Zygotes: map[string]conversion.ConvertibleObject{ + "v1": &v1.NodeClaim{}, + "v1beta1": &v1beta1.NodeClaim{}, + }, + }, + } +) + var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ v1beta1.SchemeGroupVersion.WithKind("NodePool"): &v1beta1.NodePool{}, v1beta1.SchemeGroupVersion.WithKind("NodeClaim"): &v1beta1.NodeClaim{}, @@ -59,9 +83,23 @@ func NewWebhooks() []knativeinjection.ControllerConstructor { certificates.NewController, NewCRDValidationWebhook, NewConfigValidationWebhook, + NewCRDConversionWebhook, } } +func NewCRDConversionWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { + nodeclassCtx := injection.GetNodeClasses(ctx) + client := injection.GetClient(ctx) + return conversion.NewConversionController( + ctx, + "/conversion/karpenter.sh", + ConversionResource, + func(ctx context.Context) context.Context { + return injection.WithClient(injection.WithNodeClasses(ctx, nodeclassCtx), client) + }, + ) +} + func NewCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { return validation.NewAdmissionController(ctx, "validation.webhook.karpenter.sh",