From 1e5fd43abd206a4264c57fc2173a0b06067c7264 Mon Sep 17 00:00:00 2001 From: DingGGu Date: Fri, 14 Jun 2024 14:56:03 +0900 Subject: [PATCH 1/5] Add tenancy option dedicated on LaunchTemplate --- pkg/apis/v1beta1/ec2nodeclass.go | 10 ++++++++++ pkg/apis/v1beta1/ec2nodeclass_validation.go | 9 +++++++++ pkg/providers/amifamily/resolver.go | 2 ++ pkg/providers/launchtemplate/launchtemplate.go | 3 +++ 4 files changed, 24 insertions(+) diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 66a2926b2ced..3b179c8ba077 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -113,6 +113,16 @@ type EC2NodeClassSpec struct { // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":2,"httpTokens":"required"} // +optional MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // Tenancy of the instance. An instance with a tenancy of dedicated runs on single-tenant hardware. + // + // If not set, Tenancy will be set to default. + // Currently, Karpenter only support default and dedicated option. + // For more information, + // See the AWS::EC2::LaunchTemplate Placement Document. + // https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-placement.html + // +kubebuilder:validation:Enum:={default,dedicated} + // +optional + Tenancy *string `json:"tenancy,omitempty"` // Context is a Reserved field in EC2 APIs // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html // +optional diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation.go b/pkg/apis/v1beta1/ec2nodeclass_validation.go index 4fc18527f00d..9c67d858750a 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation.go @@ -32,6 +32,7 @@ const ( amiFamilyPath = "amiFamily" tagsPath = "tags" metadataOptionsPath = "metadataOptions" + tenancyPath = "tenancy" blockDeviceMappingsPath = "blockDeviceMappings" rolePath = "role" instanceProfilePath = "instanceProfile" @@ -79,6 +80,7 @@ func (in *EC2NodeClassSpec) validate(_ context.Context) (errs *apis.FieldError) in.validateAMISelectorTerms().ViaField(amiSelectorTermsPath), in.validateMetadataOptions().ViaField(metadataOptionsPath), in.validateAMIFamily().ViaField(amiFamilyPath), + in.validateTenancy().ViaField(tenancyPath), in.validateBlockDeviceMappings().ViaField(blockDeviceMappingsPath), in.validateTags().ViaField(tagsPath), ) @@ -305,3 +307,10 @@ func (in *EC2NodeClassSpec) validateRoleImmutability(originalSpec *EC2NodeClassS } return nil } + +func (in *EC2NodeClassSpec) validateTenancy() (errs *apis.FieldError) { + if in.Tenancy != nil { + return in.validateStringEnum(*in.Tenancy, "tenancy", ec2.Tenancy_Values()) + } + return nil +} diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index c257111900b1..eeaf3c784ce3 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -71,6 +71,7 @@ type LaunchTemplate struct { AMIID string InstanceTypes []*cloudprovider.InstanceType `hash:"ignore"` DetailedMonitoring bool + Tenancy string EFACount int CapacityType string } @@ -231,6 +232,7 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla MetadataOptions: nodeClass.Spec.MetadataOptions, DetailedMonitoring: aws.BoolValue(nodeClass.Spec.DetailedMonitoring), AMIID: amiID, + Tenancy: aws.StringValue(nodeClass.Spec.Tenancy), InstanceTypes: instanceTypes, EFACount: efaCount, CapacityType: capacityType, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index d40fda648fb1..1004fc41e3d4 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -259,6 +259,9 @@ func (p *DefaultProvider) createLaunchTemplate(ctx context.Context, options *ami SecurityGroupIds: lo.Ternary(networkInterfaces != nil, nil, lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) })), UserData: aws.String(userData), ImageId: aws.String(options.AMIID), + Placement: &ec2.LaunchTemplatePlacementRequest{ + Tenancy: aws.String(options.Tenancy), + }, MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{ HttpEndpoint: options.MetadataOptions.HTTPEndpoint, HttpProtocolIpv6: options.MetadataOptions.HTTPProtocolIPv6, From 6866d6057cdf07e755c88ba13b7257cbd44b4c10 Mon Sep 17 00:00:00 2001 From: DingGGu Date: Fri, 14 Jun 2024 15:09:40 +0900 Subject: [PATCH 2/5] codegen ec2nodeclasses --- .../crds/karpenter.k8s.aws_ec2nodeclasses.yaml | 14 ++++++++++++++ pkg/apis/v1beta1/zz_generated.deepcopy.go | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index aec3d01d61d4..fa93d911dbca 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -433,6 +433,20 @@ spec: rule: self.all(k, k !='karpenter.sh/nodeclaim') - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + tenancy: + description: |- + Tenancy of the instance. An instance with a tenancy of dedicated runs on single-tenant hardware. + + + If not set, Tenancy will be set to default. + Currently, Karpenter only support default and dedicated option. + For more information, + See the AWS::EC2::LaunchTemplate Placement Document. + https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-placement.html + enum: + - default + - dedicated + type: string userData: description: |- UserData to be applied to the provisioned nodes. diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index f248b480be5b..95cbba016a22 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -284,6 +284,11 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = new(MetadataOptions) (*in).DeepCopyInto(*out) } + if in.Tenancy != nil { + in, out := &in.Tenancy, &out.Tenancy + *out = new(string) + **out = **in + } if in.Context != nil { in, out := &in.Context, &out.Context *out = new(string) From 46a2bd8dc2ef58ba30c42292210f6491df2dc1c7 Mon Sep 17 00:00:00 2001 From: DingGGu Date: Fri, 14 Jun 2024 18:03:27 +0900 Subject: [PATCH 3/5] doc: documentation for dedicated instance --- .../content/en/v0.37/concepts/nodeclasses.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/website/content/en/v0.37/concepts/nodeclasses.md b/website/content/en/v0.37/concepts/nodeclasses.md index 37306468461d..a99936cbb41f 100644 --- a/website/content/en/v0.37/concepts/nodeclasses.md +++ b/website/content/en/v0.37/concepts/nodeclasses.md @@ -117,6 +117,10 @@ spec: # Optional, configures if the instance should be launched with an associated public IP address. # If not specified, the default value depends on the subnet's public IP auto-assign setting. associatePublicIPAddress: true + + # Optional, configures tenancy for the instance + # Currently only support default and dedicated + tenancy: default status: # Resolved subnets subnets: @@ -1110,6 +1114,19 @@ If a `NodeClaim` requests `vpc.amazonaws.com/efa` resources, `spec.associatePubl requires that the field is only set to true when configuring an instance with a single ENI at launch. When using this field, it is advised that users segregate their EFA workload to use a separate `NodePool` / `EC2NodeClass` pair. {{% /alert %}} +## spec.tenancy + +A string field that sets the [Dedicated instances](https://docs.aws.amazon.com/ko_kr/AWSEC2/latest/UserGuide/dedicated-instance.html). Currently allowed values are default and dedicated. + +```yaml +spec: + tenancy: dedicated +``` + +By default, EC2 instances run on shared tenancy hardware. This means that multiple AWS accounts can share the same physical hardware. + +Dedicated instances are EC2 instances that run on hardware dedicated to a single AWS account. This means that Dedicated Instances are physically isolated at the host hardware level from instances belonging to other AWS accounts, even if that account is associated with a single payer account. However, Dedicated Instances can share hardware with other instances in the same AWS account that are not Dedicated Instances. + ## status.subnets [`status.subnets`]({{< ref "#statussubnets" >}}) contains the resolved `id` and `zone` of the subnets that were selected by the [`spec.subnetSelectorTerms`]({{< ref "#specsubnetselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. From 5f9bd01c45cfc0f597ea932c4386cd4ede33747f Mon Sep 17 00:00:00 2001 From: DingGGu Date: Fri, 14 Jun 2024 18:03:45 +0900 Subject: [PATCH 4/5] test: LaunchTemplate render tenancy option --- pkg/providers/launchtemplate/suite_test.go | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index d2a0d6382ffe..d2e5b44221de 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -2156,6 +2156,31 @@ var _ = Describe("LaunchTemplate Provider", func() { }) }) }) + Context("Tenancy dedicated", func() { + It("should default tenancy was not set", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) + awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { + Expect(aws.StringValue(ltInput.LaunchTemplateData.Placement.Tenancy)).To(BeNil()) + }) + }) + It("should pass tenancy setting to the dedicated", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.Tenancy = aws.String("dedicated") + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) + awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { + Expect(aws.StringValue(ltInput.LaunchTemplateData.Placement.Tenancy)).To(Equal("dedicated")) + }) + }) + }) }) // ExpectTags verifies that the expected tags are a subset of the tags found From ae90694d7a0d892e85aca19b463cd20921d1cb19 Mon Sep 17 00:00:00 2001 From: DingGGu Date: Fri, 14 Jun 2024 18:09:02 +0900 Subject: [PATCH 5/5] Use ko_kr url in documents --- website/content/en/v0.37/concepts/nodeclasses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/content/en/v0.37/concepts/nodeclasses.md b/website/content/en/v0.37/concepts/nodeclasses.md index a99936cbb41f..03593a151210 100644 --- a/website/content/en/v0.37/concepts/nodeclasses.md +++ b/website/content/en/v0.37/concepts/nodeclasses.md @@ -1116,7 +1116,7 @@ requires that the field is only set to true when configuring an instance with a ## spec.tenancy -A string field that sets the [Dedicated instances](https://docs.aws.amazon.com/ko_kr/AWSEC2/latest/UserGuide/dedicated-instance.html). Currently allowed values are default and dedicated. +A string field that sets the [Dedicated instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/dedicated-instance.html). Currently allowed values are default and dedicated. ```yaml spec: