diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index 01ae95842babb..75f943307f8e5 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -201,7 +201,7 @@ const cluster = new eks.Cluster(this, 'HelloEKS', { }); cluster.addNodegroupCapacity('custom-node-group', { - instanceType: new ec2.InstanceType('m5.large'), + instanceTypes: [new ec2.InstanceType('m5.large')], minSize: 4, diskSize: 100, amiType: eks.NodegroupAmiType.AL2_X86_64_GPU, @@ -209,6 +209,27 @@ cluster.addNodegroupCapacity('custom-node-group', { }); ``` +#### Spot Instances Support + +Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using +Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. + +> For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types). + + +```ts +cluster.addNodegroupCapacity('extra-ng-spot', { + instanceTypes: [ + new ec2.InstanceType('c5.large'), + new ec2.InstanceType('c5a.large'), + new ec2.InstanceType('c5d.large'), + ], + minSize: 3, + capacityType: eks.CapacityType.SPOT, +}); + +``` + #### Launch Template Support You can specify a launch template that the node group will use. Note that when using a custom AMI, Amazon EKS doesn't merge any user data. @@ -236,7 +257,9 @@ cluster.addNodegroupCapacity('extra-ng', { }); ``` -> For more details visit [Launch Template Support](https://docs.aws.amazon.com/en_ca/eks/latest/userguide/launch-templates.html). +You may specify one or instance types in either the `instanceTypes` property of `NodeGroup` or in the launch template, **but not both**. + +> For more details visit [Launch Template Support](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html). Graviton 2 instance types are supported including `c6g`, `m6g`, `r6g` and `t4g`. @@ -552,7 +575,7 @@ Amazon Linux 2 AMI for ARM64 will be automatically selected. ```ts // add a managed ARM64 nodegroup cluster.addNodegroupCapacity('extra-ng-arm', { - instanceType: new ec2.InstanceType('m6g.medium'), + instanceTypes: [new ec2.InstanceType('m6g.medium')], minSize: 2, }); diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index dfbb347bb1e5d..1985f893ca048 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1123,7 +1123,7 @@ export class Cluster extends ClusterBase { this.addAutoScalingGroupCapacity('DefaultCapacity', { instanceType, minCapacity }) : undefined; this.defaultNodegroup = props.defaultCapacityType !== DefaultCapacityType.EC2 ? - this.addNodegroupCapacity('DefaultCapacity', { instanceType, minSize: minCapacity }) : undefined; + this.addNodegroupCapacity('DefaultCapacity', { instanceTypes: [instanceType], minSize: minCapacity }) : undefined; } const outputConfigCommand = props.outputConfigCommand === undefined ? true : props.outputConfigCommand; diff --git a/packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts b/packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts index ea7e7a448e4f8..d475e69574fb3 100644 --- a/packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts +++ b/packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts @@ -1,6 +1,6 @@ import { InstanceType, ISecurityGroup, SubnetSelection } from '@aws-cdk/aws-ec2'; import { IRole, ManagedPolicy, Role, ServicePrincipal } from '@aws-cdk/aws-iam'; -import { IResource, Resource } from '@aws-cdk/core'; +import { IResource, Resource, Annotations } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { Cluster, ICluster } from './cluster'; import { CfnNodegroup } from './eks.generated'; @@ -37,6 +37,20 @@ export enum NodegroupAmiType { AL2_ARM_64 = 'AL2_ARM_64' } +/** + * Capacity type of the managed node group + */ +export enum CapacityType { + /** + * spot instances + */ + SPOT = 'SPOT', + /** + * on-demand instances + */ + ON_DEMAND = 'ON_DEMAND' +} + /** * The remote access (SSH) configuration to use with your node group. * @@ -95,7 +109,7 @@ export interface NodegroupOptions { /** * The AMI type for your node group. * - * @default - auto-determined from the instanceType property. + * @default - auto-determined from the instanceTypes property. */ readonly amiType?: NodegroupAmiType; /** @@ -138,8 +152,15 @@ export interface NodegroupOptions { * `AL2_x86_64_GPU` with the amiType parameter. * * @default t3.medium + * @deprecated Use `instanceTypes` instead. */ readonly instanceType?: InstanceType; + /** + * The instance types to use for your node group. + * @default t3.medium will be used according to the cloudformation document. + * @see - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-instancetypes + */ + readonly instanceTypes?: InstanceType[]; /** * The Kubernetes labels to be applied to the nodes in the node group when they are created. * @@ -183,6 +204,12 @@ export interface NodegroupOptions { * @default - no launch template */ readonly launchTemplateSpec?: LaunchTemplateSpec; + /** + * The capacity type of the nodegroup. + * + * @default - ON_DEMAND + */ + readonly capacityType?: CapacityType; } /** @@ -199,6 +226,10 @@ export interface NodegroupProps extends NodegroupOptions { * The Nodegroup resource class */ export class Nodegroup extends Resource implements INodegroup { + /** + * Default instanceTypes + */ + public static readonly DEFAULT_INSTANCE_TYPES = [new InstanceType('t3.medium')]; /** * Import the Nodegroup from attributes */ @@ -253,6 +284,25 @@ export class Nodegroup extends Resource implements INodegroup { throw new Error(`Minimum capacity ${this.minSize} can't be greater than desired size ${this.desiredSize}`); } + if (props.instanceType && props.instanceTypes) { + throw new Error('"instanceType is deprecated, please use "instanceTypes" only.'); + } + + if (props.instanceType) { + Annotations.of(this).addWarning('"instanceType" is deprecated and will be removed in the next major version. please use "instanceTypes" instead'); + } + const instanceTypes = props.instanceTypes ?? (props.instanceType ? [props.instanceType] : Nodegroup.DEFAULT_INSTANCE_TYPES); + // get unique AMI types from instanceTypes + const uniqAmiTypes = getAmiTypes(instanceTypes); + // uniqAmiTypes.length should be at least 1 + if (uniqAmiTypes.length > 1) { + throw new Error('instanceTypes of different CPU architectures is not allowed'); + } + const determinedAmiType = uniqAmiTypes[0]; + if (props.amiType && props.amiType !== determinedAmiType) { + throw new Error(`The specified AMI does not match the instance types architecture, either specify ${determinedAmiType} or dont specify any`); + } + if (!props.nodeRole) { const ngRole = new Role(this, 'NodeGroupRole', { assumedBy: new ServicePrincipal('ec2.amazonaws.com'), @@ -271,11 +321,12 @@ export class Nodegroup extends Resource implements INodegroup { nodegroupName: props.nodegroupName, nodeRole: this.role.roleArn, subnets: this.cluster.vpc.selectSubnets(props.subnets).subnetIds, - amiType: props.amiType ?? (props.instanceType ? getAmiTypeForInstanceType(props.instanceType).toString() : - undefined), + // AmyType is not allowed by CFN when specifying an image id in your launch template. + amiType: props.launchTemplateSpec === undefined ? determinedAmiType : undefined, diskSize: props.diskSize, forceUpdateEnabled: props.forceUpdate ?? true, - instanceTypes: props.instanceType ? [props.instanceType.toString()] : undefined, + instanceTypes: props.instanceTypes ? props.instanceTypes.map(t => t.toString()) : + props.instanceType ? [props.instanceType.toString()] : undefined, labels: props.labels, releaseVersion: props.releaseVersion, remoteAccess: props.remoteAccess ? { @@ -291,17 +342,21 @@ export class Nodegroup extends Resource implements INodegroup { tags: props.tags, }); + if (props.capacityType) { + resource.addPropertyOverride('CapacityType', props.capacityType.valueOf()); + } + if (props.launchTemplateSpec) { if (props.diskSize) { // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize throw new Error('diskSize must be specified within the launch template'); } - if (props.instanceType) { - // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html - // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize - throw new Error('Instance types must be specified within the launch template'); - } + /** + * Instance types can be specified either in `instanceType` or launch template but not both. AS we can not check the content of + * the provided launch template and the `instanceType` property is preferrable. We allow users to define `instanceType` property here. + * see - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-instancetypes + */ // TODO: update this when the L1 resource spec is updated. resource.addPropertyOverride('LaunchTemplate', { Id: props.launchTemplateSpec.id, @@ -340,3 +395,8 @@ function getAmiTypeForInstanceType(instanceType: InstanceType) { NodegroupAmiType.AL2_X86_64; } +function getAmiTypes(instanceType: InstanceType[]) { + const amiTypes = instanceType.map(i =>getAmiTypeForInstanceType(i)); + // retuen unique AMI types + return [...new Set(amiTypes)]; +} diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index e3c699e7fa491..23013f70f2925 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -1251,6 +1251,13 @@ ] }, "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"", + { + "Fn::GetAtt": [ + "ClusterNodegroupextrangspotNodeGroupRoleB53B4857", + "Arn" + ] + }, + "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"", { "Fn::GetAtt": [ "ClusterNodegroupextrangarmNodeGroupRoleADF5749F", @@ -3251,6 +3258,109 @@ } } }, + "ClusterNodegroupextrangspotNodeGroupRoleB53B4857": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "ec2.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKSWorkerNodePolicy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKS_CNI_Policy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + ] + } + ] + } + }, + "ClusterNodegroupextrangspotB327AE6B": { + "Type": "AWS::EKS::Nodegroup", + "Properties": { + "ClusterName": { + "Ref": "Cluster9EE0221C" + }, + "NodeRole": { + "Fn::GetAtt": [ + "ClusterNodegroupextrangspotNodeGroupRoleB53B4857", + "Arn" + ] + }, + "Subnets": [ + { + "Ref": "VpcPrivateSubnet1Subnet536B997A" + }, + { + "Ref": "VpcPrivateSubnet2Subnet3788AAA1" + }, + { + "Ref": "VpcPrivateSubnet3SubnetF258B56E" + } + ], + "AmiType": "AL2_x86_64", + "ForceUpdateEnabled": true, + "InstanceTypes": [ + "c5.large", + "c5a.large", + "c5d.large" + ], + "ScalingConfig": { + "DesiredSize": 3, + "MaxSize": 3, + "MinSize": 3 + }, + "CapacityType": "SPOT" + } + }, "ClusterNodegroupextrangarmNodeGroupRoleADF5749F": { "Type": "AWS::IAM::Role", "Properties": { diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts index ea02eb890073c..12c7334bd96a5 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts @@ -53,6 +53,8 @@ class EksClusterStack extends TestStack { this.assertNodeGroupX86(); + this.assertNodeGroupSpot(); + this.assertNodeGroupArm(); this.assertNodeGroupCustomAmi(); @@ -162,6 +164,20 @@ class EksClusterStack extends TestStack { nodeRole: this.cluster.defaultCapacity ? this.cluster.defaultCapacity.role : undefined, }); } + private assertNodeGroupSpot() { + // add a extra nodegroup + this.cluster.addNodegroupCapacity('extra-ng-spot', { + instanceTypes: [ + new ec2.InstanceType('c5.large'), + new ec2.InstanceType('c5a.large'), + new ec2.InstanceType('c5d.large'), + ], + minSize: 3, + // reusing the default capacity nodegroup instance role when available + nodeRole: this.cluster.defaultCapacity ? this.cluster.defaultCapacity.role : undefined, + capacityType: eks.CapacityType.SPOT, + }); + } private assertNodeGroupCustomAmi() { // add a extra nodegroup const userData = ec2.UserData.forLinux(); diff --git a/packages/@aws-cdk/aws-eks/test/test.nodegroup.ts b/packages/@aws-cdk/aws-eks/test/test.nodegroup.ts index a6cdf5889574a..ff962572a6f92 100644 --- a/packages/@aws-cdk/aws-eks/test/test.nodegroup.ts +++ b/packages/@aws-cdk/aws-eks/test/test.nodegroup.ts @@ -185,6 +185,177 @@ export = { )); test.done(); }, + 'create nodegroup with on-demand capacity type'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + + // WHEN + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + new eks.Nodegroup(stack, 'Nodegroup', { + cluster, + instanceType: new ec2.InstanceType('m5.large'), + capacityType: eks.CapacityType.ON_DEMAND, + }); + + // THEN + expect(stack).to(haveResourceLike('AWS::EKS::Nodegroup', { + InstanceTypes: [ + 'm5.large', + ], + CapacityType: 'ON_DEMAND', + }, + )); + test.done(); + }, + 'create nodegroup with spot capacity type'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + + // WHEN + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + new eks.Nodegroup(stack, 'Nodegroup', { + cluster, + instanceTypes: [ + new ec2.InstanceType('m5.large'), + new ec2.InstanceType('t3.large'), + new ec2.InstanceType('c5.large'), + ], + capacityType: eks.CapacityType.SPOT, + }); + // THEN + expect(stack).to(haveResourceLike('AWS::EKS::Nodegroup', { + InstanceTypes: [ + 'm5.large', + 't3.large', + 'c5.large', + ], + CapacityType: 'SPOT', + }, + )); + test.done(); + }, + 'create nodegroup with on-demand capacity type and multiple instance types'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + + // WHEN + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + new eks.Nodegroup(stack, 'Nodegroup', { + cluster, + instanceTypes: [ + new ec2.InstanceType('m5.large'), + new ec2.InstanceType('t3.large'), + new ec2.InstanceType('c5.large'), + ], + capacityType: eks.CapacityType.ON_DEMAND, + }); + // THEN + expect(stack).to(haveResourceLike('AWS::EKS::Nodegroup', { + InstanceTypes: [ + 'm5.large', + 't3.large', + 'c5.large', + ], + CapacityType: 'ON_DEMAND', + }, + )); + test.done(); + }, + 'throws when both instanceTypes and instanceType defined'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + + // WHEN + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + // THEN + test.throws(() => cluster.addNodegroupCapacity('ng', { + instanceType: new ec2.InstanceType('m5.large'), + instanceTypes: [ + new ec2.InstanceType('m5.large'), + new ec2.InstanceType('t3.large'), + new ec2.InstanceType('c5.large'), + ], + capacityType: eks.CapacityType.SPOT, + }), /"instanceType is deprecated, please use "instanceTypes" only/); + test.done(); + }, + 'create nodegroup with neither instanceTypes nor instanceType defined'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + + // WHEN + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + version: CLUSTER_VERSION, + }); + new eks.Nodegroup(stack, 'Nodegroup', { + cluster, + capacityType: eks.CapacityType.SPOT, + }); + // THEN + expect(stack).to(haveResourceLike('AWS::EKS::Nodegroup', { + CapacityType: 'SPOT', + }, + )); + test.done(); + }, + 'throws when instanceTypes provided with different CPU architrcture'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + // THEN + test.throws(() => cluster.addNodegroupCapacity('ng', { + instanceTypes: [ + // X86 + new ec2.InstanceType('c5.large'), + new ec2.InstanceType('c5a.large'), + // ARM64 + new ec2.InstanceType('m6g.large'), + ], + }), /instanceTypes of different CPU architectures is not allowed/); + test.done(); + }, + 'throws when amiType provided is incorrect'(test: Test) { + // GIVEN + const { stack, vpc } = testFixture(); + const cluster = new eks.Cluster(stack, 'Cluster', { + vpc, + defaultCapacity: 0, + version: CLUSTER_VERSION, + }); + // THEN + test.throws(() => cluster.addNodegroupCapacity('ng', { + instanceTypes: [ + new ec2.InstanceType('c5.large'), + new ec2.InstanceType('c5a.large'), + new ec2.InstanceType('c5d.large'), + ], + // incorrect amiType + amiType: eks.NodegroupAmiType.AL2_ARM_64, + }), /The specified AMI does not match the instance types architecture/); + test.done(); + }, + 'remoteAccess without security group provided'(test: Test) { // GIVEN const { stack, vpc } = testFixture(); @@ -384,37 +555,4 @@ export = { }), /diskSize must be specified within the launch template/); test.done(); }, - 'throws when both instanceType and launch template specified'(test: Test) { - // GIVEN - const { stack, vpc } = testFixture(); - - // WHEN - const cluster = new eks.Cluster(stack, 'Cluster', { - vpc, - defaultCapacity: 0, - version: CLUSTER_VERSION, - }); - const userData = ec2.UserData.forLinux(); - userData.addCommands( - 'set -o xtrace', - `/etc/eks/bootstrap.sh ${cluster.clusterName}`, - ); - const lt = new ec2.CfnLaunchTemplate(stack, 'LaunchTemplate', { - launchTemplateData: { - imageId: new eks.EksOptimizedImage().getImage(stack).imageId, - instanceType: new ec2.InstanceType('t3.small').toString(), - userData: cdk.Fn.base64(userData.render()), - }, - }); - // THEN - test.throws(() => - cluster.addNodegroupCapacity('ng-lt', { - instanceType: new ec2.InstanceType('c5.large'), - launchTemplateSpec: { - id: lt.ref, - version: lt.attrDefaultVersionNumber, - }, - }), /Instance types must be specified within the launch template/); - test.done(); - }, };