From 9c0d8a6a37569b2cfb41179ca84eac32fe9eaca8 Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Wed, 24 Oct 2018 13:45:58 +0100 Subject: [PATCH] Refactor the main API struct This will accommodate for - multiple nodegroups - public and private subnets - use taskmanager tasks for nodegroup(s) deletion - cleanup nodegroup terminology --- README.md | 2 +- integration/integration_test.go | 3 - pkg/cfn/builder/api_test.go | 128 +++++++++++------ pkg/cfn/builder/cluster.go | 53 ++++--- pkg/cfn/builder/iam.go | 18 ++- pkg/cfn/builder/nodegroup.go | 50 ++++--- pkg/cfn/builder/outputs.go | 36 +++++ pkg/cfn/builder/vpc.go | 67 +++++---- pkg/cfn/manager/cluster.go | 2 +- pkg/cfn/manager/nodegroup.go | 82 +++++++---- pkg/cfn/manager/tasks.go | 92 ++++++++++-- pkg/ctl/create/cluster.go | 42 +++--- pkg/ctl/delete/cluster.go | 22 ++- pkg/ctl/get/cluster.go | 2 +- pkg/ctl/scale/nodegroup.go | 11 +- pkg/ctl/utils/describe_stacks.go | 2 +- pkg/ctl/utils/wait_nodes.go | 11 +- pkg/ctl/utils/write_kubeconfig.go | 2 +- pkg/eks/api.go | 20 +-- pkg/eks/api/api.go | 179 ++++++++++++++++++++---- pkg/eks/auth.go | 61 ++++---- pkg/eks/nodegroup.go | 21 +-- pkg/kops/kops.go | 10 +- pkg/nodebootstrap/userdata.go | 13 +- pkg/utils/kubeconfig/kubeconfig_test.go | 48 ++++--- 25 files changed, 661 insertions(+), 316 deletions(-) diff --git a/README.md b/README.md index f3796686a3..ce4e31658e 100644 --- a/README.md +++ b/README.md @@ -158,7 +158,7 @@ eksctl delete cluster --name= [--region=] ### Scaling nodegroup -The default nodegroup can be scaled by using the `eksctl scale nodegroup` command. For example, to scale to 5 nodes: +The initial nodegroup can be scaled by using the `eksctl scale nodegroup` command. For example, to scale to 5 nodes: ``` eksctl scale nodegroup --name= --nodes=5 diff --git a/integration/integration_test.go b/integration/integration_test.go index bcd8f651e6..f35716f385 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -9,9 +9,6 @@ import ( "github.com/weaveworks/eksctl/pkg/eks/api" "github.com/weaveworks/eksctl/pkg/testutils" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) const ( diff --git a/pkg/cfn/builder/api_test.go b/pkg/cfn/builder/api_test.go index 7d8a31657c..f43957e237 100644 --- a/pkg/cfn/builder/api_test.go +++ b/pkg/cfn/builder/api_test.go @@ -3,6 +3,7 @@ package builder_test import ( "encoding/base64" "encoding/json" + "net" "path/filepath" "strings" @@ -90,33 +91,84 @@ var _ = Describe("CloudFormation template builder API", func() { testAZs := []string{"us-west-2b", "us-west-2a", "us-west-2c"} + newClusterConfig := func() *api.ClusterConfig { + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() + + cfg.Region = "us-west-2" + cfg.ClusterName = clusterName + cfg.AvailabilityZones = testAZs + ng.InstanceType = "t2.medium" + + return cfg + } + Describe("GetAllOutputsFromClusterStack", func() { caCertData, err := base64.StdEncoding.DecodeString(caCert) It("should not error", func() { Expect(err).ShouldNot(HaveOccurred()) }) expected := &api.ClusterConfig{ + Region: "us-west-2", ClusterName: clusterName, - SecurityGroup: "sg-0b44c48bcba5b7362", - Subnets: []string{"subnet-0f98135715dfcf55f", "subnet-0ade11bad78dced9e", "subnet-0e2e63ff1712bf6ef"}, - VPC: "vpc-0e265ad953062b94b", Endpoint: endpoint, CertificateAuthorityData: caCertData, ARN: arn, - NodeInstanceRoleARN: "", AvailabilityZones: testAZs, - } - initial := &api.ClusterConfig{ - ClusterName: clusterName, - AvailabilityZones: testAZs, + VPC: api.ClusterVPC{ + Network: api.Network{ + ID: "vpc-0e265ad953062b94b", + CIDR: &net.IPNet{ + IP: []byte{192, 168, 0, 0}, + Mask: []byte{255, 255, 0, 0}, + }, + }, + SecurityGroup: "sg-0b44c48bcba5b7362", + Subnets: map[api.SubnetTopology]map[string]api.Network{ + "Public": map[string]api.Network{ + "us-west-2b": { + ID: "subnet-0f98135715dfcf55f", + CIDR: &net.IPNet{ + IP: []byte{192, 168, 64, 0}, + Mask: []byte{255, 255, 192, 0}, + }, + }, + "us-west-2a": { + ID: "subnet-0ade11bad78dced9e", + CIDR: &net.IPNet{ + IP: []byte{192, 168, 128, 0}, + Mask: []byte{255, 255, 192, 0}, + }, + }, + "us-west-2c": { + ID: "subnet-0e2e63ff1712bf6ef", + CIDR: &net.IPNet{ + IP: []byte{192, 168, 192, 0}, + Mask: []byte{255, 255, 192, 0}, + }, + }, + }, + }, + }, + NodeGroups: []*api.NodeGroup{ + { + AMI: "", + InstanceType: "t2.medium", + SubnetTopology: "Public", + }, + }, } + initial := newClusterConfig() + + initial.SetSubnets() + rs := NewClusterResourceSet(initial) rs.AddAllResources() sampleStack := newStackWithOutputs(map[string]string{ "SecurityGroup": "sg-0b44c48bcba5b7362", - "Subnets": "subnet-0f98135715dfcf55f,subnet-0ade11bad78dced9e,subnet-0e2e63ff1712bf6ef", + "SubnetsPublic": "subnet-0f98135715dfcf55f,subnet-0ade11bad78dced9e,subnet-0e2e63ff1712bf6ef", "VPC": "vpc-0e265ad953062b94b", "Endpoint": endpoint, "CertificateAuthorityData": caCert, @@ -130,17 +182,14 @@ var _ = Describe("CloudFormation template builder API", func() { }) It("should be equal", func() { - Expect(initial).To(Equal(expected)) + Expect(*initial).To(Equal(*expected)) }) }) Describe("AutoNameTag", func() { - rs := NewNodeGroupResourceSet(&api.ClusterConfig{ - ClusterName: clusterName, - AvailabilityZones: testAZs, - NodeType: "t2.medium", - Region: "us-west-2", - }, "eksctl-test-123-cluster", 0) + cfg := newClusterConfig() + + rs := NewNodeGroupResourceSet(cfg, "eksctl-test-123-cluster", 0) err := rs.AddAllResources() It("should add all resources without errors", func() { @@ -179,12 +228,15 @@ var _ = Describe("CloudFormation template builder API", func() { }) Describe("NodeGroupTags", func() { - rs := NewNodeGroupResourceSet(&api.ClusterConfig{ - ClusterName: clusterName, - AvailabilityZones: testAZs, - NodeType: "t2.medium", - Region: "us-west-2", - }, "eksctl-test-123-cluster", 0) + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() + + cfg.Region = "us-west-2" + cfg.ClusterName = clusterName + cfg.AvailabilityZones = testAZs + ng.InstanceType = "t2.medium" + + rs := NewNodeGroupResourceSet(cfg, "eksctl-test-123-cluster", 0) rs.AddAllResources() template, err := rs.RenderJSON() @@ -210,17 +262,15 @@ var _ = Describe("CloudFormation template builder API", func() { }) Describe("NodeGroupAutoScaling", func() { - rs := NewNodeGroupResourceSet(&api.ClusterConfig{ - ClusterName: clusterName, - AvailabilityZones: testAZs, - NodeType: "t2.medium", - Region: "us-west-2", - Addons: api.ClusterAddons{ - WithIAM: api.AddonIAM{ - PolicyAutoScaling: true, - }, + cfg := newClusterConfig() + + cfg.Addons = api.ClusterAddons{ + WithIAM: api.AddonIAM{ + PolicyAutoScaling: true, }, - }, "eksctl-test-123-cluster", 0) + } + + rs := NewNodeGroupResourceSet(cfg, "eksctl-test-123-cluster", 0) rs.AddAllResources() template, err := rs.RenderJSON() @@ -251,20 +301,18 @@ var _ = Describe("CloudFormation template builder API", func() { }) Describe("UserData", func() { + cfg := newClusterConfig() var c *cloudconfig.CloudConfig caCertData, err := base64.StdEncoding.DecodeString(caCert) It("should not error", func() { Expect(err).ShouldNot(HaveOccurred()) }) - rs := NewNodeGroupResourceSet(&api.ClusterConfig{ - ClusterName: clusterName, - AvailabilityZones: testAZs, - NodeType: "m5.large", - Region: "us-west-2", - Endpoint: endpoint, - CertificateAuthorityData: caCertData, - }, "eksctl-test-123-cluster", 0) + cfg.Endpoint = endpoint + cfg.CertificateAuthorityData = caCertData + cfg.NodeGroups[0].InstanceType = "m5.large" + + rs := NewNodeGroupResourceSet(cfg, "eksctl-test-123-cluster", 0) rs.AddAllResources() template, err := rs.RenderJSON() diff --git a/pkg/cfn/builder/cluster.go b/pkg/cfn/builder/cluster.go index b512a57995..25f019f76b 100644 --- a/pkg/cfn/builder/cluster.go +++ b/pkg/cfn/builder/cluster.go @@ -1,35 +1,28 @@ package builder import ( - "net" - cfn "github.com/aws/aws-sdk-go/service/cloudformation" gfn "github.com/awslabs/goformation/cloudformation" "github.com/weaveworks/eksctl/pkg/eks/api" ) -const ( - cfnOutputClusterCertificateAuthorityData = "CertificateAuthorityData" - cfnOutputClusterEndpoint = "Endpoint" - cfnOutputClusterARN = "ARN" - cfnOutputClusterStackName = "ClusterStackName" -) - // ClusterResourceSet stores the resource information of the cluster type ClusterResourceSet struct { rs *resourceSet spec *api.ClusterConfig vpc *gfn.Value - subnets []*gfn.Value + subnets map[api.SubnetTopology][]*gfn.Value securityGroups []*gfn.Value + outputs *ClusterStackOutputs } // NewClusterResourceSet returns a resource set for the new cluster func NewClusterResourceSet(spec *api.ClusterConfig) *ClusterResourceSet { return &ClusterResourceSet{ - rs: newResourceSet(), - spec: spec, + rs: newResourceSet(), + spec: spec, + outputs: &ClusterStackOutputs{}, } } @@ -38,18 +31,11 @@ func (c *ClusterResourceSet) AddAllResources() error { templateDescriptionFeatures := clusterTemplateDescriptionDefaultFeatures - if c.spec.VPC != "" && len(c.spec.Subnets) >= 3 { + if c.spec.VPC.ID != "" && c.spec.VPC.HasSufficientPublicSubnets() { c.importResourcesForVPC() templateDescriptionFeatures = " (with shared VPC and dedicated IAM role) " } else { - _, globalCIDR, _ := net.ParseCIDR("192.168.0.0/16") - - subnets := map[string]*net.IPNet{} - _, subnets[c.spec.AvailabilityZones[0]], _ = net.ParseCIDR("192.168.64.0/18") - _, subnets[c.spec.AvailabilityZones[1]], _ = net.ParseCIDR("192.168.128.0/18") - _, subnets[c.spec.AvailabilityZones[2]], _ = net.ParseCIDR("192.168.192.0/18") - - c.addResourcesForVPC(globalCIDR, subnets) + c.addResourcesForVPC() } c.addOutputsForVPC() @@ -86,7 +72,7 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(version string) { RoleArn: gfn.MakeFnGetAttString("ServiceRole.Arn"), Version: gfn.NewString(version), ResourcesVpcConfig: &gfn.AWSEKSCluster_ResourcesVpcConfig{ - SubnetIds: c.subnets, + SubnetIds: c.subnets[api.SubnetTopologyPublic], SecurityGroupIds: c.securityGroups, }, }) @@ -98,5 +84,26 @@ func (c *ClusterResourceSet) addResourcesForControlPlane(version string) { // GetAllOutputs collects all outputs of the cluster func (c *ClusterResourceSet) GetAllOutputs(stack cfn.Stack) error { - return c.rs.GetAllOutputs(stack, c.spec) + if err := c.rs.GetAllOutputs(stack, c.outputs); err != nil { + return err + } + + c.spec.VPC.ID = c.outputs.VPC + c.spec.VPC.SecurityGroup = c.outputs.SecurityGroup + + // TODO: shouldn't assume the order is the same, can probably do an API lookup + for i, subnet := range c.outputs.SubnetsPrivate { + c.spec.VPC.ImportSubnet(api.SubnetTopologyPrivate, c.spec.AvailabilityZones[i], subnet) + } + + for i, subnet := range c.outputs.SubnetsPublic { + c.spec.VPC.ImportSubnet(api.SubnetTopologyPublic, c.spec.AvailabilityZones[i], subnet) + } + + c.spec.ClusterStackName = c.outputs.ClusterStackName + c.spec.Endpoint = c.outputs.Endpoint + c.spec.CertificateAuthorityData = c.outputs.CertificateAuthorityData + c.spec.ARN = c.outputs.ARN + + return nil } diff --git a/pkg/cfn/builder/iam.go b/pkg/cfn/builder/iam.go index 910d9d2529..299d739aba 100644 --- a/pkg/cfn/builder/iam.go +++ b/pkg/cfn/builder/iam.go @@ -5,8 +5,6 @@ import ( ) const ( - cfnOutputNodeInstanceRoleARN = "NodeInstanceRoleARN" - iamPolicyAmazonEKSServicePolicyARN = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy" iamPolicyAmazonEKSClusterPolicyARN = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" @@ -84,19 +82,19 @@ func (n *NodeGroupResourceSet) WithIAM() bool { func (n *NodeGroupResourceSet) addResourcesForIAM() { n.rs.withIAM = true - if len(n.spec.NodePolicyARNs) == 0 { - n.spec.NodePolicyARNs = iamDefaultNodePolicyARNs + if len(n.spec.PolicyARNs) == 0 { + n.spec.PolicyARNs = iamDefaultNodePolicyARNs } - if n.spec.Addons.WithIAM.PolicyAmazonEC2ContainerRegistryPowerUser { - n.spec.NodePolicyARNs = append(n.spec.NodePolicyARNs, iamPolicyAmazonEC2ContainerRegistryPowerUserARN) + if n.clusterSpec.Addons.WithIAM.PolicyAmazonEC2ContainerRegistryPowerUser { + n.spec.PolicyARNs = append(n.spec.PolicyARNs, iamPolicyAmazonEC2ContainerRegistryPowerUserARN) } else { - n.spec.NodePolicyARNs = append(n.spec.NodePolicyARNs, iamPolicyAmazonEC2ContainerRegistryReadOnlyARN) + n.spec.PolicyARNs = append(n.spec.PolicyARNs, iamPolicyAmazonEC2ContainerRegistryReadOnlyARN) } refIR := n.newResource("NodeInstanceRole", &gfn.AWSIAMRole{ Path: gfn.NewString("/"), AssumeRolePolicyDocument: makeAssumeRolePolicyDocument("ec2.amazonaws.com"), - ManagedPolicyArns: makeStringSlice(n.spec.NodePolicyARNs...), + ManagedPolicyArns: makeStringSlice(n.spec.PolicyARNs...), }) n.instanceProfile = n.newResource("NodeInstanceProfile", &gfn.AWSIAMInstanceProfile{ @@ -132,7 +130,7 @@ func (n *NodeGroupResourceSet) addResourcesForIAM() { }, ) - if n.spec.Addons.WithIAM.PolicyAutoScaling { + if n.clusterSpec.Addons.WithIAM.PolicyAutoScaling { n.rs.attachAllowPolicy("PolicyAutoScaling", refIR, "*", []string{ "autoscaling:DescribeAutoScalingGroups", @@ -145,5 +143,5 @@ func (n *NodeGroupResourceSet) addResourcesForIAM() { ) } - n.rs.newOutputFromAtt(cfnOutputNodeInstanceRoleARN, "NodeInstanceRole.Arn", true) + n.rs.newOutputFromAtt(cfnOutputInstanceRoleARN, "NodeInstanceRole.Arn", true) } diff --git a/pkg/cfn/builder/nodegroup.go b/pkg/cfn/builder/nodegroup.go index 866df77f28..bfb261221a 100644 --- a/pkg/cfn/builder/nodegroup.go +++ b/pkg/cfn/builder/nodegroup.go @@ -15,7 +15,8 @@ import ( type NodeGroupResourceSet struct { rs *resourceSet id int - spec *api.ClusterConfig + clusterSpec *api.ClusterConfig + spec *api.NodeGroup clusterStackName string nodeGroupName string instanceProfile *gfn.Value @@ -37,7 +38,8 @@ func NewNodeGroupResourceSet(spec *api.ClusterConfig, clusterStackName string, i id: id, clusterStackName: clusterStackName, nodeGroupName: fmt.Sprintf("%s-%d", spec.ClusterName, id), - spec: spec, + clusterSpec: spec, + spec: spec.NodeGroups[id], } } @@ -49,15 +51,15 @@ func (n *NodeGroupResourceSet) AddAllResources() error { n.vpc = makeImportValue(n.clusterStackName, cfnOutputClusterVPC) - userData, err := nodebootstrap.NewUserDataForAmazonLinux2(n.spec) + userData, err := nodebootstrap.NewUserDataForAmazonLinux2(n.clusterSpec, n.id) if err != nil { return err } n.userData = gfn.NewString(userData) - if n.spec.MinNodes == 0 && n.spec.MaxNodes == 0 { - n.spec.MinNodes = n.spec.Nodes - n.spec.MaxNodes = n.spec.Nodes + if n.spec.MinSize == 0 && n.spec.MaxSize == 0 { + n.spec.MinSize = n.spec.DesiredCapacity + n.spec.MaxSize = n.spec.DesiredCapacity } n.addResourcesForIAM() @@ -87,19 +89,19 @@ func (n *NodeGroupResourceSet) addResourcesForNodeGroup() { IamInstanceProfile: n.instanceProfile, SecurityGroups: n.securityGroups, - ImageId: gfn.NewString(n.spec.NodeAMI), - InstanceType: gfn.NewString(n.spec.NodeType), + ImageId: gfn.NewString(n.spec.AMI), + InstanceType: gfn.NewString(n.spec.InstanceType), UserData: n.userData, } - if n.spec.NodeSSH { + if n.spec.AllowSSH { lc.KeyName = gfn.NewString(n.spec.SSHPublicKeyName) } - if n.spec.NodeVolumeSize > 0 { + if n.spec.VolumeSize > 0 { lc.BlockDeviceMappings = []gfn.AWSAutoScalingLaunchConfiguration_BlockDeviceMapping{ { DeviceName: gfn.NewString("/dev/xvda"), Ebs: &gfn.AWSAutoScalingLaunchConfiguration_BlockDevice{ - VolumeSize: gfn.NewInteger(n.spec.NodeVolumeSize), + VolumeSize: gfn.NewInteger(n.spec.VolumeSize), }, }, } @@ -107,22 +109,28 @@ func (n *NodeGroupResourceSet) addResourcesForNodeGroup() { refLC := n.newResource("NodeLaunchConfig", lc) // currently goformation type system doesn't allow specifying `VPCZoneIdentifier: { "Fn::ImportValue": ... }`, // and tags don't have `PropagateAtLaunch` field, so we have a custom method here until this gets resolved + var vpcZoneIdentifier interface{} + if len(n.spec.AvailabilityZones) > 0 { + vpcZoneIdentifier = n.clusterSpec.VPC.SubnetIDs(api.SubnetTopologyPublic) + } else { + vpcZoneIdentifier = map[string][]interface{}{ + gfn.FnSplit: []interface{}{ + ",", + makeImportValue(n.clusterStackName, cfnOutputClusterSubnets+string(api.SubnetTopologyPublic)), + }, + } + } n.newResource("NodeGroup", &awsCloudFormationResource{ Type: "AWS::AutoScaling::AutoScalingGroup", Properties: map[string]interface{}{ "LaunchConfigurationName": refLC, - "DesiredCapacity": fmt.Sprintf("%d", n.spec.Nodes), - "MinSize": fmt.Sprintf("%d", n.spec.MinNodes), - "MaxSize": fmt.Sprintf("%d", n.spec.MaxNodes), - "VPCZoneIdentifier": map[string][]interface{}{ - gfn.FnSplit: []interface{}{ - ",", - makeImportValue(n.clusterStackName, cfnOutputClusterSubnets), - }, - }, + "DesiredCapacity": fmt.Sprintf("%d", n.spec.DesiredCapacity), + "MinSize": fmt.Sprintf("%d", n.spec.MinSize), + "MaxSize": fmt.Sprintf("%d", n.spec.MaxSize), + "VPCZoneIdentifier": vpcZoneIdentifier, "Tags": []map[string]interface{}{ {"Key": "Name", "Value": fmt.Sprintf("%s-Node", n.nodeGroupName), "PropagateAtLaunch": "true"}, - {"Key": "kubernetes.io/cluster/" + n.spec.ClusterName, "Value": "owned", "PropagateAtLaunch": "true"}, + {"Key": "kubernetes.io/cluster/" + n.clusterSpec.ClusterName, "Value": "owned", "PropagateAtLaunch": "true"}, }, }, UpdatePolicy: map[string]map[string]string{ diff --git a/pkg/cfn/builder/outputs.go b/pkg/cfn/builder/outputs.go index da64272854..5930eaad7d 100644 --- a/pkg/cfn/builder/outputs.go +++ b/pkg/cfn/builder/outputs.go @@ -14,6 +14,42 @@ import ( "github.com/kubicorn/kubicorn/pkg/logger" ) +// Outputs of CloudFormation stacks are collected into a struct with fields +// matching names of the outputs. Here is a set of reflect-based helpers that +// make this happen. Some data types get special treatment, e.g. string slices +// and byte slices. + +const ( + // outputs that are destined for ClusterStackOutputs + cfnOutputClusterVPC = "VPC" + cfnOutputClusterSecurityGroup = "SecurityGroup" + cfnOutputClusterSubnets = "Subnets" + + cfnOutputClusterCertificateAuthorityData = "CertificateAuthorityData" + cfnOutputClusterEndpoint = "Endpoint" + cfnOutputClusterARN = "ARN" + cfnOutputClusterStackName = "ClusterStackName" + + // this is set inside of NodeGroup + cfnOutputInstanceRoleARN = "InstanceRoleARN" +) + +// ClusterStackOutputs is a struct that hold all of cluster stack outputs, +// it's needed because some of the destination fields in ClusterConfig are +// deeply nested and we would need to do something complicated to handle +// those otherwise +type ClusterStackOutputs struct { + VPC string + SecurityGroup string + SubnetsPrivate []string + SubnetsPublic []string + + ClusterStackName string + Endpoint string + CertificateAuthorityData []byte + ARN string +} + // newOutput defines a new output and optionally exports it func (r *resourceSet) newOutput(name string, value interface{}, export bool) { o := map[string]interface{}{"Value": value} diff --git a/pkg/cfn/builder/vpc.go b/pkg/cfn/builder/vpc.go index 1531446ee8..7c33e33764 100644 --- a/pkg/cfn/builder/vpc.go +++ b/pkg/cfn/builder/vpc.go @@ -1,22 +1,33 @@ package builder import ( - "net" "strings" gfn "github.com/awslabs/goformation/cloudformation" + "github.com/weaveworks/eksctl/pkg/eks/api" ) -const ( - cfnOutputClusterVPC = "VPC" - cfnOutputClusterSubnets = "Subnets" - cfnOutputClusterSecurityGroup = "SecurityGroup" -) +func (c *ClusterResourceSet) addSubnets(refRT *gfn.Value, topology api.SubnetTopology) { + c.subnets = make(map[api.SubnetTopology][]*gfn.Value) + for az, subnet := range c.spec.VPC.Subnets[topology] { + alias := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) + refSubnet := c.newResource("Subnet"+string(topology)+alias, &gfn.AWSEC2Subnet{ + AvailabilityZone: gfn.NewString(az), + CidrBlock: gfn.NewString(subnet.CIDR.String()), + VpcId: c.vpc, + }) + c.newResource("RouteTableAssociation"+string(topology)+alias, &gfn.AWSEC2SubnetRouteTableAssociation{ + SubnetId: refSubnet, + RouteTableId: refRT, + }) + c.subnets[topology] = append(c.subnets[topology], refSubnet) + } +} //nolint:interfacer -func (c *ClusterResourceSet) addResourcesForVPC(globalCIDR *net.IPNet, subnets map[string]*net.IPNet) { +func (c *ClusterResourceSet) addResourcesForVPC() { c.vpc = c.newResource("VPC", &gfn.AWSEC2VPC{ - CidrBlock: gfn.NewString(globalCIDR.String()), + CidrBlock: gfn.NewString(c.spec.VPC.CIDR.String()), EnableDnsSupport: gfn.True(), EnableDnsHostnames: gfn.True(), }) @@ -27,41 +38,39 @@ func (c *ClusterResourceSet) addResourcesForVPC(globalCIDR *net.IPNet, subnets m VpcId: c.vpc, }) - refRT := c.newResource("RouteTable", &gfn.AWSEC2RouteTable{ + refPrivateRT := c.newResource("PrivateRouteTable", &gfn.AWSEC2RouteTable{ + VpcId: c.vpc, + }) + + c.addSubnets(refPrivateRT, api.SubnetTopologyPrivate) + + refPublicRT := c.newResource("PublicRouteTable", &gfn.AWSEC2RouteTable{ VpcId: c.vpc, }) c.newResource("PublicSubnetRoute", &gfn.AWSEC2Route{ - RouteTableId: refRT, + RouteTableId: refPublicRT, DestinationCidrBlock: gfn.NewString("0.0.0.0/0"), GatewayId: refIG, }) - for az, subnet := range subnets { - alias := strings.ToUpper(strings.Join(strings.Split(az, "-"), "")) - refSubnet := c.newResource("Subnet"+alias, &gfn.AWSEC2Subnet{ - AvailabilityZone: gfn.NewString(az), - CidrBlock: gfn.NewString(subnet.String()), - VpcId: c.vpc, - }) - c.newResource("RouteTableAssociation"+alias, &gfn.AWSEC2SubnetRouteTableAssociation{ - SubnetId: refSubnet, - RouteTableId: refRT, - }) - c.subnets = append(c.subnets, refSubnet) - } + c.addSubnets(refPublicRT, api.SubnetTopologyPublic) } func (c *ClusterResourceSet) importResourcesForVPC() { - c.vpc = gfn.NewString(c.spec.VPC) - for _, subnet := range c.spec.Subnets { - c.subnets = append(c.subnets, gfn.NewString(subnet)) + c.vpc = gfn.NewString(c.spec.VPC.ID) + for _, topology := range c.spec.VPC.SubnetTopologies() { + for _, subnet := range c.spec.VPC.SubnetIDs(topology) { + c.subnets[topology] = append(c.subnets[topology], gfn.NewString(subnet)) + } } } func (c *ClusterResourceSet) addOutputsForVPC() { c.rs.newOutput(cfnOutputClusterVPC, c.vpc, true) - c.rs.newJoinedOutput(cfnOutputClusterSubnets, c.subnets, true) + for _, topology := range c.spec.VPC.SubnetTopologies() { + c.rs.newJoinedOutput(cfnOutputClusterSubnets+string(topology), c.subnets[topology], true) + } } func (c *ClusterResourceSet) addResourcesForSecurityGroups() { @@ -93,7 +102,7 @@ func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { VpcId: makeImportValue(n.clusterStackName, cfnOutputClusterVPC), GroupDescription: gfn.NewString("Communication between the control plane and " + desc), Tags: []gfn.Tag{{ - Key: gfn.NewString("kubernetes.io/cluster/" + n.spec.ClusterName), + Key: gfn.NewString("kubernetes.io/cluster/" + n.clusterSpec.ClusterName), Value: gfn.NewString("owned"), }}, }) @@ -147,7 +156,7 @@ func (n *NodeGroupResourceSet) addResourcesForSecurityGroups() { FromPort: apiPort, ToPort: apiPort, }) - if n.spec.NodeSSH { + if n.spec.AllowSSH { n.newResource("SSHIPv4", &gfn.AWSEC2SecurityGroupIngress{ GroupId: refSG, CidrIp: anywhereIPv4, diff --git a/pkg/cfn/manager/cluster.go b/pkg/cfn/manager/cluster.go index 7037b40f27..f00eecd5a1 100644 --- a/pkg/cfn/manager/cluster.go +++ b/pkg/cfn/manager/cluster.go @@ -11,7 +11,7 @@ func (c *StackCollection) makeClusterStackName() string { } // CreateCluster creates the cluster -func (c *StackCollection) CreateCluster(errs chan error) error { +func (c *StackCollection) CreateCluster(errs chan error, _ interface{}) error { name := c.makeClusterStackName() logger.Info("creating cluster stack %q", name) stack := builder.NewClusterResourceSet(c.spec) diff --git a/pkg/cfn/manager/nodegroup.go b/pkg/cfn/manager/nodegroup.go index 88279b12da..0cd9419493 100644 --- a/pkg/cfn/manager/nodegroup.go +++ b/pkg/cfn/manager/nodegroup.go @@ -7,6 +7,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/pkg/errors" "github.com/weaveworks/eksctl/pkg/cfn/builder" + "github.com/weaveworks/eksctl/pkg/eks/api" cfn "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/kubicorn/kubicorn/pkg/logger" @@ -20,50 +21,71 @@ const ( minSizePath = "Resources.NodeGroup.Properties.MinSize" ) -func (c *StackCollection) makeNodeGroupStackName(sequence int) string { - return fmt.Sprintf("eksctl-%s-nodegroup-%d", c.spec.ClusterName, sequence) +func (c *StackCollection) makeNodeGroupStackName(id int) string { + return fmt.Sprintf("eksctl-%s-nodegroup-%d", c.spec.ClusterName, id) } -// CreateInitialNodeGroup creates the initial node group -func (c *StackCollection) CreateInitialNodeGroup(errs chan error) error { - return c.CreateNodeGroup(0, errs) -} - -// CreateNodeGroup creates the node group -func (c *StackCollection) CreateNodeGroup(seq int, errs chan error) error { - name := c.makeNodeGroupStackName(seq) +// CreateNodeGroup creates the nodegroup +func (c *StackCollection) CreateNodeGroup(errs chan error, data interface{}) error { + ng := data.(*api.NodeGroup) + name := c.makeNodeGroupStackName(ng.ID) logger.Info("creating nodegroup stack %q", name) - stack := builder.NewNodeGroupResourceSet(c.spec, c.makeClusterStackName(), seq) + stack := builder.NewNodeGroupResourceSet(c.spec, c.makeClusterStackName(), ng.ID) if err := stack.AddAllResources(); err != nil { return err } - c.tags = append(c.tags, newTag(NodeGroupIDTag, fmt.Sprintf("%d", seq))) + c.tags = append(c.tags, newTag(NodeGroupIDTag, fmt.Sprintf("%d", ng.ID))) + + for k, v := range ng.Tags { + c.tags = append(c.tags, newTag(k, v)) + } return c.CreateStack(name, stack, nil, errs) } -// DeleteNodeGroup deletes the node group -func (c *StackCollection) DeleteNodeGroup() error { - _, err := c.DeleteStack(c.makeNodeGroupStackName(0)) +func (c *StackCollection) listAllNodeGroups() ([]string, error) { + stacks, err := c.ListStacks(fmt.Sprintf("^eksctl-%s-nodegroup-\\d$", c.spec.ClusterName)) + if err != nil { + return nil, err + } + stackNames := []string{} + for _, s := range stacks { + if *s.StackStatus == cfn.StackStatusDeleteComplete { + continue + } + stackNames = append(stackNames, *s.StackName) + } + logger.Debug("nodegroups = %v", stackNames) + return stackNames, nil +} + +// DeleteNodeGroup deletes a nodegroup stack +func (c *StackCollection) DeleteNodeGroup(errs chan error, data interface{}) error { + defer close(errs) + name := data.(string) + _, err := c.DeleteStack(name) return err } -// WaitDeleteNodeGroup waits till the node group is deleted -func (c *StackCollection) WaitDeleteNodeGroup() error { - return c.WaitDeleteStack(c.makeNodeGroupStackName(0)) +// WaitDeleteNodeGroup waits until the nodegroup is deleted +func (c *StackCollection) WaitDeleteNodeGroup(errs chan error, data interface{}) error { + defer close(errs) + name := data.(string) + return c.WaitDeleteStack(name) } -// ScaleInitialNodeGroup will scale the first (sequence 0) nodegroup +// ScaleInitialNodeGroup will scale the first nodegroup (ID: 0) func (c *StackCollection) ScaleInitialNodeGroup() error { return c.ScaleNodeGroup(0) } -// ScaleNodeGroup will scale an existing node group -func (c *StackCollection) ScaleNodeGroup(sequence int) error { +// ScaleNodeGroup will scale an existing nodegroup +func (c *StackCollection) ScaleNodeGroup(id int) error { + ng := c.spec.NodeGroups[id] clusterName := c.makeClusterStackName() c.spec.ClusterStackName = clusterName - name := c.makeNodeGroupStackName(sequence) + name := c.makeNodeGroupStackName(id) logger.Info("scaling nodegroup stack %q in cluster %s", name, clusterName) // Get current stack @@ -85,30 +107,30 @@ func (c *StackCollection) ScaleNodeGroup(sequence int) error { currentMinSize := gjson.Get(template, minSizePath) // Set the new values - newCapacity := fmt.Sprintf("%d", c.spec.Nodes) + newCapacity := fmt.Sprintf("%d", ng.DesiredCapacity) template, err = sjson.Set(template, desiredCapacityPath, newCapacity) if err != nil { return errors.Wrap(err, "setting desired capacity") } - descriptionBuffer.WriteString(fmt.Sprintf("desired capacity from %s to %d", currentCapacity.Str, c.spec.Nodes)) + descriptionBuffer.WriteString(fmt.Sprintf("desired capacity from %s to %d", currentCapacity.Str, ng.DesiredCapacity)) // If the desired number of nodes is less than the min then update the min - if int64(c.spec.Nodes) < currentMinSize.Int() { - newMinSize := fmt.Sprintf("%d", c.spec.Nodes) + if int64(ng.DesiredCapacity) < currentMinSize.Int() { + newMinSize := fmt.Sprintf("%d", ng.DesiredCapacity) template, err = sjson.Set(template, minSizePath, newMinSize) if err != nil { return errors.Wrap(err, "setting min size") } - descriptionBuffer.WriteString(fmt.Sprintf(", min size from %s to %d", currentMinSize.Str, c.spec.Nodes)) + descriptionBuffer.WriteString(fmt.Sprintf(", min size from %s to %d", currentMinSize.Str, ng.DesiredCapacity)) } // If the desired number of nodes is greater than the max then update the max - if int64(c.spec.Nodes) > currentMaxSize.Int() { - newMaxSize := fmt.Sprintf("%d", c.spec.Nodes) + if int64(ng.DesiredCapacity) > currentMaxSize.Int() { + newMaxSize := fmt.Sprintf("%d", ng.DesiredCapacity) template, err = sjson.Set(template, maxSizePath, newMaxSize) if err != nil { return errors.Wrap(err, "setting max size") } - descriptionBuffer.WriteString(fmt.Sprintf(", max size from %s to %d", currentMaxSize.Str, c.spec.Nodes)) + descriptionBuffer.WriteString(fmt.Sprintf(", max size from %s to %d", currentMaxSize.Str, ng.DesiredCapacity)) } logger.Debug("stack template (post-scale change): %s", template) diff --git a/pkg/cfn/manager/tasks.go b/pkg/cfn/manager/tasks.go index 2109c091b6..13ebb95e43 100644 --- a/pkg/cfn/manager/tasks.go +++ b/pkg/cfn/manager/tasks.go @@ -6,7 +6,10 @@ import ( "github.com/kubicorn/kubicorn/pkg/logger" ) -type task func(chan error) error +type task struct { + call func(chan error, interface{}) error + data interface{} +} // Run a set of tests in parallel and wait for them to complete; // passError should take any errors and do what it needs to in @@ -21,7 +24,7 @@ func Run(passError func(error), tasks ...task) { defer wg.Done() logger.Debug("task %d started", t) errs := make(chan error) - if err := tasks[t](errs); err != nil { + if err := tasks[t].call(errs, tasks[t].data); err != nil { passError(err) return } @@ -36,19 +39,92 @@ func Run(passError func(error), tasks ...task) { wg.Wait() } -// CreateClusterWithInitialNodeGroup runs two tasks to create -// the stacks for use with CLI; any errors will be returned -// as a slice on completion of one of the two tasks -func (s *StackCollection) CreateClusterWithInitialNodeGroup() []error { +// CreateClusterWithNodeGroups runs all tasks required to create +// the stacks (a cluster and one or more nodegroups); any errors +// will be returned as a slice as soon as one of the tasks or group +// of tasks is completed +func (s *StackCollection) CreateClusterWithNodeGroups() []error { + errs := []error{} + appendErr := func(err error) { + errs = append(errs, err) + } + if Run(appendErr, task{s.CreateCluster, nil}); len(errs) > 0 { + return errs + } + + createAllNodeGroups := []task{} + for i := range s.spec.NodeGroups { + t := task{ + call: s.CreateNodeGroup, + data: s.spec.NodeGroups[i], + } + createAllNodeGroups = append(createAllNodeGroups, t) + } + if Run(appendErr, createAllNodeGroups...); len(errs) > 0 { + return errs + } + + return nil +} + +// deleteAllNodeGroupsTasks returns a list of tasks for deleting all the +// nodegroup stacks +func (s *StackCollection) deleteAllNodeGroupsTasks(call func(chan error, interface{}) error) ([]task, error) { + stacks, err := s.listAllNodeGroups() + if err != nil { + return nil, err + } + deleteAllNodeGroups := []task{} + for i := range stacks { + t := task{ + call: call, + data: stacks[i], + } + deleteAllNodeGroups = append(deleteAllNodeGroups, t) + } + return deleteAllNodeGroups, nil +} + +// DeleteAllNodeGroups runs all tasks required to delete all the nodegroup +// stacks; any errors will be returned as a slice as soon as the group +// of tasks is completed +func (s *StackCollection) DeleteAllNodeGroups() []error { + errs := []error{} + appendErr := func(err error) { + errs = append(errs, err) + } + + deleteAllNodeGroups, err := s.deleteAllNodeGroupsTasks(s.DeleteNodeGroup) + if err != nil { + appendErr(err) + return errs + } + + if Run(appendErr, deleteAllNodeGroups...); len(errs) > 0 { + return errs + } + + return nil +} + +// WaitDeleteAllNodeGroups runs all tasks required to delete all the nodegroup +// stacks, it waits for each nodegroup to get deleted; any errors will be +// returned as a slice as soon as the group of tasks is completed +func (s *StackCollection) WaitDeleteAllNodeGroups() []error { errs := []error{} appendErr := func(err error) { errs = append(errs, err) } - if Run(appendErr, s.CreateCluster); len(errs) > 0 { + + deleteAllNodeGroups, err := s.deleteAllNodeGroupsTasks(s.WaitDeleteNodeGroup) + if err != nil { + appendErr(err) return errs } - if Run(appendErr, s.CreateInitialNodeGroup); len(errs) > 0 { + + if Run(appendErr, deleteAllNodeGroups...); len(errs) > 0 { return errs } + return nil } diff --git a/pkg/ctl/create/cluster.go b/pkg/ctl/create/cluster.go index 1928ffce83..ce8833ebc6 100644 --- a/pkg/ctl/create/cluster.go +++ b/pkg/ctl/create/cluster.go @@ -32,13 +32,14 @@ var ( ) func createClusterCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() cmd := &cobra.Command{ Use: "cluster", Short: "Create a cluster", Run: func(_ *cobra.Command, args []string) { - if err := doCreateCluster(cfg, ctl.GetNameArg(args)); err != nil { + if err := doCreateCluster(cfg, ng, ctl.GetNameArg(args)); err != nil { logger.Critical("%s\n", err.Error()) os.Exit(1) } @@ -55,19 +56,19 @@ func createClusterCmd() *cobra.Command { fs.StringVarP(&cfg.Profile, "profile", "p", "", "AWS credentials profile to use (overrides the AWS_PROFILE environment variable)") fs.StringToStringVarP(&cfg.Tags, "tags", "", map[string]string{}, `A list of KV pairs used to tag the AWS resources (e.g. "Owner=John Doe,Team=Some Team")`) - fs.StringVarP(&cfg.NodeType, "node-type", "t", defaultNodeType, "node instance type") - fs.IntVarP(&cfg.Nodes, "nodes", "N", api.DefaultNodeCount, "total number of nodes (for a static ASG)") + fs.StringVarP(&ng.InstanceType, "node-type", "t", defaultNodeType, "node instance type") + fs.IntVarP(&ng.DesiredCapacity, "nodes", "N", api.DefaultNodeCount, "total number of nodes (for a static ASG)") // TODO: https://github.com/weaveworks/eksctl/issues/28 - fs.IntVarP(&cfg.MinNodes, "nodes-min", "m", 0, "minimum nodes in ASG") - fs.IntVarP(&cfg.MaxNodes, "nodes-max", "M", 0, "maximum nodes in ASG") + fs.IntVarP(&ng.MinSize, "nodes-min", "m", 0, "minimum nodes in ASG") + fs.IntVarP(&ng.MaxSize, "nodes-max", "M", 0, "maximum nodes in ASG") - fs.IntVarP(&cfg.NodeVolumeSize, "node-volume-size", "", 0, "Node volume size (in GB)") - fs.IntVar(&cfg.MaxPodsPerNode, "max-pods-per-node", 0, "maximum number of pods per node (set automatically if unspecified)") + fs.IntVarP(&ng.VolumeSize, "node-volume-size", "", 0, "Node volume size (in GB)") + fs.IntVar(&ng.MaxPodsPerNode, "max-pods-per-node", 0, "maximum number of pods per node (set automatically if unspecified)") fs.StringSliceVar(&availabilityZones, "zones", nil, "(auto-select if unspecified)") - fs.BoolVar(&cfg.NodeSSH, "ssh-access", false, "control SSH access for nodes") - fs.StringVar(&cfg.SSHPublicKeyPath, "ssh-public-key", defaultSSHPublicKey, "SSH public key to use for nodes (import from local path, or use existing EC2 key pair)") + fs.BoolVar(&ng.AllowSSH, "ssh-access", false, "control SSH access for nodes") + fs.StringVar(&ng.SSHPublicKeyPath, "ssh-public-key", defaultSSHPublicKey, "SSH public key to use for nodes (import from local path, or use existing EC2 key pair)") fs.BoolVar(&writeKubeconfig, "write-kubeconfig", true, "toggle writing of kubeconfig") fs.BoolVar(&autoKubeconfigPath, "auto-kubeconfig", false, fmt.Sprintf("save kubconfig file by cluster name, e.g. %q", kubeconfig.AutoPath(exampleClusterName))) @@ -85,14 +86,14 @@ func createClusterCmd() *cobra.Command { fs.BoolVar(&cfg.Addons.WithIAM.PolicyAutoScaling, "asg-access", false, "enable iam policy dependency for cluster-autoscaler") fs.BoolVar(&cfg.Addons.Storage, "storage-class", true, "if true (default) then a default StorageClass of type gp2 provisioned by EBS will be created") - fs.StringVar(&cfg.NodeAMI, "node-ami", ami.ResolverStatic, "Advanced use cases only. If 'static' is supplied (default) then eksctl will use static AMIs; if 'auto' is supplied then eksctl will automatically set the AMI based on region/instance type; if any other value is supplied it will override the AMI to use for the nodes. Use with extreme care.") + fs.StringVar(&ng.AMI, "node-ami", ami.ResolverStatic, "Advanced use cases only. If 'static' is supplied (default) then eksctl will use static AMIs; if 'auto' is supplied then eksctl will automatically set the AMI based on region/instance type; if any other value is supplied it will override the AMI to use for the nodes. Use with extreme care.") fs.StringVar(&kopsClusterNameForVPC, "vpc-from-kops-cluster", "", "re-use VPC from a given kops cluster") return cmd } -func doCreateCluster(cfg *api.ClusterConfig, name string) error { +func doCreateCluster(cfg *api.ClusterConfig, ng *api.NodeGroup, name string) error { ctl := eks.New(cfg) if cfg.Region != api.EKSRegionUSWest2 && cfg.Region != api.EKSRegionUSEast1 && cfg.Region != api.EKSRegionEUWest1 { @@ -115,7 +116,7 @@ func doCreateCluster(cfg *api.ClusterConfig, name string) error { kubeconfigPath = kubeconfig.AutoPath(cfg.ClusterName) } - if cfg.SSHPublicKeyPath == "" { + if ng.SSHPublicKeyPath == "" { return fmt.Errorf("--ssh-public-key must be non-empty string") } @@ -131,19 +132,20 @@ func doCreateCluster(cfg *api.ClusterConfig, name string) error { if err := kw.UseVPC(cfg); err != nil { return err } - logger.Success("using VPC (%s) and subnets (%v) from kops cluster %q", cfg.VPC, cfg.Subnets, kopsClusterNameForVPC) + logger.Success("using VPC (%s) and subnets (%v) from kops cluster %q", cfg.VPC.ID, cfg.VPC.SubnetIDs(api.SubnetTopologyPublic), kopsClusterNameForVPC) } else { // kw.UseVPC() sets AZs based on subenets used if err := ctl.SetAvailabilityZones(availabilityZones); err != nil { return err } + cfg.SetSubnets() } - if err := ctl.EnsureAMI(); err != nil { + if err := ctl.EnsureAMI(ng); err != nil { return err } - if err := ctl.LoadSSHPublicKey(); err != nil { + if err := ctl.LoadSSHPublicKey(ng); err != nil { return err } @@ -155,7 +157,7 @@ func doCreateCluster(cfg *api.ClusterConfig, name string) error { stackManager := ctl.NewStackManager() logger.Info("will create 2 separate CloudFormation stacks for cluster itself and the initial nodegroup") logger.Info("if you encounter any issues, check CloudFormation console or try 'eksctl utils describe-stacks --region=%s --name=%s'", cfg.Region, cfg.ClusterName) - errs := stackManager.CreateClusterWithInitialNodeGroup() + errs := stackManager.CreateClusterWithNodeGroups() // read any errors (it only gets non-nil errors) if len(errs) > 0 { logger.Info("%d error(s) occurred and cluster hasn't been created properly, you may wish to check CloudFormation console", len(errs)) @@ -200,12 +202,12 @@ func doCreateCluster(cfg *api.ClusterConfig, name string) error { } // authorise nodes to join - if err = ctl.CreateDefaultNodeGroupAuthConfigMap(clientSet); err != nil { + if err = ctl.CreateNodeGroupAuthConfigMap(clientSet, ng); err != nil { return err } // wait for nodes to join - if err = ctl.WaitForNodes(clientSet); err != nil { + if err = ctl.WaitForNodes(clientSet, ng); err != nil { return err } @@ -229,7 +231,7 @@ func doCreateCluster(cfg *api.ClusterConfig, name string) error { } // If GPU instance type, give instructions - if utils.IsGPUInstanceType(cfg.NodeType) { + if utils.IsGPUInstanceType(ng.InstanceType) { logger.Info("as you are using a GPU optimized instance type you will need to install NVIDIA Kubernetes device plugin.") logger.Info("\t see the following page for instructions: https://github.com/NVIDIA/k8s-device-plugin") } diff --git a/pkg/ctl/delete/cluster.go b/pkg/ctl/delete/cluster.go index 6079977e4a..7a1f4077a8 100644 --- a/pkg/ctl/delete/cluster.go +++ b/pkg/ctl/delete/cluster.go @@ -14,7 +14,7 @@ import ( ) func deleteClusterCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() cmd := &cobra.Command{ Use: "cluster", @@ -78,7 +78,17 @@ func doDeleteCluster(cfg *api.ClusterConfig, name string) error { stackManager := ctl.NewStackManager() - handleIfError(stackManager.WaitDeleteNodeGroup(), "node group") + { + errs := stackManager.WaitDeleteAllNodeGroups() + if len(errs) > 0 { + logger.Info("%d error(s) occurred while deleting nodegroup(s)", len(errs)) + for _, err := range errs { + logger.Critical("%s\n", err.Error()) + } + handleIfError(fmt.Errorf("failed to delete nodegroup(s)"), "nodegroup(s)") + } + logger.Debug("all nodegroups were deleted") + } var clusterErr bool if waitDelete { @@ -89,13 +99,13 @@ func doDeleteCluster(cfg *api.ClusterConfig, name string) error { if clusterErr { if handleIfError(ctl.DeprecatedDeleteControlPlane(), "control plane") { - handleIfError(stackManager.DeprecatedDeleteStackControlPlane(waitDelete), "stack control plane") + handleIfError(stackManager.DeprecatedDeleteStackControlPlane(waitDelete), "stack control plane (deprecated)") } } - handleIfError(stackManager.DeprecatedDeleteStackServiceRole(waitDelete), "node group") - handleIfError(stackManager.DeprecatedDeleteStackVPC(waitDelete), "stack VPC") - handleIfError(stackManager.DeprecatedDeleteStackDefaultNodeGroup(waitDelete), "default node group") + handleIfError(stackManager.DeprecatedDeleteStackServiceRole(waitDelete), "service group (deprecated)") + handleIfError(stackManager.DeprecatedDeleteStackVPC(waitDelete), "stack VPC (deprecated)") + handleIfError(stackManager.DeprecatedDeleteStackDefaultNodeGroup(waitDelete), "default nodegroup (deprecated)") ctl.MaybeDeletePublicSSHKey() diff --git a/pkg/ctl/get/cluster.go b/pkg/ctl/get/cluster.go index e88e2208a7..02217c0d62 100644 --- a/pkg/ctl/get/cluster.go +++ b/pkg/ctl/get/cluster.go @@ -12,7 +12,7 @@ import ( ) func getClusterCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() cmd := &cobra.Command{ Use: "cluster", diff --git a/pkg/ctl/scale/nodegroup.go b/pkg/ctl/scale/nodegroup.go index edfce88f0c..f217b5ca3b 100644 --- a/pkg/ctl/scale/nodegroup.go +++ b/pkg/ctl/scale/nodegroup.go @@ -11,13 +11,14 @@ import ( ) func scaleNodeGroupCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() cmd := &cobra.Command{ Use: "nodegroup", Short: "Scale a nodegroup", Run: func(_ *cobra.Command, args []string) { - if err := doScaleNodeGroup(cfg); err != nil { + if err := doScaleNodeGroup(cfg, ng); err != nil { logger.Critical("%s\n", err.Error()) os.Exit(1) } @@ -28,7 +29,7 @@ func scaleNodeGroupCmd() *cobra.Command { fs.StringVarP(&cfg.ClusterName, "name", "n", "", "EKS cluster name") - fs.IntVarP(&cfg.Nodes, "nodes", "N", -1, "total number of nodes (scale to this number)") + fs.IntVarP(&ng.DesiredCapacity, "nodes", "N", -1, "total number of nodes (scale to this number)") fs.StringVarP(&cfg.Region, "region", "r", "", "AWS region") fs.StringVarP(&cfg.Profile, "profile", "p", "", "AWS creditials profile to use (overrides the AWS_PROFILE environment variable)") @@ -38,7 +39,7 @@ func scaleNodeGroupCmd() *cobra.Command { return cmd } -func doScaleNodeGroup(cfg *api.ClusterConfig) error { +func doScaleNodeGroup(cfg *api.ClusterConfig, ng *api.NodeGroup) error { ctl := eks.New(cfg) if err := ctl.CheckAuth(); err != nil { @@ -49,7 +50,7 @@ func doScaleNodeGroup(cfg *api.ClusterConfig) error { return fmt.Errorf("no cluster name supplied. Use the --name= flag") } - if cfg.Nodes < 0 { + if ng.DesiredCapacity < 0 { return fmt.Errorf("number of nodes must be 0 or greater. Use the --nodes/-N flag") } diff --git a/pkg/ctl/utils/describe_stacks.go b/pkg/ctl/utils/describe_stacks.go index 654469bba7..102c755366 100644 --- a/pkg/ctl/utils/describe_stacks.go +++ b/pkg/ctl/utils/describe_stacks.go @@ -18,7 +18,7 @@ var ( ) func describeStacksCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() cmd := &cobra.Command{ Use: "describe-stacks", diff --git a/pkg/ctl/utils/wait_nodes.go b/pkg/ctl/utils/wait_nodes.go index 54b93b05be..86e42f1d93 100644 --- a/pkg/ctl/utils/wait_nodes.go +++ b/pkg/ctl/utils/wait_nodes.go @@ -13,13 +13,14 @@ import ( ) func waitNodesCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() + ng := cfg.NewNodeGroup() cmd := &cobra.Command{ Use: "wait-nodes", Short: "Wait for nodes", Run: func(_ *cobra.Command, _ []string) { - if err := doWaitNodes(cfg); err != nil { + if err := doWaitNodes(cfg, ng); err != nil { logger.Critical("%s\n", err.Error()) os.Exit(1) } @@ -29,13 +30,13 @@ func waitNodesCmd() *cobra.Command { fs := cmd.Flags() fs.StringVar(&utilsKubeconfigInputPath, "kubeconfig", "kubeconfig", "path to read kubeconfig") - fs.IntVarP(&cfg.MinNodes, "nodes-min", "m", api.DefaultNodeCount, "minimum number of nodes to wait for") + fs.IntVarP(&ng.MinSize, "nodes-min", "m", api.DefaultNodeCount, "minimum number of nodes to wait for") fs.DurationVar(&cfg.WaitTimeout, "timeout", api.DefaultWaitTimeout, "how long to wait") return cmd } -func doWaitNodes(cfg *api.ClusterConfig) error { +func doWaitNodes(cfg *api.ClusterConfig, ng *api.NodeGroup) error { ctl := eks.New(cfg) if utilsKubeconfigInputPath == "" { @@ -52,7 +53,7 @@ func doWaitNodes(cfg *api.ClusterConfig) error { return err } - if err := ctl.WaitForNodes(clientset); err != nil { + if err := ctl.WaitForNodes(clientset, ng); err != nil { return err } diff --git a/pkg/ctl/utils/write_kubeconfig.go b/pkg/ctl/utils/write_kubeconfig.go index 9d3afb8c09..b992ec154b 100644 --- a/pkg/ctl/utils/write_kubeconfig.go +++ b/pkg/ctl/utils/write_kubeconfig.go @@ -21,7 +21,7 @@ var ( ) func writeKubeconfigCmd() *cobra.Command { - cfg := &api.ClusterConfig{} + cfg := api.NewClusterConfig() cmd := &cobra.Command{ Use: "write-kubeconfig", diff --git a/pkg/eks/api.go b/pkg/eks/api.go index 97adf0f771..e52eff4420 100644 --- a/pkg/eks/api.go +++ b/pkg/eks/api.go @@ -141,34 +141,34 @@ func (c *ClusterProvider) CheckAuth() error { } // EnsureAMI ensures that the node AMI is set and is available -func (c *ClusterProvider) EnsureAMI() error { +func (c *ClusterProvider) EnsureAMI(ng *api.NodeGroup) error { // TODO: https://github.com/weaveworks/eksctl/issues/28 // - improve validation of parameter set overall, probably in another package - if c.Spec.NodeAMI == ami.ResolverAuto { + if ng.AMI == ami.ResolverAuto { ami.DefaultResolvers = []ami.Resolver{ami.NewAutoResolver(c.Provider.EC2())} } - if c.Spec.NodeAMI == ami.ResolverStatic || c.Spec.NodeAMI == ami.ResolverAuto { - id, err := ami.Resolve(c.Spec.Region, c.Spec.NodeType) + if ng.AMI == ami.ResolverStatic || ng.AMI == ami.ResolverAuto { + id, err := ami.Resolve(c.Spec.Region, ng.InstanceType) if err != nil { return errors.Wrap(err, "Unable to determine AMI to use") } if id == "" { - return ami.NewErrFailedResolution(c.Spec.Region, c.Spec.NodeType) + return ami.NewErrFailedResolution(c.Spec.Region, ng.InstanceType) } - c.Spec.NodeAMI = id + ng.AMI = id } // Check the AMI is available - available, err := ami.IsAvailable(c.Provider.EC2(), c.Spec.NodeAMI) + available, err := ami.IsAvailable(c.Provider.EC2(), ng.AMI) if err != nil { - return errors.Wrapf(err, "%s is not available", c.Spec.NodeAMI) + return errors.Wrapf(err, "%s is not available", ng.AMI) } if !available { - return ami.NewErrNotFound(c.Spec.NodeAMI) + return ami.NewErrNotFound(ng.AMI) } - logger.Info("using %q for nodes", c.Spec.NodeAMI) + logger.Info("using %q for nodes", ng.AMI) return nil } diff --git a/pkg/eks/api/api.go b/pkg/eks/api/api.go index 1052aaeca5..484f5e0c62 100644 --- a/pkg/eks/api/api.go +++ b/pkg/eks/api/api.go @@ -1,6 +1,7 @@ package api import ( + "net" "time" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" @@ -54,28 +55,11 @@ type ClusterConfig struct { Tags map[string]string ClusterName string - NodeAMI string - NodeType string - Nodes int - MinNodes int - MaxNodes int - - NodeVolumeSize int - - MaxPodsPerNode int - - NodePolicyARNs []string - - NodeSSH bool - SSHPublicKeyPath string - SSHPublicKey []byte - SSHPublicKeyName string - WaitTimeout time.Duration - SecurityGroup string - Subnets []string - VPC string + VPC ClusterVPC + + NodeGroups []*NodeGroup Endpoint string CertificateAuthorityData []byte @@ -83,21 +67,156 @@ type ClusterConfig struct { ClusterStackName string - NodeInstanceRoleARN string - AvailabilityZones []string Addons ClusterAddons } -// ClusterAddons provides addons for the created EKS cluster -type ClusterAddons struct { - WithIAM AddonIAM - Storage bool +// NewClusterConfig create new config for a cluster; +// it doesn't include initial nodegroup, so user must +// call NewNodeGroup to create one +func NewClusterConfig() *ClusterConfig { + return &ClusterConfig{} } -// AddonIAM provides an addon for the AWS IAM integration -type AddonIAM struct { - PolicyAmazonEC2ContainerRegistryPowerUser bool - PolicyAutoScaling bool +// SetSubnets defines CIDRs for each of the subnets, +// it must be called after SetAvailabilityZones +func (c *ClusterConfig) SetSubnets() { + _, c.VPC.CIDR, _ = net.ParseCIDR("192.168.0.0/16") + + c.VPC.Subnets = map[SubnetTopology]map[string]Network{ + SubnetTopologyPublic: map[string]Network{}, + } + + zoneCIDRs := []string{ + "192.168.64.0/18", + "192.168.128.0/18", + "192.168.192.0/18", + } + for i, zone := range c.AvailabilityZones { + _, zoneCIDR, _ := net.ParseCIDR(zoneCIDRs[i]) + c.VPC.Subnets[SubnetTopologyPublic][zone] = Network{ + CIDR: zoneCIDR, + } + } } + +// NewNodeGroup crears new nodegroup inside cluster config, +// it returns pointer to the nodegroup for convenience +func (c *ClusterConfig) NewNodeGroup() *NodeGroup { + ng := &NodeGroup{ + ID: len(c.NodeGroups), + SubnetTopology: SubnetTopologyPublic, + } + + c.NodeGroups = append(c.NodeGroups, ng) + + return ng +} + +// NodeGroup holds all configuration attributes that are +// specific to a nodegroup +type NodeGroup struct { + ID int + + AMI string + InstanceType string + AvailabilityZones []string + Tags map[string]string + SubnetTopology SubnetTopology + + DesiredCapacity int + MinSize int + MaxSize int + + VolumeSize int + + MaxPodsPerNode int + + PolicyARNs []string + InstanceRoleARN string + + AllowSSH bool + SSHPublicKeyPath string + SSHPublicKey []byte + SSHPublicKeyName string +} + +type ( + // ClusterVPC holds global subnet and all child public/private subnet + ClusterVPC struct { + Network // global CIRD and VPC ID + SecurityGroup string // cluster SG + // subnets are either public or private for use with separate nodegroups + // these are keyed by AZ for conveninece + Subnets map[SubnetTopology]map[string]Network + // for additional CIRD associations, e.g. to use with separate CIDR for + // private subnets or any ad-hoc subnets + ExtraCIDRs []*net.IPNet + } + // SubnetTopology can be SubnetTopologyPrivate or SubnetTopologyPublic + SubnetTopology string + // Network holds ID and CIDR + Network struct { + ID string + CIDR *net.IPNet + } +) + +const ( + // SubnetTopologyPrivate repesents privately-routed subnets + SubnetTopologyPrivate SubnetTopology = "Private" + // SubnetTopologyPublic repesents publicly-routed subnets + SubnetTopologyPublic SubnetTopology = "Public" +) + +// SubnetIDs returns list of subnets +func (c ClusterVPC) SubnetIDs(topology SubnetTopology) []string { + subnets := []string{} + for _, s := range c.Subnets[topology] { + subnets = append(subnets, s.ID) + } + return subnets +} + +// SubnetTopologies returns list of topologies supported +// by a given cluster config +func (c ClusterVPC) SubnetTopologies() []SubnetTopology { + topologies := []SubnetTopology{} + for topology := range c.Subnets { + topologies = append(topologies, topology) + } + return topologies +} + +// ImportSubnet loads a given subnet into cluster config +func (c ClusterVPC) ImportSubnet(topology SubnetTopology, az, subnetID string) { + if _, ok := c.Subnets[topology]; !ok { + c.Subnets[topology] = map[string]Network{} + } + if network, ok := c.Subnets[topology][az]; !ok { + c.Subnets[topology][az] = Network{ID: subnetID} + } else { + network.ID = subnetID + c.Subnets[topology][az] = network + } +} + +// HasSufficientPublicSubnets validates if there is a suffiecent +// number of subnets available to create a cluster +func (c ClusterVPC) HasSufficientPublicSubnets() bool { + return len(c.SubnetIDs(SubnetTopologyPublic)) >= 3 +} + +type ( + // ClusterAddons provides addons for the created EKS cluster + ClusterAddons struct { + WithIAM AddonIAM + Storage bool + } + // AddonIAM provides an addon for the AWS IAM integration + AddonIAM struct { + PolicyAmazonEC2ContainerRegistryPowerUser bool + PolicyAutoScaling bool + } +) diff --git a/pkg/eks/auth.go b/pkg/eks/auth.go index f05e67c474..130672bad6 100644 --- a/pkg/eks/auth.go +++ b/pkg/eks/auth.go @@ -28,8 +28,11 @@ import ( "github.com/weaveworks/eksctl/pkg/utils" ) -func (c *ClusterProvider) getKeyPairName(fingerprint *string) string { +func (c *ClusterProvider) getKeyPairName(ng *api.NodeGroup, fingerprint *string) string { keyNameParts := []string{"eksctl", c.Spec.ClusterName} + if ng != nil { + keyNameParts = append(keyNameParts, fmt.Sprintf("ng%d", ng.ID)) + } if fingerprint != nil { keyNameParts = append(keyNameParts, *fingerprint) } @@ -51,31 +54,31 @@ func (c *ClusterProvider) getKeyPair(name string) (*ec2.KeyPairInfo, error) { return output.KeyPairs[0], nil } -func (c *ClusterProvider) tryExistingSSHPublicKeyFromPath() error { - logger.Info("SSH public key file %q does not exist; will assume existing EC2 key pair", c.Spec.SSHPublicKeyPath) - existing, err := c.getKeyPair(c.Spec.SSHPublicKeyPath) +func (c *ClusterProvider) tryExistingSSHPublicKeyFromPath(ng *api.NodeGroup) error { + logger.Info("SSH public key file %q does not exist; will assume existing EC2 key pair", ng.SSHPublicKeyPath) + existing, err := c.getKeyPair(ng.SSHPublicKeyPath) if err != nil { return err } - c.Spec.SSHPublicKeyName = *existing.KeyName - logger.Info("found EC2 key pair %q", c.Spec.SSHPublicKeyName) + ng.SSHPublicKeyName = *existing.KeyName + logger.Info("found EC2 key pair %q", ng.SSHPublicKeyName) return nil } -func (c *ClusterProvider) importSSHPublicKeyIfNeeded() error { - fingerprint, err := pki.ComputeAWSKeyFingerprint(string(c.Spec.SSHPublicKey)) +func (c *ClusterProvider) importSSHPublicKeyIfNeeded(ng *api.NodeGroup) error { + fingerprint, err := pki.ComputeAWSKeyFingerprint(string(ng.SSHPublicKey)) if err != nil { return err } - c.Spec.SSHPublicKeyName = c.getKeyPairName(&fingerprint) - existing, err := c.getKeyPair(c.Spec.SSHPublicKeyName) + ng.SSHPublicKeyName = c.getKeyPairName(ng, &fingerprint) + existing, err := c.getKeyPair(ng.SSHPublicKeyName) if err != nil { if strings.HasPrefix(err.Error(), "cannot find EC2 key pair") { input := &ec2.ImportKeyPairInput{ - KeyName: &c.Spec.SSHPublicKeyName, - PublicKeyMaterial: c.Spec.SSHPublicKey, + KeyName: &ng.SSHPublicKeyName, + PublicKeyMaterial: ng.SSHPublicKey, } - logger.Info("importing SSH public key %q as %q", c.Spec.SSHPublicKeyPath, c.Spec.SSHPublicKeyName) + logger.Info("importing SSH public key %q as %q", ng.SSHPublicKeyPath, ng.SSHPublicKeyName) if _, err = c.Provider.EC2().ImportKeyPair(input); err != nil { return errors.Wrap(err, "importing SSH public key") } @@ -84,30 +87,30 @@ func (c *ClusterProvider) importSSHPublicKeyIfNeeded() error { return errors.Wrap(err, "checking existing key pair") } if *existing.KeyFingerprint != fingerprint { - return fmt.Errorf("SSH public key %s already exists, but fingerprints don't match (exected: %q, got: %q)", c.Spec.SSHPublicKeyName, fingerprint, *existing.KeyFingerprint) + return fmt.Errorf("SSH public key %s already exists, but fingerprints don't match (exected: %q, got: %q)", ng.SSHPublicKeyName, fingerprint, *existing.KeyFingerprint) } - logger.Debug("SSH public key %s already exists", c.Spec.SSHPublicKeyName) + logger.Debug("SSH public key %s already exists", ng.SSHPublicKeyName) return nil } // LoadSSHPublicKey loads the given SSH public key -func (c *ClusterProvider) LoadSSHPublicKey() error { - if !c.Spec.NodeSSH { +func (c *ClusterProvider) LoadSSHPublicKey(ng *api.NodeGroup) error { + if !ng.AllowSSH { // TODO: https://github.com/weaveworks/eksctl/issues/144 return nil } - c.Spec.SSHPublicKeyPath = utils.ExpandPath(c.Spec.SSHPublicKeyPath) - sshPublicKey, err := ioutil.ReadFile(c.Spec.SSHPublicKeyPath) + ng.SSHPublicKeyPath = utils.ExpandPath(ng.SSHPublicKeyPath) + sshPublicKey, err := ioutil.ReadFile(ng.SSHPublicKeyPath) if err != nil { if os.IsNotExist(err) { // if file not found – try to use existing EC2 key pair - return c.tryExistingSSHPublicKeyFromPath() + return c.tryExistingSSHPublicKeyFromPath(ng) } - return errors.Wrap(err, fmt.Sprintf("reading SSH public key file %q", c.Spec.SSHPublicKeyPath)) + return errors.Wrap(err, fmt.Sprintf("reading SSH public key file %q", ng.SSHPublicKeyPath)) } // on successful read – import it - c.Spec.SSHPublicKey = sshPublicKey - if err := c.importSSHPublicKeyIfNeeded(); err != nil { + ng.SSHPublicKey = sshPublicKey + if err := c.importSSHPublicKeyIfNeeded(ng); err != nil { return err } return nil @@ -121,7 +124,7 @@ func (c *ClusterProvider) MaybeDeletePublicSSHKey() { return } matching := []*string{} - prefix := c.getKeyPairName(nil) + prefix := c.getKeyPairName(nil, nil) logger.Debug("existing = %#v", existing) for _, e := range existing.KeyPairs { if strings.HasPrefix(*e.KeyName, prefix) { @@ -133,15 +136,11 @@ func (c *ClusterProvider) MaybeDeletePublicSSHKey() { } } } - if len(matching) > 1 { - logger.Debug("too many matching keys, will not delete any") - return - } - if len(matching) == 1 { + for i := range matching { input := &ec2.DeleteKeyPairInput{ - KeyName: matching[0], + KeyName: matching[i], } - logger.Debug("deleting key %q", *matching[0]) + logger.Debug("deleting key %q", *matching[i]) if _, err := c.Provider.EC2().DeleteKeyPair(input); err != nil { logger.Debug("key pair couldn't be deleted: %v", err) } diff --git a/pkg/eks/nodegroup.go b/pkg/eks/nodegroup.go index 9f68b829cd..ffe87afdec 100644 --- a/pkg/eks/nodegroup.go +++ b/pkg/eks/nodegroup.go @@ -7,6 +7,7 @@ import ( "github.com/ghodss/yaml" "github.com/kubicorn/kubicorn/pkg/logger" "github.com/pkg/errors" + "github.com/weaveworks/eksctl/pkg/eks/api" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,11 +15,11 @@ import ( clientset "k8s.io/client-go/kubernetes" ) -func (c *ClusterProvider) newNodeAuthConfigMap() (*corev1.ConfigMap, error) { +func (c *ClusterProvider) newNodeAuthConfigMap(ng *api.NodeGroup) (*corev1.ConfigMap, error) { mapRoles := make([]map[string]interface{}, 1) mapRoles[0] = make(map[string]interface{}) - mapRoles[0]["rolearn"] = c.Spec.NodeInstanceRoleARN + mapRoles[0]["rolearn"] = ng.InstanceRoleARN mapRoles[0]["username"] = "system:node:{{EC2PrivateDNSName}}" mapRoles[0]["groups"] = []string{ "system:bootstrappers", @@ -43,9 +44,9 @@ func (c *ClusterProvider) newNodeAuthConfigMap() (*corev1.ConfigMap, error) { return cm, nil } -// CreateDefaultNodeGroupAuthConfigMap creates the auth config map for the default node group -func (c *ClusterProvider) CreateDefaultNodeGroupAuthConfigMap(clientSet *clientset.Clientset) error { - cm, err := c.newNodeAuthConfigMap() +// CreateNodeGroupAuthConfigMap creates the auth config map for the default node group +func (c *ClusterProvider) CreateNodeGroupAuthConfigMap(clientSet *clientset.Clientset, ng *api.NodeGroup) error { + cm, err := c.newNodeAuthConfigMap(ng) if err != nil { return errors.Wrap(err, "constructing auth ConfigMap for DefaultNodeGroup") } @@ -82,8 +83,8 @@ func getNodes(clientSet *clientset.Clientset) (int, error) { } // WaitForNodes waits till the nodes are ready -func (c *ClusterProvider) WaitForNodes(clientSet *clientset.Clientset) error { - if c.Spec.MinNodes == 0 { +func (c *ClusterProvider) WaitForNodes(clientSet *clientset.Clientset, ng *api.NodeGroup) error { + if ng.MinSize == 0 { return nil } timer := time.After(c.Spec.WaitTimeout) @@ -98,8 +99,8 @@ func (c *ClusterProvider) WaitForNodes(clientSet *clientset.Clientset) error { return errors.Wrap(err, "listing nodes") } - logger.Info("waiting for at least %d nodes to become ready", c.Spec.MinNodes) - for !timeout && counter <= c.Spec.MinNodes { + logger.Info("waiting for at least %d nodes to become ready", ng.MinSize) + for !timeout && counter <= ng.MinSize { select { case event := <-watcher.ResultChan(): logger.Debug("event = %#v", event) @@ -119,7 +120,7 @@ func (c *ClusterProvider) WaitForNodes(clientSet *clientset.Clientset) error { } } if timeout { - return fmt.Errorf("timed out (after %s) waitiing for at least %d nodes to join the cluster and become ready", c.Spec.WaitTimeout, c.Spec.MinNodes) + return fmt.Errorf("timed out (after %s) waitiing for at least %d nodes to join the cluster and become ready", c.Spec.WaitTimeout, ng.MinSize) } if _, err = getNodes(clientSet); err != nil { diff --git a/pkg/kops/kops.go b/pkg/kops/kops.go index c3a2ed5a44..247103d070 100644 --- a/pkg/kops/kops.go +++ b/pkg/kops/kops.go @@ -55,20 +55,20 @@ func (k *Wrapper) UseVPC(spec *api.ClusterConfig) error { if len(vpcs) > 1 { return fmt.Errorf("more then one VPC found for kops cluster %q", k.clusterName) } - spec.VPC = vpcs[0] + spec.VPC.ID = vpcs[0] for _, subnet := range allSubnets { subnet := subnet.Obj.(*ec2.Subnet) for _, tag := range subnet.Tags { if k.isOwned(tag) && *subnet.VpcId == vpcs[0] { - spec.Subnets = append(spec.Subnets, *subnet.SubnetId) + spec.VPC.ImportSubnet(api.SubnetTopologyPublic, *subnet.AvailabilityZone, *subnet.SubnetId) spec.AvailabilityZones = append(spec.AvailabilityZones, *subnet.AvailabilityZone) } } } - logger.Debug("subnets = %#v", spec.Subnets) - if len(spec.Subnets) < 3 { - return fmt.Errorf("cannot use VPC from kops cluster less then 3 subnets") + logger.Debug("subnets = %#v", spec.VPC.Subnets) + if !spec.VPC.HasSufficientPublicSubnets() { + return fmt.Errorf("cannot use VPC from kops cluster with less then 3 subnets") } return nil diff --git a/pkg/nodebootstrap/userdata.go b/pkg/nodebootstrap/userdata.go index d76cc2fde4..6c5c5d7586 100644 --- a/pkg/nodebootstrap/userdata.go +++ b/pkg/nodebootstrap/userdata.go @@ -71,13 +71,14 @@ func addFilesAndScripts(config *cloudconfig.CloudConfig, files configFiles, scri return nil } -func makeAmazonLinux2Config(spec *api.ClusterConfig) (configFiles, error) { - if spec.MaxPodsPerNode == 0 { - spec.MaxPodsPerNode = maxPodsPerNodeType[spec.NodeType] +func makeAmazonLinux2Config(spec *api.ClusterConfig, nodeGroupID int) (configFiles, error) { + c := spec.NodeGroups[nodeGroupID] + if c.MaxPodsPerNode == 0 { + c.MaxPodsPerNode = maxPodsPerNodeType[c.InstanceType] } // TODO: use componentconfig or kubelet config file – https://github.com/weaveworks/eksctl/issues/156 kubeletParams := []string{ - fmt.Sprintf("MAX_PODS=%d", spec.MaxPodsPerNode), + fmt.Sprintf("MAX_PODS=%d", c.MaxPodsPerNode), // TODO: this will need to change when we provide options for using different VPCs and CIDRs – https://github.com/weaveworks/eksctl/issues/158 "CLUSTER_DNS=10.100.0.10", } @@ -113,14 +114,14 @@ func makeAmazonLinux2Config(spec *api.ClusterConfig) (configFiles, error) { } // NewUserDataForAmazonLinux2 creates new user data for Amazon Linux 2 nodes -func NewUserDataForAmazonLinux2(spec *api.ClusterConfig) (string, error) { +func NewUserDataForAmazonLinux2(spec *api.ClusterConfig, nodeGroupID int) (string, error) { config := cloudconfig.New() scripts := []string{ "bootstrap.al2.sh", } - files, err := makeAmazonLinux2Config(spec) + files, err := makeAmazonLinux2Config(spec, nodeGroupID) if err != nil { return "", err } diff --git a/pkg/utils/kubeconfig/kubeconfig_test.go b/pkg/utils/kubeconfig/kubeconfig_test.go index 55dfc14e12..d77f981e53 100644 --- a/pkg/utils/kubeconfig/kubeconfig_test.go +++ b/pkg/utils/kubeconfig/kubeconfig_test.go @@ -131,30 +131,40 @@ var _ = Describe("Kubeconfig", func() { Context("delete config", func() { // Default cluster name is 'foo' and region is 'us-west-2' var apiClusterConfigSample = eksctlapi.ClusterConfig{ - Region: "us-west-2", - Profile: "", - Tags: map[string]string{}, - ClusterName: "foo", - NodeAMI: "", - NodeType: "m5.large", - Nodes: 2, - MinNodes: 0, - MaxNodes: 0, - MaxPodsPerNode: 0, - NodePolicyARNs: []string(nil), - NodeSSH: false, - SSHPublicKeyPath: "~/.ssh/id_rsa.pub", - SSHPublicKey: []uint8(nil), - SSHPublicKeyName: "", + Region: "us-west-2", + Profile: "", + Tags: map[string]string{}, + ClusterName: "foo", + NodeGroups: []*eksctlapi.NodeGroup{ + { + AMI: "", + InstanceType: "m5.large", + AvailabilityZones: []string{"us-west-2b", "us-west-2a", "us-west-2c"}, + SubnetTopology: "Public", + AllowSSH: false, + SSHPublicKeyPath: "~/.ssh/id_rsa.pub", + SSHPublicKey: []uint8(nil), + SSHPublicKeyName: "", + DesiredCapacity: 2, + MinSize: 0, + MaxSize: 0, + MaxPodsPerNode: 0, + PolicyARNs: []string(nil), + InstanceRoleARN: "", + }, + }, + VPC: eksctlapi.ClusterVPC{ + Network: eksctlapi.Network{ + ID: "", + CIDR: nil, + }, + SecurityGroup: "", + }, WaitTimeout: 1200000000000, - SecurityGroup: "", - Subnets: []string(nil), - VPC: "", Endpoint: "", CertificateAuthorityData: []uint8(nil), ARN: "", ClusterStackName: "", - NodeInstanceRoleARN: "", AvailabilityZones: []string{"us-west-2b", "us-west-2a", "us-west-2c"}, Addons: eksctlapi.ClusterAddons{}, }