Merge pull request #703 from weaveworks/clusterconfig-taints

Add taints to nodegroup config

examples/05-advanced-nodegroups.yaml

@@ -34,6 +34,8 @@ nodeGroups:
     desiredCapacity: 4
     labels:
       nodegroup-type: very-special-science-workloads
+    taints:
+      special: "true:NoSchedule"
     privateNetworking: true
     availabilityZones: ["eu-west-2a"] # use single AZ to optimise data transfer between isntances
     preBootstrapCommand:

pkg/apis/eksctl.io/v1alpha4/types.go

@@ -380,6 +380,9 @@ type NodeGroup struct {
 	// +optional
 	Labels map[string]string `json:"labels,omitempty"`
 
+	// +optional
+	Taints map[string]string `json:"taints,omitempty"`
+
 	// TODO move to separate struct
 	// +optional
 	AllowSSH bool `json:"allowSSH"`

pkg/cfn/builder/api_test.go

@@ -997,6 +997,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=",
+				"NODE_TAINTS=",
 			}))
 
 			kubeletDropInUnit := getFile(cc, "/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf")
@@ -1066,6 +1067,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=",
+				"NODE_TAINTS=",
 			}))
 
 			kubeletDropInUnit := getFile(cc, "/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf")
@@ -1102,6 +1104,9 @@ var _ = Describe("CloudFormation template builder API", func() {
 		ng.Labels = map[string]string{
 			"os": "al2",
 		}
+		ng.Taints = map[string]string{
+			"key1": "value1:NoSchedule",
+		}
 
 		ng.OverrideBootstrapCommand = &overrideBootstrapCommand
 
@@ -1131,6 +1136,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=os=al2",
+				"NODE_TAINTS=key1=value1:NoSchedule",
 			}))
 
 			kubeletDropInUnit := getFile(cc, "/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf")
@@ -1197,6 +1203,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=os=al2",
+				"NODE_TAINTS=",
 			}))
 
 			kubeletDropInUnit := getFile(cc, "/etc/systemd/system/kubelet.service.d/10-eksclt.al2.conf")
@@ -1270,6 +1277,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=",
+				"NODE_TAINTS=",
 				"MAX_PODS=29",
 				"CLUSTER_DNS=172.20.0.10",
 			}))
@@ -1329,6 +1337,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=",
+				"NODE_TAINTS=",
 				"MAX_PODS=29",
 				"CLUSTER_DNS=172.20.0.10",
 			}))
@@ -1364,6 +1373,9 @@ var _ = Describe("CloudFormation template builder API", func() {
 		ng.Labels = map[string]string{
 			"os": "ubuntu",
 		}
+		ng.Taints = map[string]string{
+			"key1": "value1:NoSchedule",
+		}
 
 		ng.ClusterDNS = "169.254.20.10"
 
@@ -1399,6 +1411,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=os=ubuntu",
+				"NODE_TAINTS=key1=value1:NoSchedule",
 				"MAX_PODS=29",
 				"CLUSTER_DNS=169.254.20.10",
 			}))
@@ -1470,6 +1483,7 @@ var _ = Describe("CloudFormation template builder API", func() {
 			Expect(kubeletEnv.Permissions).To(Equal("0644"))
 			Expect(strings.Split(kubeletEnv.Content, "\n")).To(Equal([]string{
 				"NODE_LABELS=os=ubuntu",
+				"NODE_TAINTS=",
 				"MAX_PODS=29",
 				"CLUSTER_DNS=169.254.20.10",
 			}))

pkg/nodebootstrap/assets/10-eksclt.al2.conf

@@ -3,7 +3,7 @@
 [Service]
 # Local metadata parameters: REGION, AWS_DEFAULT_REGION
 EnvironmentFile=/etc/eksctl/metadata.env
-# Global and static parameters: MAX_PODS, CLUSTER_DNS, NODE_LABELS
+# Global and static parameters: MAX_PODS, CLUSTER_DNS, NODE_LABELS, NODE_TAINTS
 EnvironmentFile=/etc/eksctl/kubelet.env
 # Local non-static parameters: NODE_IP, INSTANCE_ID
 EnvironmentFile=/etc/eksctl/kubelet.local.env
@@ -12,6 +12,7 @@ ExecStart=
 ExecStart=/usr/bin/kubelet \
   --node-ip=${NODE_IP} \
   --node-labels=${NODE_LABELS},alpha.eksctl.io/instance-id=${INSTANCE_ID} \
+  --register-node=true --register-with-taints=${NODE_TAINTS} \
   --allow-privileged=true \
   --cloud-provider=aws \
   --container-runtime=docker \

pkg/nodebootstrap/assets/bootstrap.ubuntu.sh

@@ -36,6 +36,7 @@ systemctl reset-failed
     "network-plugin=cni"
     "cgroup-driver=cgroupfs"
     "register-node=true"
+    "register-with-taints=${NODE_TAINTS}"
     "kubeconfig=/etc/eksctl/kubeconfig.yaml"
     "feature-gates=RotateKubeletServerCertificate=true"
     "anonymous-auth=false"

pkg/nodebootstrap/userdata.go

@@ -136,13 +136,17 @@ func makeKubeletConfigYAML(spec *api.ClusterConfig, ng *api.NodeGroup) ([]byte,
 }
 
 func makeCommonKubeletEnvParams(spec *api.ClusterConfig, ng *api.NodeGroup) []string {
-	labels := []string{}
-	for k, v := range ng.Labels {
-		labels = append(labels, fmt.Sprintf("%s=%s", k, v))
+	kvs := func(kv map[string]string) string {
+		var params []string
+		for k, v := range kv {
+			params = append(params, fmt.Sprintf("%s=%s", k, v))
+		}
+		return strings.Join(params, ",")
 	}
 
 	return []string{
-		fmt.Sprintf("NODE_LABELS=%s", strings.Join(labels, ",")),
+		fmt.Sprintf("NODE_LABELS=%s", kvs(ng.Labels)),
+		fmt.Sprintf("NODE_TAINTS=%s", kvs(ng.Taints)),
 	}
 }