diff --git a/pkg/node/node.go b/pkg/node/node.go index 3a1b975..b08a153 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -31,11 +31,7 @@ import ( // Node represents and EC2 instance. type Node struct { *ec2.Instance - MaxPods int - ReservedCPU string - ReservedMemory string - ClusterDNS string - Region string + Region string } type metadataClient interface { @@ -61,12 +57,8 @@ func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) { } instance := output.Reservations[0].Instances[0] node := Node{ - Instance: instance, - MaxPods: maxPods(instance.InstanceType), - ReservedCPU: reservedCPU(instance.InstanceType), - ReservedMemory: reservedMemory(instance.InstanceType), - ClusterDNS: clusterDNS(instance.PrivateIpAddress), - Region: *region, + Instance: instance, + Region: *region, } if node.ClusterName() == "" { sleepFor := b.Duration(tries) @@ -92,6 +84,12 @@ func (n *Node) ClusterName() string { return "" } +// Labels returns list of kubernetes labels for this node +// +// If the node is a spot instance the node-role.kubernetes.io/spot-worker label +// will be set, otherwise the node-role.kubernetes.io/worker is set. +// +// Other custom labels can be set using EC2 tags with the k8s.io/cluster-autoscaler/node-template/label/ prefix func (n *Node) Labels() []string { labels := make(map[string]string) @@ -115,6 +113,7 @@ func (n *Node) Labels() []string { return l } +// Spot returns true is this node is a spot instance func (n *Node) Spot() bool { if n.InstanceLifecycle != nil && *n.InstanceLifecycle == ec2.InstanceLifecycleTypeSpot { return true @@ -122,6 +121,9 @@ func (n *Node) Spot() bool { return false } +// Taints returns a list of kuberntes taints for this node +// +// Taints can be set using EC2 tags with the k8s.io/cluster-autoscaler/node-template/taint/ prefix func (n *Node) Taints() []string { var taints []string re := regexp.MustCompile(`k8s.io\/cluster-autoscaler\/node-template\/taint\/(.*)`) @@ -141,20 +143,25 @@ func instanceID(m metadataClient) (*string, error) { return &result, nil } -func maxPods(instanceType *string) int { - enis := InstanceENIsAvailable[*instanceType] - ips := InstanceIPsAvailable[*instanceType] +// MaxPods returns the maximum number of pods that can be scheduled to this node +// +// see https://github.com/aws/amazon-vpc-cni-k8s#setup for more info +func (n *Node) MaxPods() int { + enis := InstanceENIsAvailable[*n.InstanceType] + ips := InstanceIPsAvailable[*n.InstanceType] if ips == 0 { return 0 } return enis * (ips - 1) } +// ReservedCPU returns the CPU in millicores that should be reserved for Kuberntes own use on this node +// // The calculation here is based on information found in the GKE documentation // here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture // I think that it should also apply to AWS -func reservedCPU(instanceType *string) string { - cores := InstanceCores[*instanceType] +func (n *Node) ReservedCPU() string { + cores := InstanceCores[*n.InstanceType] reserved := 0.0 for core := 1; core <= cores; core++ { switch core { @@ -169,17 +176,19 @@ func reservedCPU(instanceType *string) string { } } if reserved == 0.0 { - log.Printf("The number of CPU cores is unknown for the %s instance type, --kube-reserved will not be configured", *instanceType) + log.Printf("The number of CPU cores is unknown for the %s instance type, --kube-reserved will not be configured", *n.InstanceType) return "" } return fmt.Sprintf("%.0fm", reserved) } +// ReservedMemory returns the memory that should be reserved for Kuberntes own use on this node +// // The calculation here is based on information found in the GKE documentation // here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture // I think that it should also apply to AWS -func reservedMemory(instanceType *string) string { - memory := InstanceMemory[*instanceType] +func (n *Node) ReservedMemory() string { + memory := InstanceMemory[*n.InstanceType] reserved := 0.0 for i := 0; i < memory; i++ { switch { @@ -196,14 +205,15 @@ func reservedMemory(instanceType *string) string { } } if reserved == 0.0 { - log.Printf("The Memory of the %s instance type is unknown, --kube-reserved will not be configured", *instanceType) + log.Printf("The Memory of the %s instance type is unknown, --kube-reserved will not be configured", *n.InstanceType) return "" } return fmt.Sprintf("%.0fMi", reserved) } -func clusterDNS(ip *string) string { - if ip != nil && len(*ip) > 3 && (*ip)[0:3] == "10." { +// ClusterDNS returns the in cluster IP address that kube-dns should avalible at +func (n *Node) ClusterDNS() string { + if n.PrivateIpAddress != nil && len(*n.PrivateIpAddress) > 3 && (*n.PrivateIpAddress)[0:3] == "10." { return "172.20.0.10" } return "10.100.0.10" diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index cdf6b5e..640749c 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -107,6 +107,9 @@ func TestNodeLabels(t *testing.T) { instanceLifecycle: ec2.InstanceLifecycleTypeSpot, } node, err = New(e, metadata, ®ion) + if err != nil { + t.Errorf("unexpected error: %s", err) + } expected = []string{ "node-role.kubernetes.io/spot-worker=true", @@ -165,8 +168,8 @@ func TestClusterDNS(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if node.ClusterDNS != "172.20.0.10" { - t.Errorf("expected ClusterDNS to be 172.20.0.10 got: %s", node.ClusterDNS) + if node.ClusterDNS() != "172.20.0.10" { + t.Errorf("expected ClusterDNS to be 172.20.0.10 got: %s", node.ClusterDNS()) } e = &mockEC2{ @@ -181,8 +184,8 @@ func TestClusterDNS(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if node.ClusterDNS != "10.100.0.10" { - t.Errorf("expected ClusterDNS to be 10.100.0.10 got: %s", node.ClusterDNS) + if node.ClusterDNS() != "10.100.0.10" { + t.Errorf("expected ClusterDNS to be 10.100.0.10 got: %s", node.ClusterDNS()) } } @@ -295,8 +298,8 @@ func TestMaxPods(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if node.MaxPods != test.expected { - t.Errorf("expected MaxPods for %v to be: %v, but it was %v", test.instanceType, test.expected, node.MaxPods) + if node.MaxPods() != test.expected { + t.Errorf("expected MaxPods for %v to be: %v, but it was %v", test.instanceType, test.expected, node.MaxPods()) } } } @@ -371,8 +374,8 @@ func TestReservedCPU(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if node.ReservedCPU != test.expected { - t.Errorf("expected ReservedCPU for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedCPU) + if node.ReservedCPU() != test.expected { + t.Errorf("expected ReservedCPU for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedCPU()) } } } @@ -442,8 +445,8 @@ func TestMemory(t *testing.T) { t.Errorf("unexpected error: %s", err) } - if node.ReservedMemory != test.expected { - t.Errorf("expected ReservedMemory for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedMemory) + if node.ReservedMemory() != test.expected { + t.Errorf("expected ReservedMemory for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedMemory()) } } } diff --git a/pkg/system/system_test.go b/pkg/system/system_test.go index 5cd71ad..be5320d 100644 --- a/pkg/system/system_test.go +++ b/pkg/system/system_test.go @@ -33,7 +33,7 @@ func TestConfigure(t *testing.T) { hn := &FakeHostname{} init := &FakeInit{} - i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "60m", "960Mi", map[string]string{}, false) + i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", map[string]string{}, false) c := cluster( "aws-om-cluster", "https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com", @@ -117,12 +117,12 @@ Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --cluster-dns=172.20.0.10 --pod- fs.Check(t, "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf", expected, 0640) expected = `[Service] -Environment='KUBELET_MAX_PODS=--max-pods=18' +Environment='KUBELET_MAX_PODS=--max-pods=27' ` fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-max-pods.conf", expected, 0640) expected = `[Service] -Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=60m,memory=960Mi' +Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=70m,memory=1024Mi' ` fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640) @@ -151,38 +151,16 @@ Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/worker=t } } -func TestConfigureNoReserved(t *testing.T) { - fs := &FakeFileSystem{} - hn := &FakeHostname{} - init := &FakeInit{} - - i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", map[string]string{}, false) - c := cluster( - "aws-om-cluster", - "https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com", - "dGhpc2lzdGhlY2VydGRhdGE=", - ) - system := System{Filesystem: fs, Hostname: hn, Init: init} - err := system.Configure(i, c) - - if err != nil { - t.Errorf("unexpected error %v", err) - } - - expected := `[Service]` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640) -} - func TestConfigureSpotInstanceLabels(t *testing.T) { fs := &FakeFileSystem{} hn := &FakeHostname{} init := &FakeInit{} - labels := map[string]string{ + tags := map[string]string{ "node-role.kubernetes.io/worker": "true", } - i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", labels, true) + i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, true) c := cluster( "aws-om-cluster", "https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com", @@ -210,7 +188,7 @@ func TestConfigureLabels(t *testing.T) { "k8s.io/cluster-autoscaler/node-template/label/gpu-type": "K80", } - i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", tags, false) + i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, false) c := cluster( "aws-om-cluster", "https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com", @@ -238,7 +216,7 @@ func TestConfigureTaints(t *testing.T) { "k8s.io/cluster-autoscaler/node-template/taint/node-role.kubernetes.io/worker": "true:PreferNoSchedule", } - i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", tags, false) + i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, false) c := cluster( "aws-om-cluster", "https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com", @@ -257,7 +235,7 @@ Environment='KUBELET_NODE_TAINTS=--register-with-taints="node-role.kubernetes.io fs.Check(t, "/etc/systemd/system/kubelet.service.d/50-taints.conf", expected, 0640) } -func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory string, tags map[string]string, spot bool) *node.Node { +func instance(ip, dnsName string, tags map[string]string, spot bool) *node.Node { var ec2tags []*ec2.Tag for key, value := range tags { ec2tags = append(ec2tags, &ec2.Tag{ @@ -270,18 +248,16 @@ func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory strin il := ec2.InstanceLifecycleTypeSpot instanceLifecycle = &il } + instanceType := "c5.large" return &node.Node{ Instance: &ec2.Instance{ PrivateIpAddress: &ip, PrivateDnsName: &dnsName, Tags: ec2tags, + InstanceType: &instanceType, InstanceLifecycle: instanceLifecycle, }, - MaxPods: maxPods, - ClusterDNS: "172.20.0.10", - Region: "us-east-1", - ReservedCPU: reservedCPU, - ReservedMemory: reservedMemory, + Region: "us-east-1", } }