Skip to content
This repository has been archived by the owner on Oct 15, 2020. It is now read-only.

Commit

Permalink
Refactor Node object to use methods rather than Struct fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@errm
errm committed Nov 21, 2018
1 parent ba21b2c commit 077331b
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 67 deletions.
54 changes: 32 additions & 22 deletions pkg/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@ import (
// Node represents and EC2 instance.
type Node struct {
*ec2.Instance
MaxPods int
ReservedCPU string
ReservedMemory string
ClusterDNS string
Region string
Region string
}

type metadataClient interface {
Expand All @@ -61,12 +57,8 @@ func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) {
}
instance := output.Reservations[0].Instances[0]
node := Node{
Instance: instance,
MaxPods: maxPods(instance.InstanceType),
ReservedCPU: reservedCPU(instance.InstanceType),
ReservedMemory: reservedMemory(instance.InstanceType),
ClusterDNS: clusterDNS(instance.PrivateIpAddress),
Region: *region,
Instance: instance,
Region: *region,
}
if node.ClusterName() == "" {
sleepFor := b.Duration(tries)
Expand All @@ -92,6 +84,12 @@ func (n *Node) ClusterName() string {
return ""
}

// Labels returns list of kubernetes labels for this node
//
// If the node is a spot instance the node-role.kubernetes.io/spot-worker label
// will be set, otherwise the node-role.kubernetes.io/worker is set.
//
// Other custom labels can be set using EC2 tags with the k8s.io/cluster-autoscaler/node-template/label/ prefix
func (n *Node) Labels() []string {
labels := make(map[string]string)

Expand All @@ -115,13 +113,17 @@ func (n *Node) Labels() []string {
return l
}

// Spot returns true is this node is a spot instance
func (n *Node) Spot() bool {
if n.InstanceLifecycle != nil && *n.InstanceLifecycle == ec2.InstanceLifecycleTypeSpot {
return true
}
return false
}

// Taints returns a list of kuberntes taints for this node
//
// Taints can be set using EC2 tags with the k8s.io/cluster-autoscaler/node-template/taint/ prefix
func (n *Node) Taints() []string {
var taints []string
re := regexp.MustCompile(`k8s.io\/cluster-autoscaler\/node-template\/taint\/(.*)`)
Expand All @@ -141,20 +143,25 @@ func instanceID(m metadataClient) (*string, error) {
return &result, nil
}

func maxPods(instanceType *string) int {
enis := InstanceENIsAvailable[*instanceType]
ips := InstanceIPsAvailable[*instanceType]
// MaxPods returns the maximum number of pods that can be scheduled to this node
//
// see https://github.com/aws/amazon-vpc-cni-k8s#setup for more info
func (n *Node) MaxPods() int {
enis := InstanceENIsAvailable[*n.InstanceType]
ips := InstanceIPsAvailable[*n.InstanceType]
if ips == 0 {
return 0
}
return enis * (ips - 1)
}

// ReservedCPU returns the CPU in millicores that should be reserved for Kuberntes own use on this node
//
// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedCPU(instanceType *string) string {
cores := InstanceCores[*instanceType]
func (n *Node) ReservedCPU() string {
cores := InstanceCores[*n.InstanceType]
reserved := 0.0
for core := 1; core <= cores; core++ {
switch core {
Expand All @@ -169,17 +176,19 @@ func reservedCPU(instanceType *string) string {
}
}
if reserved == 0.0 {
log.Printf("The number of CPU cores is unknown for the %s instance type, --kube-reserved will not be configured", *instanceType)
log.Printf("The number of CPU cores is unknown for the %s instance type, --kube-reserved will not be configured", *n.InstanceType)
return ""
}
return fmt.Sprintf("%.0fm", reserved)
}

// ReservedMemory returns the memory that should be reserved for Kuberntes own use on this node
//
// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedMemory(instanceType *string) string {
memory := InstanceMemory[*instanceType]
func (n *Node) ReservedMemory() string {
memory := InstanceMemory[*n.InstanceType]
reserved := 0.0
for i := 0; i < memory; i++ {
switch {
Expand All @@ -196,14 +205,15 @@ func reservedMemory(instanceType *string) string {
}
}
if reserved == 0.0 {
log.Printf("The Memory of the %s instance type is unknown, --kube-reserved will not be configured", *instanceType)
log.Printf("The Memory of the %s instance type is unknown, --kube-reserved will not be configured", *n.InstanceType)
return ""
}
return fmt.Sprintf("%.0fMi", reserved)
}

func clusterDNS(ip *string) string {
if ip != nil && len(*ip) > 3 && (*ip)[0:3] == "10." {
// ClusterDNS returns the in cluster IP address that kube-dns should avalible at
func (n *Node) ClusterDNS() string {
if n.PrivateIpAddress != nil && len(*n.PrivateIpAddress) > 3 && (*n.PrivateIpAddress)[0:3] == "10." {
return "172.20.0.10"
}
return "10.100.0.10"
Expand Down
23 changes: 13 additions & 10 deletions pkg/node/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ func TestNodeLabels(t *testing.T) {
instanceLifecycle: ec2.InstanceLifecycleTypeSpot,
}
node, err = New(e, metadata, &region)
if err != nil {
t.Errorf("unexpected error: %s", err)
}

expected = []string{
"node-role.kubernetes.io/spot-worker=true",
Expand Down Expand Up @@ -165,8 +168,8 @@ func TestClusterDNS(t *testing.T) {
t.Errorf("unexpected error: %s", err)
}

if node.ClusterDNS != "172.20.0.10" {
t.Errorf("expected ClusterDNS to be 172.20.0.10 got: %s", node.ClusterDNS)
if node.ClusterDNS() != "172.20.0.10" {
t.Errorf("expected ClusterDNS to be 172.20.0.10 got: %s", node.ClusterDNS())
}

e = &mockEC2{
Expand All @@ -181,8 +184,8 @@ func TestClusterDNS(t *testing.T) {
t.Errorf("unexpected error: %s", err)
}

if node.ClusterDNS != "10.100.0.10" {
t.Errorf("expected ClusterDNS to be 10.100.0.10 got: %s", node.ClusterDNS)
if node.ClusterDNS() != "10.100.0.10" {
t.Errorf("expected ClusterDNS to be 10.100.0.10 got: %s", node.ClusterDNS())
}
}

Expand Down Expand Up @@ -295,8 +298,8 @@ func TestMaxPods(t *testing.T) {
t.Errorf("unexpected error: %s", err)
}

if node.MaxPods != test.expected {
t.Errorf("expected MaxPods for %v to be: %v, but it was %v", test.instanceType, test.expected, node.MaxPods)
if node.MaxPods() != test.expected {
t.Errorf("expected MaxPods for %v to be: %v, but it was %v", test.instanceType, test.expected, node.MaxPods())
}
}
}
Expand Down Expand Up @@ -371,8 +374,8 @@ func TestReservedCPU(t *testing.T) {
t.Errorf("unexpected error: %s", err)
}

if node.ReservedCPU != test.expected {
t.Errorf("expected ReservedCPU for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedCPU)
if node.ReservedCPU() != test.expected {
t.Errorf("expected ReservedCPU for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedCPU())
}
}
}
Expand Down Expand Up @@ -442,8 +445,8 @@ func TestMemory(t *testing.T) {
t.Errorf("unexpected error: %s", err)
}

if node.ReservedMemory != test.expected {
t.Errorf("expected ReservedMemory for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedMemory)
if node.ReservedMemory() != test.expected {
t.Errorf("expected ReservedMemory for %v to be: %v, but it was %v", test.instanceType, test.expected, node.ReservedMemory())
}
}
}
Expand Down
46 changes: 11 additions & 35 deletions pkg/system/system_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestConfigure(t *testing.T) {
hn := &FakeHostname{}
init := &FakeInit{}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "60m", "960Mi", map[string]string{}, false)
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", map[string]string{}, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand Down Expand Up @@ -117,12 +117,12 @@ Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --cluster-dns=172.20.0.10 --pod-
fs.Check(t, "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf", expected, 0640)

expected = `[Service]
Environment='KUBELET_MAX_PODS=--max-pods=18'
Environment='KUBELET_MAX_PODS=--max-pods=27'
`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-max-pods.conf", expected, 0640)

expected = `[Service]
Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=60m,memory=960Mi'
Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=70m,memory=1024Mi'
`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640)

Expand Down Expand Up @@ -151,38 +151,16 @@ Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/worker=t
}
}

func TestConfigureNoReserved(t *testing.T) {
fs := &FakeFileSystem{}
hn := &FakeHostname{}
init := &FakeInit{}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", map[string]string{}, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
"dGhpc2lzdGhlY2VydGRhdGE=",
)
system := System{Filesystem: fs, Hostname: hn, Init: init}
err := system.Configure(i, c)

if err != nil {
t.Errorf("unexpected error %v", err)
}

expected := `[Service]`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640)
}

func TestConfigureSpotInstanceLabels(t *testing.T) {
fs := &FakeFileSystem{}
hn := &FakeHostname{}
init := &FakeInit{}

labels := map[string]string{
tags := map[string]string{
"node-role.kubernetes.io/worker": "true",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", labels, true)
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, true)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand Down Expand Up @@ -210,7 +188,7 @@ func TestConfigureLabels(t *testing.T) {
"k8s.io/cluster-autoscaler/node-template/label/gpu-type": "K80",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", tags, false)
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand Down Expand Up @@ -238,7 +216,7 @@ func TestConfigureTaints(t *testing.T) {
"k8s.io/cluster-autoscaler/node-template/taint/node-role.kubernetes.io/worker": "true:PreferNoSchedule",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", tags, false)
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", tags, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand All @@ -257,7 +235,7 @@ Environment='KUBELET_NODE_TAINTS=--register-with-taints="node-role.kubernetes.io
fs.Check(t, "/etc/systemd/system/kubelet.service.d/50-taints.conf", expected, 0640)
}

func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory string, tags map[string]string, spot bool) *node.Node {
func instance(ip, dnsName string, tags map[string]string, spot bool) *node.Node {
var ec2tags []*ec2.Tag
for key, value := range tags {
ec2tags = append(ec2tags, &ec2.Tag{
Expand All @@ -270,18 +248,16 @@ func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory strin
il := ec2.InstanceLifecycleTypeSpot
instanceLifecycle = &il
}
instanceType := "c5.large"
return &node.Node{
Instance: &ec2.Instance{
PrivateIpAddress: &ip,
PrivateDnsName: &dnsName,
Tags: ec2tags,
InstanceType: &instanceType,
InstanceLifecycle: instanceLifecycle,
},
MaxPods: maxPods,
ClusterDNS: "172.20.0.10",
Region: "us-east-1",
ReservedCPU: reservedCPU,
ReservedMemory: reservedMemory,
Region: "us-east-1",
}
}

Expand Down

0 comments on commit 077331b

Please sign in to comment.