Skip to content
This repository has been archived by the owner on Oct 15, 2020. It is now read-only.

Commit

Permalink
Automate adding the node-role.kubernetes.io label
Browse files Browse the repository at this point in the history
  • Loading branch information
@errm
errm committed Nov 20, 2018
1 parent a5d395a commit c6e9fcf
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 34 deletions.
29 changes: 23 additions & 6 deletions pkg/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ type Node struct {
ReservedMemory string
ClusterDNS string
Region string
Labels []string
Taints []string
}

Expand Down Expand Up @@ -69,7 +68,6 @@ func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) {
ReservedMemory: reservedMemory(instance.InstanceType),
ClusterDNS: clusterDNS(instance.PrivateIpAddress),
Region: *region,
Labels: lables(instance.Tags),
Taints: taints(instance.Tags),
}
if node.ClusterName() == "" {
Expand All @@ -96,17 +94,36 @@ func (n *Node) ClusterName() string {
return ""
}

func lables(tags []*ec2.Tag) []string {
var l []string
func (n *Node) Labels() []string {
labels := make(map[string]string)

if n.Spot() {
labels["node-role.kubernetes.io/spot-worker"] = "true"
} else {
labels["node-role.kubernetes.io/worker"] = "true"
}

re := regexp.MustCompile(`k8s.io\/cluster-autoscaler\/node-template\/label\/(.*)`)
for _, t := range tags {
for _, t := range n.Tags {
if matches := re.FindStringSubmatch(*t.Key); len(matches) == 2 {
l = append(l, matches[1]+"="+*t.Value)
labels[matches[1]] = *t.Value
}
}

l := make([]string, 0, len(labels))
for key, value := range labels {
l = append(l, key+"="+value)
}
return l
}

func (n *Node) Spot() bool {
if n.InstanceLifecycle != nil && *n.InstanceLifecycle == ec2.InstanceLifecycleTypeSpot {
return true
}
return false
}

func taints(tags []*ec2.Tag) []string {
var ts []string
re := regexp.MustCompile(`k8s.io\/cluster-autoscaler\/node-template\/taint\/(.*)`)
Expand Down
48 changes: 38 additions & 10 deletions pkg/node/node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ func TestNodeLabels(t *testing.T) {
{},
{
tag("kubernetes.io/cluster/cluster-name", "owned"),
tag("k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/spot-worker", "true"),
tag("k8s.io/cluster-autoscaler/node-template/label/nvidia-gpu", "K80"),
},
},
Expand All @@ -88,12 +87,34 @@ func TestNodeLabels(t *testing.T) {
}

expected := []string{
"node-role.kubernetes.io/worker=true",
"nvidia-gpu=K80",
}

if !reflect.DeepEqual(node.Labels(), expected) {
t.Errorf("Expected node.Labels to be %v but was %v", expected, node.Labels())
}

e = &mockEC2{
tags: [][]*ec2.Tag{
{},
{},
{
tag("kubernetes.io/cluster/cluster-name", "owned"),
tag("k8s.io/cluster-autoscaler/node-template/label/nvidia-gpu", "K80"),
},
},
instanceLifecycle: ec2.InstanceLifecycleTypeSpot,
}
node, err = New(e, metadata, &region)

expected = []string{
"node-role.kubernetes.io/spot-worker=true",
"nvidia-gpu=K80",
}

if !reflect.DeepEqual(node.Labels, expected) {
t.Errorf("Expected node.Labels to be %v but was %v", expected, node.Labels)
if !reflect.DeepEqual(node.Labels(), expected) {
t.Errorf("Expected node.Labels to be %v but was %v", expected, node.Labels())
}
}

Expand Down Expand Up @@ -437,9 +458,10 @@ func tag(key, value string) *ec2.Tag {
type mockEC2 struct {
PrivateIPAddress string
ec2iface.EC2API
tags [][]*ec2.Tag
instanceType string
err error
tags [][]*ec2.Tag
instanceType string
instanceLifecycle string
err error
}

func (m *mockEC2) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
Expand All @@ -449,15 +471,21 @@ func (m *mockEC2) DescribeInstances(input *ec2.DescribeInstancesInput) (*ec2.Des
var tags []*ec2.Tag
//Pop the first set of tags
tags, m.tags = m.tags[0], m.tags[1:]

var instanceLifecycle *string
if m.instanceLifecycle != "" {
instanceLifecycle = &m.instanceLifecycle
}
if len(input.InstanceIds) > 0 {
return &ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{{
Instances: []*ec2.Instance{
{
InstanceId: input.InstanceIds[0],
Tags: tags,
InstanceType: &m.instanceType,
PrivateIpAddress: &m.PrivateIPAddress,
InstanceId: input.InstanceIds[0],
Tags: tags,
InstanceType: &m.instanceType,
PrivateIpAddress: &m.PrivateIPAddress,
InstanceLifecycle: instanceLifecycle,
},
},
},
Expand Down
50 changes: 32 additions & 18 deletions pkg/system/system_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestConfigure(t *testing.T) {
hn := &FakeHostname{}
init := &FakeInit{}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "60m", "960Mi", []string{}, []string{})
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "60m", "960Mi", map[string]string{}, []string{}, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand Down Expand Up @@ -126,7 +126,9 @@ Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=60m,memory=960Mi'
`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640)

expected = `[Service]`
expected = `[Service]
Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/worker=true"'
`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/40-labels.conf", expected, 0640)

expected = `[Service]`
Expand Down Expand Up @@ -154,7 +156,7 @@ func TestConfigureNoReserved(t *testing.T) {
hn := &FakeHostname{}
init := &FakeInit{}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", []string{}, []string{})
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", map[string]string{}, []string{}, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand All @@ -171,16 +173,16 @@ func TestConfigureNoReserved(t *testing.T) {
fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640)
}

func TestConfigureLabels(t *testing.T) {
func TestConfigureSpotInstanceLabels(t *testing.T) {
fs := &FakeFileSystem{}
hn := &FakeHostname{}
init := &FakeInit{}

labels := []string{
"node-role.kubernetes.io/worker=true",
labels := map[string]string{
"node-role.kubernetes.io/worker": "true",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", labels, []string{})
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", labels, []string{}, true)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand All @@ -194,22 +196,21 @@ func TestConfigureLabels(t *testing.T) {
}

expected := `[Service]
Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/worker=true"'
Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/spot-worker=true"'
`
fs.Check(t, "/etc/systemd/system/kubelet.service.d/40-labels.conf", expected, 0640)
}

func TestConfigureMultipleLabels(t *testing.T) {
func TestConfigureLabels(t *testing.T) {
fs := &FakeFileSystem{}
hn := &FakeHostname{}
init := &FakeInit{}

labels := []string{
"node-role.kubernetes.io/worker=true",
"gpu-type=K80",
tags := map[string]string{
"k8s.io/cluster-autoscaler/node-template/label/gpu-type": "K80",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", labels, []string{})
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", tags, []string{}, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand Down Expand Up @@ -237,7 +238,7 @@ func TestConfigureTaints(t *testing.T) {
"node-role.kubernetes.io/worker=true:PreferNoSchedule",
}

i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", []string{}, taints)
i := instance("10.6.28.199", "ip-10-6-28-199.us-west-2.compute.internal", 18, "", "", map[string]string{}, taints, false)
c := cluster(
"aws-om-cluster",
"https://74770F6B05F7A8FB0F02CFB5F7AF530C.yl4.us-west-2.eks.amazonaws.com",
Expand All @@ -256,18 +257,31 @@ Environment='KUBELET_NODE_TAINTS=--register-with-taints="node-role.kubernetes.io
fs.Check(t, "/etc/systemd/system/kubelet.service.d/50-taints.conf", expected, 0640)
}

func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory string, labels, taints []string) *node.Node {
func instance(ip, dnsName string, maxPods int, reservedCPU, reservedMemory string, tags map[string]string, taints []string, spot bool) *node.Node {
var ec2tags []*ec2.Tag
for key, value := range tags {
ec2tags = append(ec2tags, &ec2.Tag{
Key: &key,
Value: &value,
})
}
var instanceLifecycle *string
if spot {
il := ec2.InstanceLifecycleTypeSpot
instanceLifecycle = &il
}
return &node.Node{
Instance: &ec2.Instance{
PrivateIpAddress: &ip,
PrivateDnsName: &dnsName,
PrivateIpAddress: &ip,
PrivateDnsName: &dnsName,
Tags: ec2tags,
InstanceLifecycle: instanceLifecycle,
},
MaxPods: maxPods,
ClusterDNS: "172.20.0.10",
Region: "us-east-1",
ReservedCPU: reservedCPU,
ReservedMemory: reservedMemory,
Labels: labels,
Taints: taints,
}
}
Expand Down

0 comments on commit c6e9fcf

Please sign in to comment.