Skip to content
This repository has been archived by the owner on Oct 15, 2020. It is now read-only.

Commit

Permalink
Calculate a resonable value for the --kube-reserved flag
Browse files Browse the repository at this point in the history 
Kubernetes uses --kube-reserved to calculate the alocatable resources
avalible on a node.

This code uses the calucations documented by GKE
here : https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
to calculate appropriate values for these flags, for the current instance type.
  • Loading branch information
@errm
errm committed Sep 17, 2018
1 parent a1582f1 commit d9104ed
Show file tree
Hide file tree
Showing 7 changed files with 495 additions and 12 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ When run on an ec2 node ekstrap performs several tasks.
* Writes a kubeconfig file configured to connect to your EKS cluster to `/var/lib/kubelet/kubeconfig`.
* Writes a systemd unit file to `/lib/systemd/system/kubelet.service`.
* Writes the cluster CA certificate to `/etc/kubernetes/pki/ca.crt`.
* Calculates an appropriate value for for [--kube-reserved](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/)
* Restarts the kubelet unit.

In order to run ekstrap your instance should have an IAM instance profile that allows the `EC2::DescribeInstances` action and the `EKS::DescribeCluster` action. Both of these actions are already included in the AWS managed policy `arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy` along with the other permissions that the kubelet requires to connect to your cluster, it is recommended therefore to simply attach this policy to your instance role/profile.
Expand All @@ -25,7 +26,7 @@ If you wish to provide extra aruguments to the kubelet you can create a drop-in

For example to [taint nodes with GPU hardware](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#example-use-cases) you could add:

_/etc/systemd/system/kubelet.service.d/30-kubelet-extra-args.conf_
_/etc/systemd/system/kubelet.service.d/40-kubelet-extra-args.conf_
```
[Service]
Environment='KUBELET_EXTRA_ARGS=--register-with-taints="gpu=true:PreferNoSchedule"'
Expand Down
273 changes: 273 additions & 0 deletions pkg/node/ec2_resources.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
/*
Copyright 2018 Edward Robinson.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

// InstanceCores contains a mapping of instance types to the number of cores
// Based on info from https://aws.amazon.com/ec2/instance-types/ and
// https://aws.amazon.com/ec2/previous-generation/
var InstanceCores = map[string]int{
"c1.medium": 2,
"c1.xlarge": 8,
"c3.large": 2,
"c3.xlarge": 4,
"c3.2xlarge": 8,
"c3.4xlarge": 16,
"c3.8xlarge": 32,
"c4.large": 2,
"c4.xlarge": 4,
"c4.2xlarge": 8,
"c4.4xlarge": 16,
"c4.8xlarge": 32,
"c5.large": 2,
"c5.xlarge": 4,
"c5.2xlarge": 8,
"c5.4xlarge": 16,
"c5.9xlarge": 36,
"c5.18xlarge": 72,
"c5d.large": 2,
"c5d.xlarge": 4,
"c5d.2xlarge": 8,
"c5d.4xlarge": 16,
"c5d.9xlarge": 36,
"c5d.18xlarge": 72,
"cc2.8xlarge": 32,
"cr1.8xlarge": 32,
"d2.xlarge": 4,
"d2.2xlarge": 8,
"d2.4xlarge": 16,
"d2.8xlarge": 36,
"f1.2xlarge": 8,
"f1.16xlarge": 64,
"g2.2xlarge": 8,
"g2.8xlarge": 32,
"g3.4xlarge": 16,
"g3.8xlarge": 32,
"g3.16xlarge": 64,
"h1.2xlarge": 8,
"h1.4xlarge": 16,
"h1.8xlarge": 32,
"h1.16xlarge": 64,
"hs1.8xlarge": 16,
"i2.xlarge": 4,
"i2.2xlarge": 8,
"i2.4xlarge": 16,
"i2.8xlarge": 32,
"i3.large": 2,
"i3.xlarge": 4,
"i3.2xlarge": 8,
"i3.4xlarge": 16,
"i3.8xlarge": 32,
"i3.16xlarge": 64,
"i3.metal": 72,
"m1.small": 1,
"m1.medium": 1,
"m1.large": 2,
"m1.xlarge": 4,
"m2.xlarge": 2,
"m2.2xlarge": 4,
"m2.4xlarge": 8,
"m3.medium": 1,
"m3.large": 2,
"m3.xlarge": 4,
"m3.2xlarge": 8,
"m4.large": 2,
"m4.xlarge": 4,
"m4.2xlarge": 8,
"m4.4xlarge": 16,
"m4.10xlarge": 40,
"m4.16xlarge": 64,
"m5.large": 2,
"m5.xlarge": 4,
"m5.2xlarge": 8,
"m5.4xlarge": 16,
"m5.12xlarge": 48,
"m5.24xlarge": 96,
"m5d.large": 2,
"m5d.xlarge": 4,
"m5d.2xlarge": 8,
"m5d.4xlarge": 16,
"m5d.12xlarge": 48,
"m5d.24xlarge": 96,
"p2.xlarge": 4,
"p2.8xlarge": 32,
"p2.16xlarge": 64,
"p3.2xlarge": 8,
"p3.8xlarge": 32,
"p3.16xlarge": 64,
"r3.large": 2,
"r3.xlarge": 4,
"r3.2xlarge": 8,
"r3.4xlarge": 16,
"r3.8xlarge": 32,
"r4.large": 2,
"r4.xlarge": 4,
"r4.2xlarge": 8,
"r4.4xlarge": 16,
"r4.8xlarge": 32,
"r4.16xlarge": 64,
"t1.micro": 1,
"t2.nano": 1,
"t2.micro": 1,
"t2.small": 1,
"t2.medium": 2,
"t2.large": 2,
"t2.xlarge": 4,
"t2.2xlarge": 8,
"t3.nano": 2,
"t3.micro": 2,
"t3.small": 2,
"t3.medium": 2,
"t3.large": 2,
"t3.xlarge": 4,
"t3.2xlarge": 8,
"x1.16xlarge": 64,
"x1.32xlarge": 128,
"x1e.xlarge": 4,
"x1e.2xlarge": 8,
"x1e.4xlarge": 16,
"x1e.8xlarge": 32,
"x1e.16xlarge": 64,
"x1e.32xlarge": 128,
}

// InstanceMemory contains a mapping of instance types to the amount of memory
// Based on info from https://aws.amazon.com/ec2/instance-types/ and
// https://aws.amazon.com/ec2/previous-generation/
var InstanceMemory = map[string]int{
"c1.medium": 1740,
"c1.xlarge": 7168,
"c3.large": 3840,
"c3.xlarge": 7680,
"c3.2xlarge": 15360,
"c3.4xlarge": 30720,
"c3.8xlarge": 61440,
"c4.large": 3840,
"c4.xlarge": 7680,
"c4.2xlarge": 15360,
"c4.4xlarge": 30720,
"c4.8xlarge": 61440,
"c5.large": 4096,
"c5.xlarge": 8192,
"c5.2xlarge": 16384,
"c5.4xlarge": 32768,
"c5.9xlarge": 73728,
"c5.18xlarge": 147456,
"c5d.large": 4096,
"c5d.xlarge": 8192,
"c5d.2xlarge": 16384,
"c5d.4xlarge": 32768,
"c5d.9xlarge": 73728,
"c5d.18xlarge": 147456,
"cc2.8xlarge": 61952,
"cr1.8xlarge": 249856,
"d2.xlarge": 31232,
"d2.2xlarge": 62464,
"d2.4xlarge": 124928,
"d2.8xlarge": 249856,
"f1.2xlarge": 124928,
"f1.16xlarge": 999424,
"g2.2xlarge": 15360,
"g2.8xlarge": 61440,
"g3.4xlarge": 124928,
"g3.8xlarge": 249856,
"g3.16xlarge": 499712,
"h1.2xlarge": 32768,
"h1.4xlarge": 65536,
"h1.8xlarge": 131072,
"h1.16xlarge": 262144,
"hs1.8xlarge": 119808,
"i2.xlarge": 31232,
"i2.2xlarge": 62464,
"i2.4xlarge": 124928,
"i2.8xlarge": 249856,
"i3.large": 15616,
"i3.xlarge": 31232,
"i3.2xlarge": 62464,
"i3.4xlarge": 124928,
"i3.8xlarge": 249856,
"i3.16xlarge": 499712,
"i3.metal": 524288,
"m1.small": 1740,
"m1.medium": 3840,
"m1.large": 7680,
"m1.xlarge": 15360,
"m2.xlarge": 17510,
"m2.2xlarge": 435020,
"m2.4xlarge": 70041,
"m3.medium": 3840,
"m3.large": 7680,
"m3.xlarge": 15360,
"m3.2xlarge": 30720,
"m4.large": 8192,
"m4.xlarge": 16384,
"m4.2xlarge": 32768,
"m4.4xlarge": 65536,
"m4.10xlarge": 163840,
"m4.16xlarge": 262144,
"m5.large": 8192,
"m5.xlarge": 16384,
"m5.2xlarge": 32768,
"m5.4xlarge": 65536,
"m5.12xlarge": 196608,
"m5.24xlarge": 393216,
"m5d.large": 8192,
"m5d.xlarge": 16384,
"m5d.2xlarge": 32768,
"m5d.4xlarge": 65536,
"m5d.12xlarge": 196608,
"m5d.24xlarge": 393216,
"p2.xlarge": 62464,
"p2.8xlarge": 499712,
"p2.16xlarge": 749568,
"p3.2xlarge": 62464,
"p3.8xlarge": 249856,
"p3.16xlarge": 499712,
"r3.large": 15616,
"r3.xlarge": 31232,
"r3.2xlarge": 62464,
"r3.4xlarge": 124928,
"r3.8xlarge": 249856,
"r4.large": 15616,
"r4.xlarge": 31232,
"r4.2xlarge": 62464,
"r4.4xlarge": 124928,
"r4.8xlarge": 249856,
"r4.16xlarge": 499712,
"t1.micro": 627,
"t2.nano": 512,
"t2.micro": 1024,
"t2.small": 2048,
"t2.medium": 4096,
"t2.large": 8192,
"t2.xlarge": 16384,
"t2.2xlarge": 32768,
"t3.nano": 512,
"t3.micro": 1024,
"t3.small": 2048,
"t3.medium": 4096,
"t3.large": 8192,
"t3.xlarge": 16384,
"t3.2xlarge": 32768,
"x1.16xlarge": 999424,
"x1.32xlarge": 1998848,
"x1e.xlarge": 124928,
"x1e.2xlarge": 249856,
"x1e.4xlarge": 499712,
"x1e.8xlarge": 999424,
"x1e.16xlarge": 1998848,
"x1e.32xlarge": 3997696,
}
59 changes: 55 additions & 4 deletions pkg/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"

"fmt"
"log"
"regexp"
"time"
Expand All @@ -30,9 +31,11 @@ import (
// Node represents and EC2 instance.
type Node struct {
*ec2.Instance
MaxPods int
ClusterDNS string
Region string
MaxPods int
ReservedCPU string
ReservedMemory string
ClusterDNS string
Region string
}

type metadataClient interface {
Expand All @@ -57,7 +60,7 @@ func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) {
return nil, err
}
instance := output.Reservations[0].Instances[0]
node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region}
node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ReservedCPU: reservedCPU(instance.InstanceType), ReservedMemory: reservedMemory(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region}
if node.ClusterName() == "" {
sleepFor := b.Duration(tries)
log.Printf("The kubernetes.io/cluster/<name> tag is not yet set, will try again in %s", sleepFor)
Expand Down Expand Up @@ -99,6 +102,54 @@ func maxPods(instanceType *string) int {
return enis * (ips - 1)
}

// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedCPU(instanceType *string) string {
cores := InstanceCores[*instanceType]
reserved := 0.0
for i := 0; i < cores; i++ {
if i < 1 {
reserved += 60.0
} else if i < 2 {
reserved += 10.0
} else if i < 4 {
reserved += 5.0
} else {
reserved += 2.5
}
}
if reserved == 0.0 {
reserved = 60.0
}
return fmt.Sprintf("%.0fm", reserved)
}

// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedMemory(instanceType *string) string {
memory := InstanceMemory[*instanceType]
reserved := 0.0
for i := 0; i < memory; i++ {
if i < 4096 {
reserved += 0.25
} else if i < 8192 {
reserved += 0.2
} else if i < 16384 {
reserved += 0.1
} else if i < 131072 {
reserved += 0.06
} else {
reserved += 0.02
}
}
if reserved == 0.0 {
reserved = 960.0
}
return fmt.Sprintf("%.0fMi", reserved)
}

func clusterDNS(ip *string) string {
if ip != nil && len(*ip) > 3 && (*ip)[0:3] == "10." {
return "172.20.0.10"
Expand Down
Loading

0 comments on commit d9104ed

Please sign in to comment.