Skip to content
This repository has been archived by the owner on Oct 15, 2020. It is now read-only.

Commit

Permalink
Merge pull request #44 from errm/kube-reserved
Browse files Browse the repository at this point in the history 
Calculate a resonable value for the --kube-reserved flag
  • Loading branch information
@errm
errm committed Sep 18, 2018
2 parents 4f7df97 + 9df3894 commit e695b61
Show file tree
Hide file tree
Showing 7 changed files with 525 additions and 14 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ When run on an ec2 node ekstrap performs several tasks.
* Writes a kubeconfig file configured to connect to your EKS cluster to `/var/lib/kubelet/kubeconfig`.
* Writes a systemd unit file to `/lib/systemd/system/kubelet.service`.
* Writes the cluster CA certificate to `/etc/kubernetes/pki/ca.crt`.
* Calculates an appropriate value for for [--kube-reserved](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/)
* Restarts the kubelet unit.

In order to run ekstrap your instance should have an IAM instance profile that allows the `EC2::DescribeInstances` action and the `EKS::DescribeCluster` action. Both of these actions are already included in the AWS managed policy `arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy` along with the other permissions that the kubelet requires to connect to your cluster, it is recommended therefore to simply attach this policy to your instance role/profile.
Expand All @@ -25,7 +26,7 @@ If you wish to provide extra aruguments to the kubelet you can create a drop-in

For example to [taint nodes with GPU hardware](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#example-use-cases) you could add:

_/etc/systemd/system/kubelet.service.d/30-kubelet-extra-args.conf_
_/etc/systemd/system/kubelet.service.d/40-kubelet-extra-args.conf_
```
[Service]
Environment='KUBELET_EXTRA_ARGS=--register-with-taints="gpu=true:PreferNoSchedule"'
Expand Down
273 changes: 273 additions & 0 deletions pkg/node/ec2_resources.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
/*
Copyright 2018 Edward Robinson.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

// InstanceCores contains a mapping of instance types to the number of cores
// Based on info from https://aws.amazon.com/ec2/instance-types/ and
// https://aws.amazon.com/ec2/previous-generation/
var InstanceCores = map[string]int{
"c1.medium": 2,
"c1.xlarge": 8,
"c3.large": 2,
"c3.xlarge": 4,
"c3.2xlarge": 8,
"c3.4xlarge": 16,
"c3.8xlarge": 32,
"c4.large": 2,
"c4.xlarge": 4,
"c4.2xlarge": 8,
"c4.4xlarge": 16,
"c4.8xlarge": 32,
"c5.large": 2,
"c5.xlarge": 4,
"c5.2xlarge": 8,
"c5.4xlarge": 16,
"c5.9xlarge": 36,
"c5.18xlarge": 72,
"c5d.large": 2,
"c5d.xlarge": 4,
"c5d.2xlarge": 8,
"c5d.4xlarge": 16,
"c5d.9xlarge": 36,
"c5d.18xlarge": 72,
"cc2.8xlarge": 32,
"cr1.8xlarge": 32,
"d2.xlarge": 4,
"d2.2xlarge": 8,
"d2.4xlarge": 16,
"d2.8xlarge": 36,
"f1.2xlarge": 8,
"f1.16xlarge": 64,
"g2.2xlarge": 8,
"g2.8xlarge": 32,
"g3.4xlarge": 16,
"g3.8xlarge": 32,
"g3.16xlarge": 64,
"h1.2xlarge": 8,
"h1.4xlarge": 16,
"h1.8xlarge": 32,
"h1.16xlarge": 64,
"hs1.8xlarge": 16,
"i2.xlarge": 4,
"i2.2xlarge": 8,
"i2.4xlarge": 16,
"i2.8xlarge": 32,
"i3.large": 2,
"i3.xlarge": 4,
"i3.2xlarge": 8,
"i3.4xlarge": 16,
"i3.8xlarge": 32,
"i3.16xlarge": 64,
"i3.metal": 72,
"m1.small": 1,
"m1.medium": 1,
"m1.large": 2,
"m1.xlarge": 4,
"m2.xlarge": 2,
"m2.2xlarge": 4,
"m2.4xlarge": 8,
"m3.medium": 1,
"m3.large": 2,
"m3.xlarge": 4,
"m3.2xlarge": 8,
"m4.large": 2,
"m4.xlarge": 4,
"m4.2xlarge": 8,
"m4.4xlarge": 16,
"m4.10xlarge": 40,
"m4.16xlarge": 64,
"m5.large": 2,
"m5.xlarge": 4,
"m5.2xlarge": 8,
"m5.4xlarge": 16,
"m5.12xlarge": 48,
"m5.24xlarge": 96,
"m5d.large": 2,
"m5d.xlarge": 4,
"m5d.2xlarge": 8,
"m5d.4xlarge": 16,
"m5d.12xlarge": 48,
"m5d.24xlarge": 96,
"p2.xlarge": 4,
"p2.8xlarge": 32,
"p2.16xlarge": 64,
"p3.2xlarge": 8,
"p3.8xlarge": 32,
"p3.16xlarge": 64,
"r3.large": 2,
"r3.xlarge": 4,
"r3.2xlarge": 8,
"r3.4xlarge": 16,
"r3.8xlarge": 32,
"r4.large": 2,
"r4.xlarge": 4,
"r4.2xlarge": 8,
"r4.4xlarge": 16,
"r4.8xlarge": 32,
"r4.16xlarge": 64,
"t1.micro": 1,
"t2.nano": 1,
"t2.micro": 1,
"t2.small": 1,
"t2.medium": 2,
"t2.large": 2,
"t2.xlarge": 4,
"t2.2xlarge": 8,
"t3.nano": 2,
"t3.micro": 2,
"t3.small": 2,
"t3.medium": 2,
"t3.large": 2,
"t3.xlarge": 4,
"t3.2xlarge": 8,
"x1.16xlarge": 64,
"x1.32xlarge": 128,
"x1e.xlarge": 4,
"x1e.2xlarge": 8,
"x1e.4xlarge": 16,
"x1e.8xlarge": 32,
"x1e.16xlarge": 64,
"x1e.32xlarge": 128,
}

// InstanceMemory contains a mapping of instance types to the amount of memory
// Based on info from https://aws.amazon.com/ec2/instance-types/ and
// https://aws.amazon.com/ec2/previous-generation/
var InstanceMemory = map[string]int{
"c1.medium": 1740,
"c1.xlarge": 7168,
"c3.large": 3840,
"c3.xlarge": 7680,
"c3.2xlarge": 15360,
"c3.4xlarge": 30720,
"c3.8xlarge": 61440,
"c4.large": 3840,
"c4.xlarge": 7680,
"c4.2xlarge": 15360,
"c4.4xlarge": 30720,
"c4.8xlarge": 61440,
"c5.large": 4096,
"c5.xlarge": 8192,
"c5.2xlarge": 16384,
"c5.4xlarge": 32768,
"c5.9xlarge": 73728,
"c5.18xlarge": 147456,
"c5d.large": 4096,
"c5d.xlarge": 8192,
"c5d.2xlarge": 16384,
"c5d.4xlarge": 32768,
"c5d.9xlarge": 73728,
"c5d.18xlarge": 147456,
"cc2.8xlarge": 61952,
"cr1.8xlarge": 249856,
"d2.xlarge": 31232,
"d2.2xlarge": 62464,
"d2.4xlarge": 124928,
"d2.8xlarge": 249856,
"f1.2xlarge": 124928,
"f1.16xlarge": 999424,
"g2.2xlarge": 15360,
"g2.8xlarge": 61440,
"g3.4xlarge": 124928,
"g3.8xlarge": 249856,
"g3.16xlarge": 499712,
"h1.2xlarge": 32768,
"h1.4xlarge": 65536,
"h1.8xlarge": 131072,
"h1.16xlarge": 262144,
"hs1.8xlarge": 119808,
"i2.xlarge": 31232,
"i2.2xlarge": 62464,
"i2.4xlarge": 124928,
"i2.8xlarge": 249856,
"i3.large": 15616,
"i3.xlarge": 31232,
"i3.2xlarge": 62464,
"i3.4xlarge": 124928,
"i3.8xlarge": 249856,
"i3.16xlarge": 499712,
"i3.metal": 524288,
"m1.small": 1740,
"m1.medium": 3840,
"m1.large": 7680,
"m1.xlarge": 15360,
"m2.xlarge": 17510,
"m2.2xlarge": 435020,
"m2.4xlarge": 70041,
"m3.medium": 3840,
"m3.large": 7680,
"m3.xlarge": 15360,
"m3.2xlarge": 30720,
"m4.large": 8192,
"m4.xlarge": 16384,
"m4.2xlarge": 32768,
"m4.4xlarge": 65536,
"m4.10xlarge": 163840,
"m4.16xlarge": 262144,
"m5.large": 8192,
"m5.xlarge": 16384,
"m5.2xlarge": 32768,
"m5.4xlarge": 65536,
"m5.12xlarge": 196608,
"m5.24xlarge": 393216,
"m5d.large": 8192,
"m5d.xlarge": 16384,
"m5d.2xlarge": 32768,
"m5d.4xlarge": 65536,
"m5d.12xlarge": 196608,
"m5d.24xlarge": 393216,
"p2.xlarge": 62464,
"p2.8xlarge": 499712,
"p2.16xlarge": 749568,
"p3.2xlarge": 62464,
"p3.8xlarge": 249856,
"p3.16xlarge": 499712,
"r3.large": 15616,
"r3.xlarge": 31232,
"r3.2xlarge": 62464,
"r3.4xlarge": 124928,
"r3.8xlarge": 249856,
"r4.large": 15616,
"r4.xlarge": 31232,
"r4.2xlarge": 62464,
"r4.4xlarge": 124928,
"r4.8xlarge": 249856,
"r4.16xlarge": 499712,
"t1.micro": 627,
"t2.nano": 512,
"t2.micro": 1024,
"t2.small": 2048,
"t2.medium": 4096,
"t2.large": 8192,
"t2.xlarge": 16384,
"t2.2xlarge": 32768,
"t3.nano": 512,
"t3.micro": 1024,
"t3.small": 2048,
"t3.medium": 4096,
"t3.large": 8192,
"t3.xlarge": 16384,
"t3.2xlarge": 32768,
"x1.16xlarge": 999424,
"x1.32xlarge": 1998848,
"x1e.xlarge": 124928,
"x1e.2xlarge": 249856,
"x1e.4xlarge": 499712,
"x1e.8xlarge": 999424,
"x1e.16xlarge": 1998848,
"x1e.32xlarge": 3997696,
}
63 changes: 59 additions & 4 deletions pkg/node/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"

"fmt"
"log"
"regexp"
"time"
Expand All @@ -30,9 +31,11 @@ import (
// Node represents and EC2 instance.
type Node struct {
*ec2.Instance
MaxPods int
ClusterDNS string
Region string
MaxPods int
ReservedCPU string
ReservedMemory string
ClusterDNS string
Region string
}

type metadataClient interface {
Expand All @@ -57,7 +60,7 @@ func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) {
return nil, err
}
instance := output.Reservations[0].Instances[0]
node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region}
node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ReservedCPU: reservedCPU(instance.InstanceType), ReservedMemory: reservedMemory(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region}
if node.ClusterName() == "" {
sleepFor := b.Duration(tries)
log.Printf("The kubernetes.io/cluster/<name> tag is not yet set, will try again in %s", sleepFor)
Expand Down Expand Up @@ -99,6 +102,58 @@ func maxPods(instanceType *string) int {
return enis * (ips - 1)
}

// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedCPU(instanceType *string) string {
cores := InstanceCores[*instanceType]
reserved := 0.0
for core := 1; core <= cores; core++ {
switch core {
case 1:
reserved += 60.0
case 2:
reserved += 10.0
case 3, 4:
reserved += 5.0
default:
reserved += 2.5
}
}
if reserved == 0.0 {
log.Printf("The number of CPU cores is unknown for the %s instance type, --kube-reserved will not be configured", *instanceType)
return ""
}
return fmt.Sprintf("%.0fm", reserved)
}

// The calculation here is based on information found in the GKE documentation
// here: https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture
// I think that it should also apply to AWS
func reservedMemory(instanceType *string) string {
memory := InstanceMemory[*instanceType]
reserved := 0.0
for i := 0; i < memory; i++ {
switch {
case i < 4096:
reserved += 0.25
case i < 8192:
reserved += 0.2
case i < 16384:
reserved += 0.1
case i < 131072:
reserved += 0.06
default:
reserved += 0.02
}
}
if reserved == 0.0 {
log.Printf("The Memory of the %s instance type is unknown, --kube-reserved will not be configured", *instanceType)
return ""
}
return fmt.Sprintf("%.0fMi", reserved)
}

func clusterDNS(ip *string) string {
if ip != nil && len(*ip) > 3 && (*ip)[0:3] == "10." {
return "172.20.0.10"
Expand Down
Loading

0 comments on commit e695b61

Please sign in to comment.