Merge pull request #42 from errm/align-with-standard-eks-bootstrap

Align with standard EKS bootstrap

.gitignore

@@ -3,3 +3,4 @@ main
 dist/
 .travis/ekstrap.asc
 coverage.txt
+**/*-packr.go

.travis/setup.sh

@@ -5,3 +5,4 @@ set -euo pipefail
 openssl aes-256-cbc -K $encrypted_189aefeda93d_key -iv $encrypted_189aefeda93d_iv -in .travis/ekstrap.asc.enc -out .travis/ekstrap.asc -d
 gpg --import .travis/ekstrap.asc
 make install-linter
+go get -u github.com/gobuffalo/packr/...

Makefile

@@ -9,7 +9,7 @@ GOMETALINTER = gometalinter ./...
 GORELEASER = goreleaser release --rm-dist --debug
 
 all: test lint $(BINARY_NAME)
-$(BINARY_NAME):
+$(BINARY_NAME): generate
 	$(GOBUILD) -o $(BINARY_NAME) -v
 compress: $(BINARY_NAME)
 	$(UPX) $(BINARY_NAME)
@@ -20,13 +20,14 @@ install-linter:
 	$(GOMETALINTER) --install
 lint:
 	$(GOMETALINTER)
-release: .goreleaser.yml
+release: generate .goreleaser.yml
 	$(GORELEASER)
 snapshot: .goreleaser.yml
 	$(GORELEASER) --snapshot
 install: $(BINARY_NAME)
 	install -m755 $(BINARY_NAME) /usr/sbin
-
+generate:
+	$(GOCMD) generate
 clean:
 	rm -rf \
 		./$(BINARY_NAME) \

README.md

@@ -19,6 +19,18 @@ When run on an ec2 node ekstrap performs several tasks.
 
 In order to run ekstrap your instance should have an IAM instance profile that allows the `EC2::DescribeInstances` action and the `EKS::DescribeCluster` action. Both of these actions are already included in the AWS managed policy `arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy` along with the other permissions that the kubelet requires to connect to your cluster, it is recommended therefore to simply attach this policy to your instance role/profile.
 
+### Extra Arguments
+
+If you wish to provide extra aruguments to the kubelet you can create a drop-in that sets the `KUBELET_EXTRA_ARGS` environment variable.
+
+For example to [taint nodes with GPU hardware](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#example-use-cases) you could add:
+
+_/etc/systemd/system/kubelet.service.d/30-kubelet-extra-args.conf_
+```
+[Service]
+Environment='KUBELET_EXTRA_ARGS=--register-with-taints="gpu=true:PreferNoSchedule"'
+```
+
 ## Installation
 
 The simplest way to install ekstrap is to use our packagecloud repository.

main.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package main
 
+//go:generate packr
+
 import (
 	"log"
 
@@ -45,7 +47,7 @@ func region() *string {
 }
 
 func main() {
-	instance, err := node.New(ec2.New(sess), metadata)
+	instance, err := node.New(ec2.New(sess), metadata, region())
 	check(err)
 
 	cluster, err := eks.Cluster(eksSvc.New(sess), instance.ClusterName())

pkg/node/node.go

@@ -32,6 +32,7 @@ type Node struct {
 	*ec2.Instance
 	MaxPods    int
 	ClusterDNS string
+	Region     string
 }
 
 type metadataClient interface {
@@ -44,7 +45,7 @@ var b = backoff.Backoff{Seq: []int{1, 1, 2}}
 //
 // If the EC2 instance doesn't have the expected kubernetes tag, it will backoff and retry.
 // If it isn't able to query EC2 or there are any other errors, an error will be returned.
-func New(e ec2iface.EC2API, m metadataClient) (*Node, error) {
+func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) {
 	id, err := instanceID(m)
 	if err != nil {
 		return nil, err
@@ -56,7 +57,7 @@ func New(e ec2iface.EC2API, m metadataClient) (*Node, error) {
 			return nil, err
 		}
 		instance := output.Reservations[0].Instances[0]
-		node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress)}
+		node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region}
 		if node.ClusterName() == "" {
 			sleepFor := b.Duration(tries)
 			log.Printf("The kubernetes.io/cluster/<name> tag is not yet set, will try again in %s", sleepFor)

pkg/node/node_test.go

@@ -44,7 +44,8 @@ func TestNewNode(t *testing.T) {
 			"instance-id": "1234",
 		},
 	}
-	node, err := New(e, metadata)
+	region := "us-east-1"
+	node, err := New(e, metadata, &region)
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
 	}
@@ -56,6 +57,10 @@ func TestNewNode(t *testing.T) {
 	if node.ClusterName() != "cluster-name" {
 		t.Error("Expected returned node to have cluster-name")
 	}
+
+	if node.Region != region {
+		t.Errorf("Expected %s, to eq %s", node.Region, region)
+	}
 }
 
 func TestClusterDNS(t *testing.T) {
@@ -65,7 +70,8 @@ func TestClusterDNS(t *testing.T) {
 			{tag("kubernetes.io/cluster/cluster-name", "owned")},
 		},
 	}
-	node, err := New(e, mockMetadata{})
+	region := "us-east-1"
+	node, err := New(e, mockMetadata{}, &region)
 
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
@@ -81,7 +87,7 @@ func TestClusterDNS(t *testing.T) {
 			{tag("kubernetes.io/cluster/cluster-name", "owned")},
 		},
 	}
-	node, err = New(e, mockMetadata{})
+	node, err = New(e, mockMetadata{}, &region)
 
 	if err != nil {
 		t.Errorf("unexpected error: %s", err)
@@ -99,7 +105,8 @@ func TestNewErrors(t *testing.T) {
 	e := &mockEC2{err: ec2Error}
 	metadata := mockMetadata{err: metadataError}
 
-	_, err := New(e, metadata)
+	region := "us-east-1"
+	_, err := New(e, metadata, &region)
 	if err != metadataError {
 		t.Errorf("expected error: %s to be %s", err, metadataError)
 	}
@@ -110,7 +117,7 @@ func TestNewErrors(t *testing.T) {
 		},
 	}
 
-	_, err = New(e, metadata)
+	_, err = New(e, metadata, &region)
 	if err != ec2Error {
 		t.Errorf("expected error: %s to be %s", err, ec2Error)
 	}
@@ -194,7 +201,8 @@ func TestMaxPods(t *testing.T) {
 				"instance-id": "1234",
 			},
 		}
-		node, err := New(e, metadata)
+		region := "us-west-2"
+		node, err := New(e, metadata, &region)
 		if err != nil {
 			t.Errorf("unexpected error: %s", err)
 		}

pkg/system/system.go

@@ -19,6 +19,7 @@ package system
 import (
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/errm/ekstrap/pkg/node"
+	"github.com/gobuffalo/packr"
 
 	"bytes"
 	"encoding/base64"
@@ -76,18 +77,17 @@ func (s System) Configure(n *node.Node, cluster *eks.Cluster) error {
 
 func (s System) configs() ([]config, error) {
 	configs := []config{}
-	for path, content := range defaultTemplates {
-		template, err := template.New(path).Funcs(template.FuncMap{"b64dec": base64decode}).Parse(content)
-		if err != nil {
-			return configs, err
-		}
+	box := packr.NewBox("./templates")
+	err := box.Walk(func(path string, f packr.File) error {
+		template, err := template.New(path).Funcs(template.FuncMap{"b64dec": base64decode}).Parse(box.String(path))
 		configs = append(configs, config{
 			template:   template,
-			path:       path,
+			path:       "/" + path,
 			filesystem: s.Filesystem,
 		})
-	}
-	return configs, nil
+		return err
+	})
+	return configs, err
 }
 
 func base64decode(v string) (string, error) {

pkg/system/system_test.go

@@ -46,8 +46,8 @@ func TestConfigure(t *testing.T) {
 		t.Errorf("unexpected error %v", err)
 	}
 
-	if len(fs.files) != 3 {
-		t.Errorf("expected 3 files, got %v", len(fs.files))
+	if len(fs.files) != 5 {
+		t.Errorf("expected 5 files, got %v", len(fs.files))
 	}
 
 	expected := `apiVersion: v1
@@ -68,11 +68,12 @@ users:
   user:
     exec:
       apiVersion: client.authentication.k8s.io/v1alpha1
-      command: /usr/local/bin/heptio-authenticator-aws
+      command: aws-iam-authenticator
       args:
         - token
         - "-i"
-        - "aws-om-cluster"`
+        - "aws-om-cluster"
+`
 	fs.Check(t, "/var/lib/kubelet/kubeconfig", expected, 0640)
 
 	expected = `[Unit]
@@ -82,18 +83,46 @@ After=docker.service
 Requires=docker.service
 
 [Service]
-ExecStart=/usr/bin/kubelet   --address=0.0.0.0   --allow-privileged=true   --cloud-provider=aws   --cluster-dns=172.20.0.10   --cluster-domain=cluster.local   --cni-bin-dir=/opt/cni/bin   --cni-conf-dir=/etc/cni/net.d   --container-runtime=docker   --node-ip=10.6.28.199   --network-plugin=cni   --cgroup-driver=cgroupfs   --register-node=true   --kubeconfig=/var/lib/kubelet/kubeconfig   --feature-gates=RotateKubeletServerCertificate=true   --anonymous-auth=false   --client-ca-file=/etc/kubernetes/pki/ca.crt --max-pods=18
+ExecStart=/usr/bin/kubelet \
+  --address=0.0.0.0 \
+  --authentication-token-webhook \
+  --authorization-mode=Webhook \
+  --allow-privileged=true \
+  --cloud-provider=aws \
+  --cluster-domain=cluster.local \
+  --cni-bin-dir=/opt/cni/bin \
+  --cni-conf-dir=/etc/cni/net.d \
+  --container-runtime=docker \
+  --network-plugin=cni \
+  --cgroup-driver=cgroupfs \
+  --register-node=true \
+  --kubeconfig=/var/lib/kubelet/kubeconfig \
+  --feature-gates=RotateKubeletServerCertificate=true \
+  --anonymous-auth=false \
+  --client-ca-file=/etc/kubernetes/pki/ca.crt $KUBELET_ARGS $KUBELET_MAX_PODS $KUBELET_EXTRA_ARGS
 
-Restart=on-failure
 Restart=always
 StartLimitInterval=0
-RestartSec=10
+RestartSec=5
 
 [Install]
-WantedBy=multi-user.target`
-	fs.Check(t, "/lib/systemd/system/kubelet.service", expected, 0640)
+WantedBy=multi-user.target
+`
+	fs.Check(t, "/etc/systemd/system/kubelet.service", expected, 0640)
 
-	fs.Check(t, "/etc/kubernetes/pki/ca.crt", "thisisthecertdata", 0640)
+	expected = `[Service]
+Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --cluster-dns=172.20.0.10 --pod-infra-container-image=602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/pause-amd64:3.1'
+`
+	fs.Check(t, "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf", expected, 0640)
+
+	expected = `[Service]
+Environment='KUBELET_MAX_PODS=--max-pods=18'
+`
+	fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-max-pods.conf", expected, 0640)
+
+	expected = `thisisthecertdata
+`
+	fs.Check(t, "/etc/kubernetes/pki/ca.crt", expected, 0640)
 
 	if hn.hostname != "ip-10-6-28-199.us-west-2.compute.internal" {
 		t.Errorf("expected hostname to be ip-10-6-28-199.us-west-2.compute.internal, got %v", hn.hostname)
@@ -116,6 +145,7 @@ func instance(ip, dnsName string, maxPods int) *node.Node {
 		},
 		MaxPods:    maxPods,
 		ClusterDNS: "172.20.0.10",
+		Region:     "us-east-1",
 	}
 }
 
@@ -151,7 +181,7 @@ func (f *FakeFileSystem) Check(t *testing.T, path string, contents string, mode
 			}
 			actual := string(file.Contents)
 			if contents != actual {
-				t.Errorf("File contents not as expected:\nactual:\n%v\n\nexpected:\n%v", actual, contents)
+				t.Errorf("File contents not as expected:\nactual:\n%#v\n\nexpected:\n%#v", actual, contents)
 			}
 			return
 		}

pkg/system/templates/etc/kubernetes/pki/ca.crt

@@ -0,0 +1 @@
+{{.Cluster.CertificateAuthority.Data | b64dec}}