diff --git a/main.go b/main.go index 10d01e4..b646571 100644 --- a/main.go +++ b/main.go @@ -45,7 +45,7 @@ func region() *string { } func main() { - instance, err := node.New(ec2.New(sess), metadata) + instance, err := node.New(ec2.New(sess), metadata, region()) check(err) cluster, err := eks.Cluster(eksSvc.New(sess), instance.ClusterName()) diff --git a/pkg/node/node.go b/pkg/node/node.go index 4802f93..14428e5 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -32,6 +32,7 @@ type Node struct { *ec2.Instance MaxPods int ClusterDNS string + Region string } type metadataClient interface { @@ -44,7 +45,7 @@ var b = backoff.Backoff{Seq: []int{1, 1, 2}} // // If the EC2 instance doesn't have the expected kubernetes tag, it will backoff and retry. // If it isn't able to query EC2 or there are any other errors, an error will be returned. -func New(e ec2iface.EC2API, m metadataClient) (*Node, error) { +func New(e ec2iface.EC2API, m metadataClient, region *string) (*Node, error) { id, err := instanceID(m) if err != nil { return nil, err @@ -56,7 +57,7 @@ func New(e ec2iface.EC2API, m metadataClient) (*Node, error) { return nil, err } instance := output.Reservations[0].Instances[0] - node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress)} + node := Node{Instance: instance, MaxPods: maxPods(instance.InstanceType), ClusterDNS: clusterDNS(instance.PrivateIpAddress), Region: *region} if node.ClusterName() == "" { sleepFor := b.Duration(tries) log.Printf("The kubernetes.io/cluster/ tag is not yet set, will try again in %s", sleepFor) diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go index 4625673..bfe4b48 100644 --- a/pkg/node/node_test.go +++ b/pkg/node/node_test.go @@ -44,7 +44,8 @@ func TestNewNode(t *testing.T) { "instance-id": "1234", }, } - node, err := New(e, metadata) + region := "us-east-1" + node, err := New(e, metadata, ®ion) if err != nil { t.Errorf("unexpected error: %s", err) } @@ -56,6 +57,10 @@ func TestNewNode(t *testing.T) { if node.ClusterName() != "cluster-name" { t.Error("Expected returned node to have cluster-name") } + + if node.Region != region { + t.Errorf("Expected %s, to eq %s", node.Region, region) + } } func TestClusterDNS(t *testing.T) { @@ -65,7 +70,8 @@ func TestClusterDNS(t *testing.T) { {tag("kubernetes.io/cluster/cluster-name", "owned")}, }, } - node, err := New(e, mockMetadata{}) + region := "us-east-1" + node, err := New(e, mockMetadata{}, ®ion) if err != nil { t.Errorf("unexpected error: %s", err) @@ -81,7 +87,7 @@ func TestClusterDNS(t *testing.T) { {tag("kubernetes.io/cluster/cluster-name", "owned")}, }, } - node, err = New(e, mockMetadata{}) + node, err = New(e, mockMetadata{}, ®ion) if err != nil { t.Errorf("unexpected error: %s", err) @@ -99,7 +105,8 @@ func TestNewErrors(t *testing.T) { e := &mockEC2{err: ec2Error} metadata := mockMetadata{err: metadataError} - _, err := New(e, metadata) + region := "us-east-1" + _, err := New(e, metadata, ®ion) if err != metadataError { t.Errorf("expected error: %s to be %s", err, metadataError) } @@ -110,7 +117,7 @@ func TestNewErrors(t *testing.T) { }, } - _, err = New(e, metadata) + _, err = New(e, metadata, ®ion) if err != ec2Error { t.Errorf("expected error: %s to be %s", err, ec2Error) } @@ -194,7 +201,8 @@ func TestMaxPods(t *testing.T) { "instance-id": "1234", }, } - node, err := New(e, metadata) + region := "us-west-2" + node, err := New(e, metadata, ®ion) if err != nil { t.Errorf("unexpected error: %s", err) } diff --git a/pkg/system/system.go b/pkg/system/system.go index 18ede83..96519c6 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -19,6 +19,7 @@ package system import ( "github.com/aws/aws-sdk-go/service/eks" "github.com/errm/ekstrap/pkg/node" + "github.com/gobuffalo/packr" "bytes" "encoding/base64" @@ -76,18 +77,17 @@ func (s System) Configure(n *node.Node, cluster *eks.Cluster) error { func (s System) configs() ([]config, error) { configs := []config{} - for path, content := range defaultTemplates { - template, err := template.New(path).Funcs(template.FuncMap{"b64dec": base64decode}).Parse(content) - if err != nil { - return configs, err - } + box := packr.NewBox("./templates") + err := box.Walk(func(path string, f packr.File) error { + template, err := template.New(path).Funcs(template.FuncMap{"b64dec": base64decode}).Parse(box.String(path)) configs = append(configs, config{ template: template, - path: path, + path: "/" + path, filesystem: s.Filesystem, }) - } - return configs, nil + return err + }) + return configs, err } func base64decode(v string) (string, error) { diff --git a/pkg/system/system_test.go b/pkg/system/system_test.go index ec8057a..93c974d 100644 --- a/pkg/system/system_test.go +++ b/pkg/system/system_test.go @@ -46,8 +46,8 @@ func TestConfigure(t *testing.T) { t.Errorf("unexpected error %v", err) } - if len(fs.files) != 3 { - t.Errorf("expected 3 files, got %v", len(fs.files)) + if len(fs.files) != 5 { + t.Errorf("expected 5 files, got %v", len(fs.files)) } expected := `apiVersion: v1 @@ -68,11 +68,12 @@ users: user: exec: apiVersion: client.authentication.k8s.io/v1alpha1 - command: /usr/local/bin/heptio-authenticator-aws + command: /usr/local/bin/aws-iam-authenticator args: - token - "-i" - - "aws-om-cluster"` + - "aws-om-cluster" +` fs.Check(t, "/var/lib/kubelet/kubeconfig", expected, 0640) expected = `[Unit] @@ -82,18 +83,46 @@ After=docker.service Requires=docker.service [Service] -ExecStart=/usr/bin/kubelet --address=0.0.0.0 --allow-privileged=true --cloud-provider=aws --cluster-dns=172.20.0.10 --cluster-domain=cluster.local --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d --container-runtime=docker --node-ip=10.6.28.199 --network-plugin=cni --cgroup-driver=cgroupfs --register-node=true --kubeconfig=/var/lib/kubelet/kubeconfig --feature-gates=RotateKubeletServerCertificate=true --anonymous-auth=false --client-ca-file=/etc/kubernetes/pki/ca.crt --max-pods=18 +ExecStart=/usr/bin/kubelet \ + --address=0.0.0.0 \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --allow-privileged=true \ + --cloud-provider=aws \ + --cluster-domain=cluster.local \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + --container-runtime=docker \ + --network-plugin=cni \ + --cgroup-driver=cgroupfs \ + --register-node=true \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --feature-gates=RotateKubeletServerCertificate=true \ + --anonymous-auth=false \ + --client-ca-file=/etc/kubernetes/pki/ca.crt $KUBELET_ARGS $KUBELET_MAX_PODS $KUBELET_EXTRA_ARGS -Restart=on-failure Restart=always StartLimitInterval=0 -RestartSec=10 +RestartSec=5 [Install] -WantedBy=multi-user.target` - fs.Check(t, "/lib/systemd/system/kubelet.service", expected, 0640) +WantedBy=multi-user.target +` + fs.Check(t, "/etc/systemd/system/kubelet.service", expected, 0640) - fs.Check(t, "/etc/kubernetes/pki/ca.crt", "thisisthecertdata", 0640) + expected = `[Service] +Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --cluster-dns=172.20.0.10 --pod-infra-container-image=602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/pause-amd64:3.1' +` + fs.Check(t, "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf", expected, 0640) + + expected = `[Service] +Environment='KUBELET_MAX_PODS=--max-pods=18' +` + fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-max-pods.conf", expected, 0640) + + expected = `thisisthecertdata +` + fs.Check(t, "/etc/kubernetes/pki/ca.crt", expected, 0640) if hn.hostname != "ip-10-6-28-199.us-west-2.compute.internal" { t.Errorf("expected hostname to be ip-10-6-28-199.us-west-2.compute.internal, got %v", hn.hostname) @@ -116,6 +145,7 @@ func instance(ip, dnsName string, maxPods int) *node.Node { }, MaxPods: maxPods, ClusterDNS: "172.20.0.10", + Region: "us-east-1", } } @@ -151,7 +181,7 @@ func (f *FakeFileSystem) Check(t *testing.T, path string, contents string, mode } actual := string(file.Contents) if contents != actual { - t.Errorf("File contents not as expected:\nactual:\n%v\n\nexpected:\n%v", actual, contents) + t.Errorf("File contents not as expected:\nactual:\n%#v\n\nexpected:\n%#v", actual, contents) } return } diff --git a/pkg/system/templates.go b/pkg/system/templates.go deleted file mode 100644 index 3ad51f8..0000000 --- a/pkg/system/templates.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2018 Edward Robinson. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package system - -var defaultTemplates = map[string]string{ - "/var/lib/kubelet/kubeconfig": `apiVersion: v1 -kind: Config -clusters: -- name: {{.Cluster.Name}} - cluster: - server: {{.Cluster.Endpoint}} - certificate-authority-data: {{.Cluster.CertificateAuthority.Data}} -contexts: -- name: kubelet - context: - cluster: {{.Cluster.Name}} - user: kubelet -current-context: kubelet -users: -- name: kubelet - user: - exec: - apiVersion: client.authentication.k8s.io/v1alpha1 - command: /usr/local/bin/heptio-authenticator-aws - args: - - token - - "-i" - - "{{.Cluster.Name}}"`, - - "/lib/systemd/system/kubelet.service": `[Unit] -Description=kubelet: The Kubernetes Node Agent -Documentation=http://kubernetes.io/docs/ -After=docker.service -Requires=docker.service - -[Service] -ExecStart=/usr/bin/kubelet --address=0.0.0.0 --allow-privileged=true --cloud-provider=aws --cluster-dns={{.Node.ClusterDNS}} --cluster-domain=cluster.local --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d --container-runtime=docker --node-ip={{.Node.PrivateIpAddress}} --network-plugin=cni --cgroup-driver=cgroupfs --register-node=true --kubeconfig=/var/lib/kubelet/kubeconfig --feature-gates=RotateKubeletServerCertificate=true --anonymous-auth=false --client-ca-file=/etc/kubernetes/pki/ca.crt{{if .Node.MaxPods}} --max-pods={{.Node.MaxPods}}{{end}} - -Restart=on-failure -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=multi-user.target`, - - "/etc/kubernetes/pki/ca.crt": `{{.Cluster.CertificateAuthority.Data | b64dec}}`, -} diff --git a/pkg/system/templates/etc/kubernetes/pki/ca.crt b/pkg/system/templates/etc/kubernetes/pki/ca.crt new file mode 100644 index 0000000..07f6d95 --- /dev/null +++ b/pkg/system/templates/etc/kubernetes/pki/ca.crt @@ -0,0 +1 @@ +{{.Cluster.CertificateAuthority.Data | b64dec}} diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service b/pkg/system/templates/etc/systemd/system/kubelet.service new file mode 100644 index 0000000..7c57646 --- /dev/null +++ b/pkg/system/templates/etc/systemd/system/kubelet.service @@ -0,0 +1,31 @@ +[Unit] +Description=kubelet: The Kubernetes Node Agent +Documentation=http://kubernetes.io/docs/ +After=docker.service +Requires=docker.service + +[Service] +ExecStart=/usr/bin/kubelet \ + --address=0.0.0.0 \ + --authentication-token-webhook \ + --authorization-mode=Webhook \ + --allow-privileged=true \ + --cloud-provider=aws \ + --cluster-domain=cluster.local \ + --cni-bin-dir=/opt/cni/bin \ + --cni-conf-dir=/etc/cni/net.d \ + --container-runtime=docker \ + --network-plugin=cni \ + --cgroup-driver=cgroupfs \ + --register-node=true \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --feature-gates=RotateKubeletServerCertificate=true \ + --anonymous-auth=false \ + --client-ca-file=/etc/kubernetes/pki/ca.crt $KUBELET_ARGS $KUBELET_MAX_PODS $KUBELET_EXTRA_ARGS + +Restart=always +StartLimitInterval=0 +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf new file mode 100644 index 0000000..97ab6ce --- /dev/null +++ b/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf @@ -0,0 +1,2 @@ +[Service] +Environment='KUBELET_ARGS=--node-ip={{.Node.PrivateIpAddress}} --cluster-dns={{.Node.ClusterDNS}} --pod-infra-container-image=602401143452.dkr.ecr.{{.Node.Region}}.amazonaws.com/eks/pause-amd64:3.1' diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf new file mode 100644 index 0000000..d3ae0c7 --- /dev/null +++ b/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf @@ -0,0 +1,2 @@ +[Service] +Environment='KUBELET_MAX_PODS=--max-pods={{.Node.MaxPods}}' diff --git a/pkg/system/templates/var/lib/kubelet/kubeconfig b/pkg/system/templates/var/lib/kubelet/kubeconfig new file mode 100644 index 0000000..f43f13a --- /dev/null +++ b/pkg/system/templates/var/lib/kubelet/kubeconfig @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Config +clusters: +- name: {{.Cluster.Name}} + cluster: + server: {{.Cluster.Endpoint}} + certificate-authority-data: {{.Cluster.CertificateAuthority.Data}} +contexts: +- name: kubelet + context: + cluster: {{.Cluster.Name}} + user: kubelet +current-context: kubelet +users: +- name: kubelet + user: + exec: + apiVersion: client.authentication.k8s.io/v1alpha1 + command: /usr/local/bin/aws-iam-authenticator + args: + - token + - "-i" + - "{{.Cluster.Name}}"