From 17f40b152deeeab0666e8f09442e2f4459e7951f Mon Sep 17 00:00:00 2001 From: Ed Robinson Date: Thu, 20 Dec 2018 13:45:09 +0000 Subject: [PATCH] Update config to use KubeletConfig in line with offical AMI see: https://github.com/awslabs/amazon-eks-ami/pull/90 This should fix #49 that I think is cased by relying on depricated kubelet flags. --- pkg/system/system_test.go | 70 +++++++++++-------- .../etc/kubernetes/kubelet/config.yaml | 32 +++++++++ .../etc/systemd/system/kubelet.service | 15 +--- .../kubelet.service.d/10-kubelet-args.conf | 2 +- .../{40-labels.conf => 20-labels.conf} | 0 .../system/kubelet.service.d/20-max-pods.conf | 2 - .../kubelet.service.d/30-kube-reserved.conf | 4 -- .../{50-taints.conf => 30-taints.conf} | 0 8 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 pkg/system/templates/etc/kubernetes/kubelet/config.yaml rename pkg/system/templates/etc/systemd/system/kubelet.service.d/{40-labels.conf => 20-labels.conf} (100%) delete mode 100644 pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf delete mode 100644 pkg/system/templates/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf rename pkg/system/templates/etc/systemd/system/kubelet.service.d/{50-taints.conf => 30-taints.conf} (100%) diff --git a/pkg/system/system_test.go b/pkg/system/system_test.go index 4d77efd..d5bc437 100644 --- a/pkg/system/system_test.go +++ b/pkg/system/system_test.go @@ -46,8 +46,8 @@ func TestConfigure(t *testing.T) { t.Errorf("unexpected error %v", err) } - if len(fs.files) != 8 { - t.Errorf("expected 8 files, got %v", len(fs.files)) + if len(fs.files) != 7 { + t.Errorf("expected 7 files, got %v", len(fs.files)) } expected := `apiVersion: v1 @@ -84,23 +84,12 @@ Requires=docker.service [Service] ExecStart=/usr/bin/kubelet \ - --address=0.0.0.0 \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ --allow-privileged=true \ --cloud-provider=aws \ - --cluster-domain=cluster.local \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ + --config /etc/kubernetes/kubelet/config.yaml \ --container-runtime=docker \ - --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \ --network-plugin=cni \ - --cgroup-driver=cgroupfs \ - --register-node=true \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --feature-gates=RotateKubeletServerCertificate=true \ - --anonymous-auth=false \ - --client-ca-file=/etc/kubernetes/pki/ca.crt $KUBELET_ARGS $KUBELET_MAX_PODS $KUBELET_KUBE_RESERVED $KUBELET_NODE_LABELS $KUBELET_NODE_TAINTS $KUBELET_EXTRA_ARGS + --kubeconfig=/var/lib/kubelet/kubeconfig $KUBELET_ARGS $KUBELET_NODE_LABELS $KUBELET_NODE_TAINTS $KUBELET_EXTRA_ARGS Restart=always StartLimitInterval=0 @@ -112,27 +101,50 @@ WantedBy=multi-user.target fs.Check(t, "/etc/systemd/system/kubelet.service", expected, 0640) expected = `[Service] -Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --cluster-dns=172.20.0.10 --pod-infra-container-image=602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/pause-amd64:3.1' +Environment='KUBELET_ARGS=--node-ip=10.6.28.199 --pod-infra-container-image=602401143452.dkr.ecr.us-east-1.amazonaws.com/eks/pause-amd64:3.1' ` fs.Check(t, "/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf", expected, 0640) - expected = `[Service] -Environment='KUBELET_MAX_PODS=--max-pods=27' -` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-max-pods.conf", expected, 0640) - - expected = `[Service] -Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu=70m,memory=1024Mi' + expected = `kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: 0.0.0.0 +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: "/etc/kubernetes/pki/ca.crt" +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +clusterDomain: cluster.local +clusterDNS: "172.20.0.10" +cgroupDriver: cgroupfs +featureGates: + RotateKubeletServerCertificate: true +serverTLSBootstrap: true +kubeReserved: + cpu: "70m" + memory: "1024Mi" +maxPods: "27" +evictionHard: + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% ` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf", expected, 0640) + fs.Check(t, "/etc/kubernetes/kubelet/config.yaml", expected, 0640) expected = `[Service] Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/worker=true"' ` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/40-labels.conf", expected, 0640) + fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-labels.conf", expected, 0640) expected = `[Service]` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/50-taints.conf", expected, 0640) + fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-taints.conf", expected, 0640) expected = `thisisthecertdata ` @@ -176,7 +188,7 @@ func TestConfigureSpotInstanceLabels(t *testing.T) { expected := `[Service] Environment='KUBELET_NODE_LABELS=--node-labels="node-role.kubernetes.io/spot-worker=true"' ` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/40-labels.conf", expected, 0640) + fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-labels.conf", expected, 0640) } func TestConfigureLabels(t *testing.T) { @@ -204,7 +216,7 @@ func TestConfigureLabels(t *testing.T) { expected := `[Service] Environment='KUBELET_NODE_LABELS=--node-labels="gpu-type=K80,node-role.kubernetes.io/worker=true"' ` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/40-labels.conf", expected, 0640) + fs.Check(t, "/etc/systemd/system/kubelet.service.d/20-labels.conf", expected, 0640) } func TestConfigureTaints(t *testing.T) { @@ -232,7 +244,7 @@ func TestConfigureTaints(t *testing.T) { expected := `[Service] Environment='KUBELET_NODE_TAINTS=--register-with-taints="node-role.kubernetes.io/worker=true:PreferNoSchedule"' ` - fs.Check(t, "/etc/systemd/system/kubelet.service.d/50-taints.conf", expected, 0640) + fs.Check(t, "/etc/systemd/system/kubelet.service.d/30-taints.conf", expected, 0640) } func instance(ip, dnsName string, tags map[string]string, spot bool) *node.Node { diff --git a/pkg/system/templates/etc/kubernetes/kubelet/config.yaml b/pkg/system/templates/etc/kubernetes/kubelet/config.yaml new file mode 100644 index 0000000..a907152 --- /dev/null +++ b/pkg/system/templates/etc/kubernetes/kubelet/config.yaml @@ -0,0 +1,32 @@ +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +address: 0.0.0.0 +authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 2m0s + enabled: true + x509: + clientCAFile: "/etc/kubernetes/pki/ca.crt" +authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 5m0s + cacheUnauthorizedTTL: 30s +clusterDomain: cluster.local +clusterDNS: "{{.Node.ClusterDNS}}" +cgroupDriver: cgroupfs +featureGates: + RotateKubeletServerCertificate: true +serverTLSBootstrap: true +{{- if and .Node.ReservedCPU .Node.ReservedMemory }} +kubeReserved: + cpu: "{{.Node.ReservedCPU}}" + memory: "{{.Node.ReservedMemory}}" +{{ end -}} +maxPods: "{{.Node.MaxPods}}" +evictionHard: + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service b/pkg/system/templates/etc/systemd/system/kubelet.service index afb4aaf..a126ac3 100644 --- a/pkg/system/templates/etc/systemd/system/kubelet.service +++ b/pkg/system/templates/etc/systemd/system/kubelet.service @@ -6,23 +6,12 @@ Requires=docker.service [Service] ExecStart=/usr/bin/kubelet \ - --address=0.0.0.0 \ - --authentication-token-webhook \ - --authorization-mode=Webhook \ --allow-privileged=true \ --cloud-provider=aws \ - --cluster-domain=cluster.local \ - --cni-bin-dir=/opt/cni/bin \ - --cni-conf-dir=/etc/cni/net.d \ + --config /etc/kubernetes/kubelet/config.yaml \ --container-runtime=docker \ - --eviction-hard=memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5% \ --network-plugin=cni \ - --cgroup-driver=cgroupfs \ - --register-node=true \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --feature-gates=RotateKubeletServerCertificate=true \ - --anonymous-auth=false \ - --client-ca-file=/etc/kubernetes/pki/ca.crt $KUBELET_ARGS $KUBELET_MAX_PODS $KUBELET_KUBE_RESERVED $KUBELET_NODE_LABELS $KUBELET_NODE_TAINTS $KUBELET_EXTRA_ARGS + --kubeconfig=/var/lib/kubelet/kubeconfig $KUBELET_ARGS $KUBELET_NODE_LABELS $KUBELET_NODE_TAINTS $KUBELET_EXTRA_ARGS Restart=always StartLimitInterval=0 diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf index 97ab6ce..2d53228 100644 --- a/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf +++ b/pkg/system/templates/etc/systemd/system/kubelet.service.d/10-kubelet-args.conf @@ -1,2 +1,2 @@ [Service] -Environment='KUBELET_ARGS=--node-ip={{.Node.PrivateIpAddress}} --cluster-dns={{.Node.ClusterDNS}} --pod-infra-container-image=602401143452.dkr.ecr.{{.Node.Region}}.amazonaws.com/eks/pause-amd64:3.1' +Environment='KUBELET_ARGS=--node-ip={{.Node.PrivateIpAddress}} --pod-infra-container-image=602401143452.dkr.ecr.{{.Node.Region}}.amazonaws.com/eks/pause-amd64:3.1' diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/40-labels.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-labels.conf similarity index 100% rename from pkg/system/templates/etc/systemd/system/kubelet.service.d/40-labels.conf rename to pkg/system/templates/etc/systemd/system/kubelet.service.d/20-labels.conf diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf deleted file mode 100644 index d3ae0c7..0000000 --- a/pkg/system/templates/etc/systemd/system/kubelet.service.d/20-max-pods.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -Environment='KUBELET_MAX_PODS=--max-pods={{.Node.MaxPods}}' diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf deleted file mode 100644 index 59f75c6..0000000 --- a/pkg/system/templates/etc/systemd/system/kubelet.service.d/30-kube-reserved.conf +++ /dev/null @@ -1,4 +0,0 @@ -[Service] -{{- if and .Node.ReservedCPU .Node.ReservedMemory }} -Environment='KUBELET_KUBE_RESERVED=--kube-reserved=cpu={{.Node.ReservedCPU}},memory={{.Node.ReservedMemory}}' -{{ end -}} diff --git a/pkg/system/templates/etc/systemd/system/kubelet.service.d/50-taints.conf b/pkg/system/templates/etc/systemd/system/kubelet.service.d/30-taints.conf similarity index 100% rename from pkg/system/templates/etc/systemd/system/kubelet.service.d/50-taints.conf rename to pkg/system/templates/etc/systemd/system/kubelet.service.d/30-taints.conf