From 41b9f343dee3a94673c4b013e345140b166a7e49 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 25 Apr 2020 16:50:51 -0700 Subject: [PATCH] Enable Kubelet TLS bootstrap and NodeRestriction * Enable bootstrap token authentication on kube-apiserver * Generate the bootstrap.kubernetes.io/token Secret that may be used as a bootstrap token * Generate a bootstrap kubeconfig (with a bootstrap token) to be securely distributed to nodes. Each Kubelet will use the bootstrap kubeconfig to authenticate to kube-apiserver as `system:bootstrappers` and send a node-unique CSR for kube-controller-manager to automatically approve to issue a Kubelet certificate and kubeconfig (expires in 72 hours) * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the `system:node-bootstrapper` ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr nodeclient ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr selfnodeclient ClusterRole * Enable NodeRestriction admission controller to limit the scope of Node or Pod objects a Kubelet can modify to those of the node itself * Ability for a Kubelet to delete its Node object is retained as preemptible nodes or those in auto-scaling instance groups need to be able to remove themselves on shutdown. This need continues to have precedence over any risk of a node deleting itself maliciously Security notes: 1. Issued Kubelet certificates authenticate as user `system:node:NAME` and group `system:nodes` and are limited in their authorization to perform API operations by Node authorization and NodeRestriction admission. Previously, a Kubelet's authorization was broader. This is the primary security motivation. 2. The bootstrap kubeconfig credential has the same sensitivity as the previous generated TLS client-certificate kubeconfig. It must be distributed securely to nodes. Its compromise still allows an attacker to obtain a Kubelet kubeconfig 3. Bootstrapping Kubelet kubeconfig's with a limited lifetime offers a slight security improvement. * An attacker who obtains the kubeconfig can likely obtain the bootstrap kubeconfig as well, to obtain the ability to renew their access * A compromised bootstrap kubeconfig could plausibly be handled by replacing the bootstrap token Secret, distributing the token to new nodes, and expiration. Whereas a compromised TLS-client certificate kubeconfig can't be revoked (no CRL). However, replacing a bootstrap token can be impractical in real cluster environments, so the limited lifetime is mostly a theoretical benefit. * Cluster CSR objects are visible via kubectl which is nice 4. Bootstrapping node-unique Kubelet kubeconfigs means Kubelet clients have more identity information, which can improve the utility of audits and future features Rel: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ Rel: https://github.com/poseidon/terraform-render-bootstrap/pull/185 --- bootstrap.tf | 2 +- cl/controller.yaml | 6 ++++-- cl/worker.yaml | 6 ++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bootstrap.tf b/bootstrap.tf index 45179f9..6cae750 100644 --- a/bootstrap.tf +++ b/bootstrap.tf @@ -1,6 +1,6 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c62c7f5a1a3a3f9cebe7c1382077ad2dbf3727e6" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=924beb4b0cb3ca076c29c85983070d0f66dddc5c" cluster_name = var.cluster_name api_servers = [var.k8s_domain_name] diff --git a/cl/controller.yaml b/cl/controller.yaml index 9cde76e..9f5c5f5 100644 --- a/cl/controller.yaml +++ b/cl/controller.yaml @@ -57,7 +57,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -107,6 +107,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -115,7 +116,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/master \ @@ -123,6 +124,7 @@ systemd: --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always diff --git a/cl/worker.yaml b/cl/worker.yaml index 164323c..dde9d75 100644 --- a/cl/worker.yaml +++ b/cl/worker.yaml @@ -30,7 +30,7 @@ systemd: - name: kubelet.service contents: | [Unit] - Description=Kubelet via Hyperkube + Description=Kubelet Wants=rpc-statd.service [Service] Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} @@ -80,6 +80,7 @@ systemd: --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ + --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \ --cgroup-driver=$${KUBELET_CGROUP_DRIVER} \ --client-ca-file=/etc/kubernetes/ca.crt \ --cluster_dns=${cluster_dns_service_ip} \ @@ -88,7 +89,7 @@ systemd: --exit-on-lock-contention \ --healthz-port=0 \ --hostname-override=${domain_name} \ - --kubeconfig=/etc/kubernetes/kubeconfig \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ --lock-file=/var/run/lock/kubelet.lock \ --network-plugin=cni \ --node-labels=node.kubernetes.io/node \ @@ -100,6 +101,7 @@ systemd: %{~ endfor ~} --pod-manifest-path=/etc/kubernetes/manifests \ --read-only-port=0 \ + --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid Restart=always