From 924beb4b0cb3ca076c29c85983070d0f66dddc5c Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 2 Mar 2020 23:30:33 -0800 Subject: [PATCH] Enable Kubelet TLS bootstrap and NodeRestriction * Enable bootstrap token authentication on kube-apiserver * Generate the bootstrap.kubernetes.io/token Secret that may be used as a bootstrap token * Generate a bootstrap kubeconfig (with a bootstrap token) to be securely distributed to nodes. Each Kubelet will use the bootstrap kubeconfig to authenticate to kube-apiserver as `system:bootstrappers` and send a node-unique CSR for kube-controller-manager to automatically approve to issue a Kubelet certificate and kubeconfig (expires in 72 hours) * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the `system:node-bootstrapper` ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr nodeclient ClusterRole * Add ClusterRoleBinding for bootstrap token subjects (`system:bootstrappers`) to have the csr selfnodeclient ClusterRole * Enable NodeRestriction admission controller to limit the scope of Node or Pod objects a Kubelet can modify to those of the node itself * Ability for a Kubelet to delete its Node object is retained as preemptible nodes or those in auto-scaling instance groups need to be able to remove themselves on shutdown. This need continues to have precedence over any risk of a node deleting itself maliciously Security notes: 1. Issued Kubelet certificates authenticate as user `system:node:NAME` and group `system:nodes` and are limited in their authorization to perform API operations by Node authorization and NodeRestriction admission. Previously, a Kubelet's authorization was broader. This is the primary security motivation. 2. The bootstrap kubeconfig credential has the same sensitivity as the previous generated TLS client-certificate kubeconfig. It must be distributed securely to nodes. Its compromise still allows an attacker to obtain a Kubelet kubeconfig 3. Bootstrapping Kubelet kubeconfig's with a limited lifetime offers a slight security improvement. * An attacker who obtains the kubeconfig can likely obtain the bootstrap kubeconfig as well, to obtain the ability to renew their access * A compromised bootstrap kubeconfig could plausibly be handled by replacing the bootstrap token Secret, distributing the token to new nodes, and expiration. Whereas a compromised TLS-client certificate kubeconfig can't be revoked (no CRL). However, replacing a bootstrap token can be impractical in real cluster environments, so the limited lifetime is mostly a theoretical benefit. * Cluster CSR objects are visible via kubectl which is nice 4. Bootstrapping node-unique Kubelet kubeconfigs means Kubelet clients have more identity information, which can improve the utility of audits and future features Rel: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ --- auth.tf | 33 +++++++++++-------- manifests.tf | 2 ++ outputs.tf | 2 +- resources/kubeconfig-bootstrap | 15 +++++++++ .../bootstrap-cluster-role-binding.yaml | 13 ++++++++ ...trap-new-approve-cluster-role-binding.yaml | 13 ++++++++ ...ap-renew-approve-cluster-role-binding.yaml | 13 ++++++++ resources/manifests/bootstrap-token.yaml | 12 +++++++ .../kubelet-delete-cluster-role-binding.yaml | 4 +-- .../kubelet-nodes-cluster-role-binding.yaml | 12 ------- .../static-manifests/kube-apiserver.yaml | 4 ++- .../kube-controller-manager.yaml | 1 + versions.tf | 1 + 13 files changed, 96 insertions(+), 29 deletions(-) create mode 100644 resources/kubeconfig-bootstrap create mode 100644 resources/manifests/bootstrap-cluster-role-binding.yaml create mode 100644 resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml create mode 100644 resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml create mode 100644 resources/manifests/bootstrap-token.yaml delete mode 100644 resources/manifests/kubelet-nodes-cluster-role-binding.yaml diff --git a/auth.tf b/auth.tf index 06bed388..fdbf1119 100644 --- a/auth.tf +++ b/auth.tf @@ -5,18 +5,33 @@ locals { } } -# Generated kubeconfig for Kubelets -data "template_file" "kubeconfig-kubelet" { - template = file("${path.module}/resources/kubeconfig-kubelet") +# Generate a cryptographically random token id (public) +resource random_string "bootstrap-token-id" { + length = 6 + upper = false + special = false +} + +# Generate a cryptographically random token secret +resource random_string "bootstrap-token-secret" { + length = 16 + upper = false + special = false +} + +# Generated kubeconfig to bootstrap Kubelets +data "template_file" "kubeconfig-bootstrap" { + template = file("${path.module}/resources/kubeconfig-bootstrap") vars = { ca_cert = base64encode(tls_self_signed_cert.kube-ca.cert_pem) - kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem) - kubelet_key = base64encode(tls_private_key.kubelet.private_key_pem) server = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port) + token_id = random_string.bootstrap-token-id.result + token_secret = random_string.bootstrap-token-secret.result } } + # Generated admin kubeconfig to bootstrap control plane data "template_file" "kubeconfig-admin" { template = file("${path.module}/resources/kubeconfig-admin") @@ -30,14 +45,6 @@ data "template_file" "kubeconfig-admin" { } } -# Generated kubeconfig for Kubelets -resource "local_file" "kubeconfig-kubelet" { - count = var.asset_dir == "" ? 0 : 1 - - content = data.template_file.kubeconfig-kubelet.rendered - filename = "${var.asset_dir}/auth/kubeconfig-kubelet" -} - # Generated admin kubeconfig to bootstrap control plane resource "local_file" "kubeconfig-admin" { count = var.asset_dir == "" ? 0 : 1 diff --git a/manifests.tf b/manifests.tf index 2f9bb5cc..0043f82c 100644 --- a/manifests.tf +++ b/manifests.tf @@ -36,6 +36,8 @@ locals { trusted_certs_dir = var.trusted_certs_dir server = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port) daemonset_tolerations = var.daemonset_tolerations + token_id = random_string.bootstrap-token-id.result + token_secret = random_string.bootstrap-token-secret.result } ) } diff --git a/outputs.tf b/outputs.tf index 2952f497..93dbb873 100644 --- a/outputs.tf +++ b/outputs.tf @@ -5,7 +5,7 @@ output "cluster_dns_service_ip" { // Generated kubeconfig for Kubelets (i.e. lower privilege than admin) output "kubeconfig-kubelet" { - value = data.template_file.kubeconfig-kubelet.rendered + value = data.template_file.kubeconfig-bootstrap.rendered sensitive = true } diff --git a/resources/kubeconfig-bootstrap b/resources/kubeconfig-bootstrap new file mode 100644 index 00000000..f0660cd2 --- /dev/null +++ b/resources/kubeconfig-bootstrap @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: ${server} + certificate-authority-data: ${ca_cert} +users: +- name: kubelet + user: + token: ${token_id}.${token_secret} +contexts: +- context: + cluster: local + user: kubelet diff --git a/resources/manifests/bootstrap-cluster-role-binding.yaml b/resources/manifests/bootstrap-cluster-role-binding.yaml new file mode 100644 index 00000000..02d87229 --- /dev/null +++ b/resources/manifests/bootstrap-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Bind system:bootstrappers to ClusterRole for node bootstrap +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers diff --git a/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml b/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml new file mode 100644 index 00000000..2c022716 --- /dev/null +++ b/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Approve new CSRs from "system:bootstrappers" subjects +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-approve-new +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:nodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers diff --git a/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml b/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml new file mode 100644 index 00000000..12c0e1fe --- /dev/null +++ b/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Approve renewal CSRs from "system:nodes" subjects +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-approve-renew +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes diff --git a/resources/manifests/bootstrap-token.yaml b/resources/manifests/bootstrap-token.yaml new file mode 100644 index 00000000..1c0bbe15 --- /dev/null +++ b/resources/manifests/bootstrap-token.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +type: bootstrap.kubernetes.io/token +metadata: + # Name MUST be of form "bootstrap-token-" + name: bootstrap-token-${token_id} + namespace: kube-system +stringData: + description: "Typhoon generated bootstrap token" + token-id: ${token_id} + token-secret: ${token_secret} + usage-bootstrap-authentication: "true" diff --git a/resources/manifests/kubelet-delete-cluster-role-binding.yaml b/resources/manifests/kubelet-delete-cluster-role-binding.yaml index 7e736ff9..fe0581b0 100644 --- a/resources/manifests/kubelet-delete-cluster-role-binding.yaml +++ b/resources/manifests/kubelet-delete-cluster-role-binding.yaml @@ -7,6 +7,6 @@ roleRef: kind: ClusterRole name: kubelet-delete subjects: -- kind: Group +- apiGroup: rbac.authorization.k8s.io + kind: Group name: system:nodes - apiGroup: rbac.authorization.k8s.io diff --git a/resources/manifests/kubelet-nodes-cluster-role-binding.yaml b/resources/manifests/kubelet-nodes-cluster-role-binding.yaml deleted file mode 100644 index 5dfcc170..00000000 --- a/resources/manifests/kubelet-nodes-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-nodes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- kind: Group - name: system:nodes - apiGroup: rbac.authorization.k8s.io diff --git a/resources/static-manifests/kube-apiserver.yaml b/resources/static-manifests/kube-apiserver.yaml index 75bcd66e..c1638275 100644 --- a/resources/static-manifests/kube-apiserver.yaml +++ b/resources/static-manifests/kube-apiserver.yaml @@ -22,8 +22,10 @@ spec: - --advertise-address=$(POD_IP) - --allow-privileged=true - --anonymous-auth=false - - --authorization-mode=RBAC + - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/secrets/ca.crt + - --enable-admission-plugins=NodeRestriction + - --enable-bootstrap-token-auth=true - --cloud-provider=${cloud_provider} - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt diff --git a/resources/static-manifests/kube-controller-manager.yaml b/resources/static-manifests/kube-controller-manager.yaml index fee232b1..c1b04568 100644 --- a/resources/static-manifests/kube-controller-manager.yaml +++ b/resources/static-manifests/kube-controller-manager.yaml @@ -25,6 +25,7 @@ spec: - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key - --configure-cloud-routes=false + - --experimental-cluster-signing-duration=72h - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins - --kubeconfig=/etc/kubernetes/secrets/kubeconfig - --leader-elect=true diff --git a/versions.tf b/versions.tf index 4b3b1a9c..37f5ebed 100644 --- a/versions.tf +++ b/versions.tf @@ -4,6 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { local = "~> 1.2" + random = "~> 2.2" template = "~> 2.1" tls = "~> 2.0" }