Enable Kubelet TLS bootstrap and NodeRestriction

auth.tf

@@ -5,18 +5,33 @@ locals {
   }
 }
 
-# Generated kubeconfig for Kubelets
-data "template_file" "kubeconfig-kubelet" {
-  template = file("${path.module}/resources/kubeconfig-kubelet")
+# Generate a cryptographically random token id (public)
+resource random_string "bootstrap-token-id" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret
+resource random_string "bootstrap-token-secret" {
+  length  = 16
+  upper   = false
+  special = false
+}
+
+# Generated kubeconfig to bootstrap Kubelets
+data "template_file" "kubeconfig-bootstrap" {
+  template = file("${path.module}/resources/kubeconfig-bootstrap")
 
   vars = {
     ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
-    kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem)
-    kubelet_key  = base64encode(tls_private_key.kubelet.private_key_pem)
     server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    token_id     = random_string.bootstrap-token-id.result
+    token_secret = random_string.bootstrap-token-secret.result
   }
 }
 
+
 # Generated admin kubeconfig to bootstrap control plane
 data "template_file" "kubeconfig-admin" {
   template = file("${path.module}/resources/kubeconfig-admin")
@@ -30,14 +45,6 @@ data "template_file" "kubeconfig-admin" {
   }
 }
 
-# Generated kubeconfig for Kubelets
-resource "local_file" "kubeconfig-kubelet" {
-  count = var.asset_dir == "" ? 0 : 1
-
-  content  = data.template_file.kubeconfig-kubelet.rendered
-  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
-}
-
 # Generated admin kubeconfig to bootstrap control plane
 resource "local_file" "kubeconfig-admin" {
   count = var.asset_dir == "" ? 0 : 1

manifests.tf

@@ -36,6 +36,8 @@ locals {
         trusted_certs_dir      = var.trusted_certs_dir
         server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
         daemonset_tolerations  = var.daemonset_tolerations
+        token_id               = random_string.bootstrap-token-id.result
+        token_secret           = random_string.bootstrap-token-secret.result
       }
     )
   }

outputs.tf

@@ -5,7 +5,7 @@ output "cluster_dns_service_ip" {
 
 // Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
 output "kubeconfig-kubelet" {
-  value     = data.template_file.kubeconfig-kubelet.rendered
+  value     = data.template_file.kubeconfig-bootstrap.rendered
   sensitive = true
 }
 

resources/kubeconfig-bootstrap

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet

resources/manifests/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Bind system:bootstrappers to ClusterRole for node bootstrap
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-bootstrapper
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve new CSRs from "system:bootstrappers" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-new
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve renewal CSRs from "system:nodes" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-renew
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/bootstrap-token.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>"
+  name: bootstrap-token-${token_id}
+  namespace: kube-system
+stringData:
+  description: "Typhoon generated bootstrap token"
+  token-id: ${token_id}
+  token-secret: ${token_secret}
+  usage-bootstrap-authentication: "true"

resources/manifests/kubelet-delete-cluster-role-binding.yaml

@@ -7,6 +7,6 @@ roleRef:
   kind: ClusterRole
   name: kubelet-delete
 subjects:
-- kind: Group
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
   name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/static-manifests/kube-apiserver.yaml

@@ -22,8 +22,10 @@ spec:
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --anonymous-auth=false
-    - --authorization-mode=RBAC
+    - --authorization-mode=Node,RBAC
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --enable-admission-plugins=NodeRestriction
+    - --enable-bootstrap-token-auth=true
     - --cloud-provider=${cloud_provider}
     - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
     - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt

resources/static-manifests/kube-controller-manager.yaml

@@ -25,6 +25,7 @@ spec:
     - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
     - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
     - --configure-cloud-routes=false
+    - --experimental-cluster-signing-duration=72h
     - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
     - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
     - --leader-elect=true

versions.tf

@@ -4,6 +4,7 @@ terraform {
   required_version = "~> 0.12.0"
   required_providers {
     local    = "~> 1.2"
+    random   = "~> 2.2"
     template = "~> 2.1"
     tls      = "~> 2.0"
   }