diff --git a/auth.tf b/auth.tf index 06bed388..fdbf1119 100644 --- a/auth.tf +++ b/auth.tf @@ -5,18 +5,33 @@ locals { } } -# Generated kubeconfig for Kubelets -data "template_file" "kubeconfig-kubelet" { - template = file("${path.module}/resources/kubeconfig-kubelet") +# Generate a cryptographically random token id (public) +resource random_string "bootstrap-token-id" { + length = 6 + upper = false + special = false +} + +# Generate a cryptographically random token secret +resource random_string "bootstrap-token-secret" { + length = 16 + upper = false + special = false +} + +# Generated kubeconfig to bootstrap Kubelets +data "template_file" "kubeconfig-bootstrap" { + template = file("${path.module}/resources/kubeconfig-bootstrap") vars = { ca_cert = base64encode(tls_self_signed_cert.kube-ca.cert_pem) - kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem) - kubelet_key = base64encode(tls_private_key.kubelet.private_key_pem) server = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port) + token_id = random_string.bootstrap-token-id.result + token_secret = random_string.bootstrap-token-secret.result } } + # Generated admin kubeconfig to bootstrap control plane data "template_file" "kubeconfig-admin" { template = file("${path.module}/resources/kubeconfig-admin") @@ -30,14 +45,6 @@ data "template_file" "kubeconfig-admin" { } } -# Generated kubeconfig for Kubelets -resource "local_file" "kubeconfig-kubelet" { - count = var.asset_dir == "" ? 0 : 1 - - content = data.template_file.kubeconfig-kubelet.rendered - filename = "${var.asset_dir}/auth/kubeconfig-kubelet" -} - # Generated admin kubeconfig to bootstrap control plane resource "local_file" "kubeconfig-admin" { count = var.asset_dir == "" ? 0 : 1 diff --git a/manifests.tf b/manifests.tf index 2f9bb5cc..0043f82c 100644 --- a/manifests.tf +++ b/manifests.tf @@ -36,6 +36,8 @@ locals { trusted_certs_dir = var.trusted_certs_dir server = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port) daemonset_tolerations = var.daemonset_tolerations + token_id = random_string.bootstrap-token-id.result + token_secret = random_string.bootstrap-token-secret.result } ) } diff --git a/outputs.tf b/outputs.tf index 2952f497..93dbb873 100644 --- a/outputs.tf +++ b/outputs.tf @@ -5,7 +5,7 @@ output "cluster_dns_service_ip" { // Generated kubeconfig for Kubelets (i.e. lower privilege than admin) output "kubeconfig-kubelet" { - value = data.template_file.kubeconfig-kubelet.rendered + value = data.template_file.kubeconfig-bootstrap.rendered sensitive = true } diff --git a/resources/kubeconfig-bootstrap b/resources/kubeconfig-bootstrap new file mode 100644 index 00000000..f0660cd2 --- /dev/null +++ b/resources/kubeconfig-bootstrap @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + server: ${server} + certificate-authority-data: ${ca_cert} +users: +- name: kubelet + user: + token: ${token_id}.${token_secret} +contexts: +- context: + cluster: local + user: kubelet diff --git a/resources/manifests/bootstrap-cluster-role-binding.yaml b/resources/manifests/bootstrap-cluster-role-binding.yaml new file mode 100644 index 00000000..02d87229 --- /dev/null +++ b/resources/manifests/bootstrap-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Bind system:bootstrappers to ClusterRole for node bootstrap +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers diff --git a/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml b/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml new file mode 100644 index 00000000..2c022716 --- /dev/null +++ b/resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Approve new CSRs from "system:bootstrappers" subjects +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-approve-new +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:nodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers diff --git a/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml b/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml new file mode 100644 index 00000000..12c0e1fe --- /dev/null +++ b/resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml @@ -0,0 +1,13 @@ +# Approve renewal CSRs from "system:nodes" subjects +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bootstrap-approve-renew +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes diff --git a/resources/manifests/bootstrap-token.yaml b/resources/manifests/bootstrap-token.yaml new file mode 100644 index 00000000..1c0bbe15 --- /dev/null +++ b/resources/manifests/bootstrap-token.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +type: bootstrap.kubernetes.io/token +metadata: + # Name MUST be of form "bootstrap-token-" + name: bootstrap-token-${token_id} + namespace: kube-system +stringData: + description: "Typhoon generated bootstrap token" + token-id: ${token_id} + token-secret: ${token_secret} + usage-bootstrap-authentication: "true" diff --git a/resources/manifests/kubelet-delete-cluster-role-binding.yaml b/resources/manifests/kubelet-delete-cluster-role-binding.yaml index 7e736ff9..fe0581b0 100644 --- a/resources/manifests/kubelet-delete-cluster-role-binding.yaml +++ b/resources/manifests/kubelet-delete-cluster-role-binding.yaml @@ -7,6 +7,6 @@ roleRef: kind: ClusterRole name: kubelet-delete subjects: -- kind: Group +- apiGroup: rbac.authorization.k8s.io + kind: Group name: system:nodes - apiGroup: rbac.authorization.k8s.io diff --git a/resources/manifests/kubelet-nodes-cluster-role-binding.yaml b/resources/manifests/kubelet-nodes-cluster-role-binding.yaml deleted file mode 100644 index 5dfcc170..00000000 --- a/resources/manifests/kubelet-nodes-cluster-role-binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-nodes -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- kind: Group - name: system:nodes - apiGroup: rbac.authorization.k8s.io diff --git a/resources/static-manifests/kube-apiserver.yaml b/resources/static-manifests/kube-apiserver.yaml index 75bcd66e..c1638275 100644 --- a/resources/static-manifests/kube-apiserver.yaml +++ b/resources/static-manifests/kube-apiserver.yaml @@ -22,8 +22,10 @@ spec: - --advertise-address=$(POD_IP) - --allow-privileged=true - --anonymous-auth=false - - --authorization-mode=RBAC + - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/secrets/ca.crt + - --enable-admission-plugins=NodeRestriction + - --enable-bootstrap-token-auth=true - --cloud-provider=${cloud_provider} - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt diff --git a/resources/static-manifests/kube-controller-manager.yaml b/resources/static-manifests/kube-controller-manager.yaml index fee232b1..c1b04568 100644 --- a/resources/static-manifests/kube-controller-manager.yaml +++ b/resources/static-manifests/kube-controller-manager.yaml @@ -25,6 +25,7 @@ spec: - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key - --configure-cloud-routes=false + - --experimental-cluster-signing-duration=72h - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins - --kubeconfig=/etc/kubernetes/secrets/kubeconfig - --leader-elect=true diff --git a/versions.tf b/versions.tf index 4b3b1a9c..37f5ebed 100644 --- a/versions.tf +++ b/versions.tf @@ -4,6 +4,7 @@ terraform { required_version = "~> 0.12.0" required_providers { local = "~> 1.2" + random = "~> 2.2" template = "~> 2.1" tls = "~> 2.0" }