Manage clusters without using a local asset_dir

CHANGES.md

@@ -4,9 +4,15 @@ Notable changes between versions.
 
 ## Latest
 
+* Manage clusters without using a local `asset_dir` ([#595](https://github.com/poseidon/typhoon/pull/595))
+  * Change `asset_dir` to be optional. Default to "" to skip writing assets locally
+  * Keep cluster assets (TLS materials, kubeconfig) only in Terraform state, which supports different [remote backends](https://www.terraform.io/docs/backends/types/remote.html) and optional encryption at rest
+  * Allow `terraform apply` from stateless automation systems
+  * Improve asset unpacking on controllers to remove unused materials
+  * Obtain a kubeconfig from Terraform module outputs
 * Update CoreDNS from v1.6.2 to v1.6.5 ([#588](https://github.com/poseidon/typhoon/pull/588))
   * Add health `lameduck` option to wait before shutdown
-* Add CPU requests to control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589))
+* Add CPU requests for control plane static pods ([#589](https://github.com/poseidon/typhoon/pull/589))
   * May provide slight edge case benefits and aligns with upstream
 * Replace usage of `template_dir` with `templatefile` function ([#587](https://github.com/poseidon/typhoon/pull/587))
   * Require Terraform version v0.12.6+ (action required)

README.md

@@ -47,7 +47,7 @@ A preview of Typhoon for [Fedora CoreOS](https://getfedora.org/coreos/) is avail
 Define a Kubernetes cluster by using the Terraform module for your chosen platform and operating system. Here's a minimal example:
 
 ```tf
-module "google-cloud-yavin" {
+module "yavin" {
   source = "git::https://github.com/poseidon/typhoon//google-cloud/container-linux/kubernetes?ref=v1.16.3"
 
   # Google Cloud
@@ -58,12 +58,17 @@ module "google-cloud-yavin" {
 
   # configuration
   ssh_authorized_key = "ssh-rsa AAAAB3Nz..."
-  asset_dir          = "/home/user/.secrets/clusters/yavin"
 
   # optional
   worker_count = 2
   worker_preemptible = true
 }
+
+# Obtain cluster kubeconfig
+resource "local_file" "kubeconfig-yavin" {
+  content  = module.yavin.kubeconfig-admin
+  filename = "/home/user/.kube/configs/yavin-config"
+}
 ```
 
 Initialize modules, plan the changes to be made, and apply the changes.
@@ -79,7 +84,7 @@ Apply complete! Resources: 64 added, 0 changed, 0 destroyed.
 In 4-8 minutes (varies by platform), the cluster will be ready. This Google Cloud example creates a `yavin.example.com` DNS record to resolve to a network load balancer across controller nodes.
 
 ```sh
-$ export KUBECONFIG=/home/user/.secrets/clusters/yavin/auth/kubeconfig
+$ export KUBECONFIG=/home/user/.kube/configs/yavin-config
 $ kubectl get nodes
 NAME                                       ROLES    STATUS  AGE  VERSION
 yavin-controller-0.c.example-com.internal  <none>   Ready   6m   v1.16.3

aws/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -108,6 +108,8 @@ systemd:
         ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*'
         ExecStart=/usr/bin/rkt run \
             --trust-keys-from-https \
+            --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
+            --mount volume=config,target=/etc/kubernetes/secrets \
             --volume assets,kind=host,source=/opt/bootstrap/assets \
             --mount volume=assets,target=/assets \
             --volume script,kind=host,source=/opt/bootstrap/apply \
@@ -135,13 +137,35 @@ storage:
         inline: |
           KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
           KUBELET_IMAGE_TAG=v1.16.3
+    - path: /opt/bootstrap/layout
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
+          awk '/#####/ {filename=$2; next} {print > filename}' assets
+          mkdir -p /etc/ssl/etcd/etcd
+          mkdir -p /etc/kubernetes/bootstrap-secrets
+          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
+          mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
+          chown -R etcd:etcd /etc/ssl/etcd
+          chmod -R 500 /etc/ssl/etcd
+          mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
+          mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
+          sudo mkdir -p /etc/kubernetes/manifests
+          sudo mv static-manifests/* /etc/kubernetes/manifests/
+          sudo mkdir -p /opt/bootstrap/assets
+          sudo mv manifests /opt/bootstrap/assets/manifests
+          sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking
+          rm -rf assets auth static-manifests tls
     - path: /opt/bootstrap/apply
       filesystem: root
       mode: 0544
       contents:
         inline: |
           #!/bin/bash -e
-          export KUBECONFIG=/assets/auth/kubeconfig
+          export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
           until kubectl version; do
             echo "Waiting for static pod control plane"
             sleep 5

aws/container-linux/kubernetes/ssh.tf

@@ -1,3 +1,12 @@
+locals {
+  # format assets for distribution
+  assets_bundle = [
+    # header with the unpack location
+    for key, value in module.bootstrap.assets_dist:
+    format("##### %s\n%s", key, value)
+  ]
+}
+
 # Secure copy assets to controllers.
 resource "null_resource" "copy-controller-secrets" {
   count = var.controller_count
@@ -14,63 +23,13 @@ resource "null_resource" "copy-controller-secrets" {
   }
 
   provisioner "file" {
-    content     = module.bootstrap.etcd_ca_cert
-    destination = "$HOME/etcd-client-ca.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_client_cert
-    destination = "$HOME/etcd-client.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_client_key
-    destination = "$HOME/etcd-client.key"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_server_cert
-    destination = "$HOME/etcd-server.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_server_key
-    destination = "$HOME/etcd-server.key"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_peer_cert
-    destination = "$HOME/etcd-peer.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_peer_key
-    destination = "$HOME/etcd-peer.key"
-  }
-
-  provisioner "file" {
-    source      = var.asset_dir
+    content     = join("\n", local.assets_bundle)
     destination = "$HOME/assets"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "sudo mkdir -p /etc/ssl/etcd/etcd",
-      "sudo mv etcd-client* /etc/ssl/etcd/",
-      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
-      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
-      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
-      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
-      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
-      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
-      "sudo chown -R etcd:etcd /etc/ssl/etcd",
-      "sudo chmod -R 500 /etc/ssl/etcd",
-      "sudo mv $HOME/assets /opt/bootstrap/assets",
-      "sudo mkdir -p /etc/kubernetes/manifests",
-      "sudo mkdir -p /etc/kubernetes/bootstrap-secrets",
-      "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/",
-      "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/",
-      "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/",
+      "sudo /opt/bootstrap/layout",
     ]
   }
 }

aws/container-linux/kubernetes/variables.tf

@@ -99,6 +99,7 @@ variable "ssh_authorized_key" {
 variable "asset_dir" {
   type        = string
   description = "Absolute path to a directory where generated assets should be placed (contains secrets)"
+  default     = ""
 }
 
 variable "networking" {

aws/fedora-coreos/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

aws/fedora-coreos/kubernetes/fcc/controller.yaml

@@ -119,6 +119,7 @@ systemd:
         ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*'
         ExecStart=/usr/bin/podman run --name bootstrap \
             --network host \
+            --volume /etc/kubernetes/bootstrap-secrets:/etc/kubernetes/secrets:ro,Z \
             --volume /opt/bootstrap/assets:/assets:ro,Z \
             --volume /opt/bootstrap/apply:/apply:ro,Z \
             k8s.gcr.io/hyperkube:v1.16.3 \
@@ -135,12 +136,33 @@ storage:
       contents:
         inline: |
           ${kubeconfig}
+    - path: /opt/bootstrap/layout
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
+          awk '/#####/ {filename=$2; next} {print > filename}' assets
+          mkdir -p /etc/ssl/etcd/etcd
+          mkdir -p /etc/kubernetes/bootstrap-secrets
+          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
+          mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
+          chown -R etcd:etcd /etc/ssl/etcd
+          chmod -R 500 /etc/ssl/etcd
+          mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
+          mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
+          sudo mkdir -p /etc/kubernetes/manifests
+          sudo mv static-manifests/* /etc/kubernetes/manifests/
+          sudo mkdir -p /opt/bootstrap/assets
+          sudo mv manifests /opt/bootstrap/assets/manifests
+          sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking
+          rm -rf assets auth static-manifests tls
     - path: /opt/bootstrap/apply
       mode: 0544
       contents:
         inline: |
           #!/bin/bash -e
-          export KUBECONFIG=/assets/auth/kubeconfig
+          export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
           until kubectl version; do
             echo "Waiting for static pod control plane"
             sleep 5

aws/fedora-coreos/kubernetes/ssh.tf

@@ -1,3 +1,12 @@
+locals {
+  # format assets for distribution
+  assets_bundle = [
+    # header with the unpack location
+    for key, value in module.bootstrap.assets_dist:
+    format("##### %s\n%s", key, value)
+  ]
+}
+
 # Secure copy assets to controllers.
 resource "null_resource" "copy-controller-secrets" {
   count = var.controller_count
@@ -12,65 +21,15 @@ resource "null_resource" "copy-controller-secrets" {
     user    = "core"
     timeout = "15m"
   }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_ca_cert
-    destination = "$HOME/etcd-client-ca.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_client_cert
-    destination = "$HOME/etcd-client.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_client_key
-    destination = "$HOME/etcd-client.key"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_server_cert
-    destination = "$HOME/etcd-server.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_server_key
-    destination = "$HOME/etcd-server.key"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_peer_cert
-    destination = "$HOME/etcd-peer.crt"
-  }
-
-  provisioner "file" {
-    content     = module.bootstrap.etcd_peer_key
-    destination = "$HOME/etcd-peer.key"
-  }
-
+
   provisioner "file" {
-    source      = var.asset_dir
+    content     = join("\n", local.assets_bundle)
     destination = "$HOME/assets"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "sudo mkdir -p /etc/ssl/etcd/etcd",
-      "sudo mv etcd-client* /etc/ssl/etcd/",
-      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt",
-      "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt",
-      "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key",
-      "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt",
-      "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt",
-      "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key",
-      "sudo chown -R etcd:etcd /etc/ssl/etcd",
-      "sudo chmod -R 500 /etc/ssl/etcd",
-      "sudo mv $HOME/assets /opt/bootstrap/assets",
-      "sudo mkdir -p /etc/kubernetes/manifests",
-      "sudo mkdir -p /etc/kubernetes/bootstrap-secrets",
-      "sudo cp -r /opt/bootstrap/assets/tls/* /etc/kubernetes/bootstrap-secrets/",
-      "sudo cp /opt/bootstrap/assets/auth/kubeconfig /etc/kubernetes/bootstrap-secrets/",
-      "sudo cp -r /opt/bootstrap/assets/static-manifests/* /etc/kubernetes/manifests/"
+      "sudo /opt/bootstrap/layout",
     ]
   }
 }

aws/fedora-coreos/kubernetes/variables.tf

@@ -99,6 +99,7 @@ variable "ssh_authorized_key" {
 variable "asset_dir" {
   type        = string
   description = "Absolute path to a directory where generated assets should be placed (contains secrets)"
+  default     = ""
 }
 
 variable "networking" {

azure/container-linux/kubernetes/bootstrap.tf

@@ -1,6 +1,6 @@
 # Kubernetes assets (kubeconfig, manifests)
 module "bootstrap" {
-  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=0f1f16c612a6877d25a3fedcb476b3087a3de999"
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=dce49114a083436c5af6d9174b9c1248786ba3b6"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/container-linux/kubernetes/cl/controller.yaml.tmpl

@@ -106,6 +106,8 @@ systemd:
         ExecStartPre=-/usr/bin/bash -c 'set -x && [ -n "$(ls /opt/bootstrap/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootstrap/assets/manifests-*/* /opt/bootstrap/assets/manifests && rm -rf /opt/bootstrap/assets/manifests-*'
         ExecStart=/usr/bin/rkt run \
             --trust-keys-from-https \
+            --volume config,kind=host,source=/etc/kubernetes/bootstrap-secrets \
+            --mount volume=config,target=/etc/kubernetes/secrets \
             --volume assets,kind=host,source=/opt/bootstrap/assets \
             --mount volume=assets,target=/assets \
             --volume script,kind=host,source=/opt/bootstrap/apply \
@@ -133,13 +135,35 @@ storage:
         inline: |
           KUBELET_IMAGE_URL=docker://k8s.gcr.io/hyperkube
           KUBELET_IMAGE_TAG=v1.16.3
+    - path: /opt/bootstrap/layout
+      filesystem: root
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/bash -e
+          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
+          awk '/#####/ {filename=$2; next} {print > filename}' assets
+          mkdir -p /etc/ssl/etcd/etcd
+          mkdir -p /etc/kubernetes/bootstrap-secrets
+          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
+          mv tls/etcd/etcd-client* /etc/kubernetes/bootstrap-secrets/
+          chown -R etcd:etcd /etc/ssl/etcd
+          chmod -R 500 /etc/ssl/etcd
+          mv auth/kubeconfig /etc/kubernetes/bootstrap-secrets/
+          mv tls/k8s/* /etc/kubernetes/bootstrap-secrets/
+          sudo mkdir -p /etc/kubernetes/manifests
+          sudo mv static-manifests/* /etc/kubernetes/manifests/
+          sudo mkdir -p /opt/bootstrap/assets
+          sudo mv manifests /opt/bootstrap/assets/manifests
+          sudo mv manifests-networking /opt/bootstrap/assets/manifests-networking
+          rm -rf assets auth static-manifests tls
     - path: /opt/bootstrap/apply
       filesystem: root
       mode: 0544
       contents:
         inline: |
           #!/bin/bash -e
-          export KUBECONFIG=/assets/auth/kubeconfig
+          export KUBECONFIG=/etc/kubernetes/secrets/kubeconfig
           until kubectl version; do
             echo "Waiting for static pod control plane"
             sleep 5