Secure etcd deployment with TLS assets generated by kubeadm

ansible/roles/etcd/defaults/main.yml

@@ -2,3 +2,4 @@
 etcd_cluster_token: 444fddcc-beae-45bc-9da6-d941d446b595
 etcd_interface: eth0
 etcd_version: v3.2.10
+kubeadm_version: "v1.12.2"

ansible/roles/etcd/tasks/main.yml

@@ -1,4 +1,106 @@
 ---
+### Create TLS assets using kubeadm
+- name: download standalone kubeadm for cert generation
+  get_url:
+    url: "{{ kubeadm_release_url }}"
+    dest: /usr/local/bin/kubeadm
+    mode: 0744
+    force: no
+
+- name: determine whether kubeadm needs to create etcd ca
+  stat:
+    path: "/etc/kubernetes/pki/etcd/{{ item }}"
+  with_items:
+    - ca.crt
+    - ca.key
+  register: etcd_ca
+
+- name: determine whether kubeadm needs to create etcd peer and server certs
+  stat:
+    path: "/etc/kubernetes/pki/etcd/{{ item }}"
+  with_items:
+    - peer.crt
+    - peer.key
+    - server.crt
+    - server.key
+  register: etcd_local_tls_assets
+
+- name: determine whether kubeadm needs to create the apiserver etcd client certs
+  stat:
+    path: "/etc/kubernetes/pki/{{ item }}"
+  with_items:
+    - apiserver-etcd-client.crt
+    - apiserver-etcd-client.key
+  register: apiserver_etcd_certs
+  delegate_to: "{{ groups['etcd']|first }}"
+
+- name: create /etc/kubernetes directory
+  file:
+    dest: /etc/kubernetes
+    state: directory
+    owner: root
+    group: root
+
+- name: laydown etcd.conf to be used for cert generation
+  template:
+    src: etc/kubernetes/etcd.conf
+    dest: /etc/kubernetes/etcd.conf
+  with_items: "{{ etcd_ca.results }}"
+  when: item.stat.exists == False
+
+- name: generate etcd ca certs with kubeadm
+  command: "/usr/local/bin/kubeadm alpha phase certs etcd-ca"
+  delegate_to: "{{ groups['etcd']|first }}"
+  run_once: True
+  with_items: "{{ etcd_ca.results }}"
+  when: item.stat.exists == False
+
+- name: generate apiserver client certs with kubeadm
+  command: "/usr/local/bin/kubeadm alpha phase certs apiserver-etcd-client"
+  delegate_to: "{{ groups['etcd']|first }}"
+  run_once: True
+  with_items: "{{ apiserver_etcd_certs.results }}"
+  when: item.stat.exists == False
+
+- name: slurp the etcd ca certs
+  slurp:
+    src: /etc/kubernetes/pki/etcd/{{ item }}
+  with_items:
+    - ca.crt
+    - ca.key
+  register: etcd_ca_pki
+  delegate_to: "{{ groups['etcd']|first }}"
+  run_once: True
+
+- name: create kubernetes pki directory
+  file:
+    dest: /etc/kubernetes/pki/etcd
+    state: directory
+    owner: root
+    group: root
+
+- name: add etcd ca cert to the rest of the etcd nodes
+  no_log: True
+  copy:
+    dest: "{{ item.source }}"
+    content: "{{ item.content | b64decode }}"
+    owner: root
+    group: root
+    mode: 0700
+  with_items: "{{ etcd_ca_pki.results }}"
+  when: inventory_hostname != groups['etcd']|first
+
+- name: generate etcd peer certs with kubeadm
+  command: "/usr/local/bin/kubeadm alpha phase certs etcd-peer --config=/etc/kubernetes/etcd.conf"
+  with_items: "{{ etcd_local_tls_assets.results }}"
+  when: item.stat.exists == False
+
+- name: generate etcd server certs with kubeadm
+  command: "/usr/local/bin/kubeadm alpha phase certs etcd-server --config=/etc/kubernetes/etcd.conf"
+  with_items: "{{ etcd_local_tls_assets.results }}"
+  when: item.stat.exists == False
+
+### Download and start etcd
 - name: download and extract etcd binaries
   unarchive:
     remote_src: True

ansible/roles/etcd/templates/etc/kubernetes/etcd.conf

@@ -0,0 +1,10 @@
+apiVersion: "kubeadm.k8s.io/v1alpha3"
+kind: ClusterConfiguration
+etcd:
+    local:
+        serverCertSANs:
+        - "{{ hostvars[inventory_hostname]['ansible_fqdn'] }}"
+        - "{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}"
+        peerCertSANs:
+        - "{{ hostvars[inventory_hostname]['ansible_fqdn'] }}"
+        - "{{ hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address'] }}"

ansible/roles/etcd/templates/etc/systemd/system/etcd.service

@@ -14,13 +14,21 @@ TimeoutStartSec=0
 ExecStart=/usr/local/bin/etcd \
     --name={{ inventory_hostname }} \
     --data-dir=/var/lib/etcd \
-    --listen-client-urls=http://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2379 \
-    --advertise-client-urls=http://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2379 \
-    --listen-peer-urls=http://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2380 \
-    --initial-advertise-peer-urls=http://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2380 \
+    --listen-client-urls=https://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2379 \
+    --advertise-client-urls=https://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2379 \
+    --listen-peer-urls=https://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2380 \
+    --initial-advertise-peer-urls=https://{{hostvars[inventory_hostname]['ansible_' + etcd_interface]['ipv4']['address']}}:2380 \
     --initial-cluster={{ etcd_cluster_endpoints }} \
     --initial-cluster-token={{ etcd_cluster_token }} \
-    --initial-cluster-state=new
+    --initial-cluster-state=new \
+    --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt \
+    --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt \
+    --peer-key-file=/etc/kubernetes/pki/etcd/peer.key \
+    --peer-client-cert-auth=true \
+    --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt \
+    --cert-file=/etc/kubernetes/pki/etcd/server.crt \
+    --key-file=/etc/kubernetes/pki/etcd/server.key \
+    --client-cert-auth=true
 
 [Install]
 WantedBy=multi-user.target

ansible/roles/etcd/vars/main.yml

@@ -1,4 +1,5 @@
 ---
 etcd_release_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
-etcd_client_endpoints: "{{ groups['etcd']|map('extract', hostvars, ['ansible_' + etcd_interface, 'ipv4', 'address'])|map('regex_replace', '^(.*)$', 'http://\\1:2379')|list|sort }}"
-etcd_cluster_endpoints: "{% for host in groups['etcd']|sort %}{{hostvars[host]['inventory_hostname']}}=http://{{hostvars[host]['ansible_' + etcd_interface]['ipv4']['address']}}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+etcd_client_endpoints: "{{ groups['etcd']|map('extract', hostvars, ['ansible_' + etcd_interface, 'ipv4', 'address'])|map('regex_replace', '^(.*)$', 'https://\\1:2379')|list|sort }}"
+etcd_cluster_endpoints: "{% for host in groups['etcd']|sort %}{{hostvars[host]['inventory_hostname']}}=https://{{hostvars[host]['ansible_' + etcd_interface]['ipv4']['address']}}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+kubeadm_release_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"

ansible/roles/kubernetes-master/defaults/main.yml

@@ -8,3 +8,6 @@ kubernetes_master_kubeadm_config:
   apiServerCertSANs: "{{ kubernetes_common_api_ip | kube_lookup_hostname(kubernetes_common_api_fqdn, True) }}"
   etcd:
     endpoints: "{{ etcd_client_endpoints }}"
+    caFile: "/etc/kubernetes/pki/etcd/ca.crt"
+    certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt"
+    keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key"

ansible/roles/kubernetes-master/tasks/main.yml

@@ -4,6 +4,33 @@
     path: /etc/kubernetes/manifests/kube-apiserver.yaml
   register: kubeadm_apiserver_manifest
 
+- name: create kubernetes pki directory
+  file:
+    dest: /etc/kubernetes/pki/etcd
+    state: directory
+    owner: root
+    group: root
+
+- name: slurp the etcd pki assets destined for the masters
+  slurp: src=/etc/kubernetes/pki/{{ item }}
+  with_items:
+    - apiserver-etcd-client.crt
+    - apiserver-etcd-client.key
+    - etcd/ca.crt
+  register: etcd_ca_pki
+  delegate_to: "{{ groups['etcd']|first }}"
+  run_once: true
+
+- name: add etcd pki assets
+  no_log: True
+  copy:
+    dest: "{{ item.source }}"
+    content: "{{ item.content | b64decode }}"
+    owner: root
+    group: root
+    mode: 0700
+  with_items: "{{ etcd_ca_pki.results }}"
+
 - name: drop kubeadm template
   template:
     src: etc/kubernetes/kubeadm.conf