Initial implementation of K3s manager (#189)

* Unify Kubespray templates and remove unnecessary subpackage Signed-off-by: Din Music <din.music@din-cloud.com> * Makefile: Add makefile with build and test commands Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/utils/exec: Add remote and local exec Signed-off-by: Din Music <din.music@din-cloud.com> * embed: Add K3s templates and remove unnecessary host checks Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/tools/git: Add support for commit checkout Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/tools/ansible: Accept env vars as map instead of array Signed-off-by: Din Music <din.music@din-cloud.com> * pkg: Add k3s manager Signed-off-by: Din Music <din.music@din-cloud.com> * pkg: Rename executor to manager Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/tools/virtualenv: Simplify invocation by removing working dir argument Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/cluster: Manager wrapper with common logic Signed-off-by: Din Music <din.music@din-cloud.com> * embed/templates/k3s: Add support for node labels and taints, and external load balancer Signed-off-by: Din Music <din.music@din-cloud.com> * docs: Add k3s manager in configurtion section Signed-off-by: Din Music <din.music@din-cloud.com> * pkg/models/config: Always set load balancer default value Signed-off-by: Din Music <din.music@din-cloud.com> * pkg: Remove debug outputs Signed-off-by: Din Music <din.music@din-cloud.com> * embed/ansible: Fetch final k3s config Signed-off-by: Din Music <din.music@din-cloud.com> --------- Signed-off-by: Din Music <din.music@din-cloud.com>
MusicDin · Apr 16, 2024 · c68cf1e · c68cf1e
1 parent ca9f64c
commit c68cf1e
Show file tree

Hide file tree

Showing 50 changed files with 1,545 additions and 967 deletions.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,5 @@
+build:
+	go build -ldflags "-s -w" -trimpath -o kubitect ./cmd
+
+test:
+	go test ./... -v
diff --git a/docs/user-guide/configuration/kubernetes.md b/docs/user-guide/configuration/kubernetes.md
@@ -1,6 +1,7 @@
 [tag 2.0.0]: https://github.com/MusicDin/kubitect/releases/tag/v2.0.0
 [tag 2.2.0]: https://github.com/MusicDin/kubitect/releases/tag/v2.2.0
 [tag 3.0.0]: https://github.com/MusicDin/kubitect/releases/tag/v3.0.0
+[tag 3.4.0]: https://github.com/MusicDin/kubitect/releases/tag/v3.4.0
 
 <div markdown="1" class="text-center">
 # Kubernetes configuration
@@ -12,6 +13,23 @@ The Kubernetes section of the configuration file contains properties that are sp
 
 ## Configuration
 
+### Kubernetes manager
+
+:material-tag-arrow-up-outline: [v3.4.0][tag 3.4.0]
+&ensp;
+:octicons-file-symlink-file-24: Default: `kubespray`
+
+Specify manager that is used for deploying Kubernetes cluster. Supported values are `kubespray` and `k3s`.
+
+```yaml
+kubernetes:
+  manager: k3s
+```
+
+!!! warning "Warning"
+
+    Support for K3s manager has been added recently, therefore, it may not be fully stable.
+
 ### Kubernetes version
 
 :material-tag-arrow-up-outline: [v3.0.0][tag 3.0.0]
@@ -55,6 +73,10 @@ The following table shows the compatibility matrix of supported network plugins
 | **1.27**           | :material-check: | :material-check: | :material-check: | :material-check: |
 | **1.28**           | :material-check: | :material-check: | :material-check: | :material-check: |
 
+!!! note "Note"
+
+    K3s manager supports only `flannel` network plugin.
+
 ### Kubernetes DNS mode
 
 :material-tag-arrow-up-outline: [v2.0.0][tag 2.0.0]

diff --git a/docs/user-guide/reference/configuration.md b/docs/user-guide/reference/configuration.md
@@ -740,6 +740,19 @@ Each configuration property is documented with 5 columns: Property name, descrip
         </ul>
       </td>
     </tr>
+    <tr>
+      <td><code>kubernetes.manager</code></td>
+      <td>string</td>
+      <td>kubespray</td>
+      <td></td>
+      <td>
+        Manager that is used for deploying
+        Kubernetes cluster. Possible values are:
+        <ul>
+          <li><code>kubespray</code></li>
+          <li><code>k3s</code></li>
+        </ul>
+    </tr>
     <tr>
       <td><code>kubernetes.networkPlugin</code></td>
       <td>string</td>
@@ -753,6 +766,7 @@ Each configuration property is documented with 5 columns: Property name, descrip
           <li><code>flannel</code></li>
           <li><code>kube-router</code></li>
         </ul>
+        Note: k3s manager currently supports only flannel.
       </td>
     </tr>
     <tr>

diff --git a/embed/ansible/kubitect/finalize.yaml b/embed/ansible/kubitect/finalize.yaml
@@ -10,6 +10,17 @@
         dest: "{{ config_dir }}/admin.conf"
         flat: true
 
+- name: Fetch kubeconfig from first master node
+  hosts: server[0]
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    - name: Fetch Kubeconfig
+      fetch:
+        src: "/home/{{ ansible_user }}/.kube/config"
+        dest: "{{ config_dir }}/admin.conf"
+        flat: true
+
 - name: Finalize cluster deployment
   hosts: localhost
   gather_facts: false
@@ -46,22 +57,23 @@
   gather_facts: false
   any_errors_fatal: true
   become: false
+  vars:
+    addons_enabled: "{{ config.addons.rook.enabled | default(false) | bool }}"
 
   pre_tasks:
-    - name: Get system architecture fact
-      setup:
-        filter:
-          - ansible_architecture
+    - block:
+        - name: Get system architecture fact
+          setup:
+            filter:
+              - ansible_architecture
 
-    - name: Resolve system architecture
-      set_fact:
-        arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else ansible_architecture }}"
+        - name: Resolve system architecture
+          set_fact:
+            arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else ansible_architecture }}"
+      when: addons_enabled
 
   roles:
-    - role: config/cluster/import
-    - role: config/infra/import
-    - role: addons/helm
-    - role: addons/rook
-      when:
-        - config.addons.rook.enabled is defined
-        - config.addons.rook.enabled == true
+    - { role: config/cluster/import, when: addons_enabled }
+    - { role: config/infra/import, when: addons_enabled }
+    - { role: addons/helm, when: addons_enabled }
+    - { role: addons/rook, when: addons_enabled }
diff --git a/embed/ansible/kubitect/hosts-setup.yaml b/embed/ansible/kubitect/hosts-setup.yaml
diff --git a/embed/ansible/kubitect/requirements.txt b/embed/ansible/kubitect/requirements.txt
@@ -1,4 +1,3 @@
-ansible==8.5.0
 ansible-core==2.16.2
 jinja2==3.1.2
 netaddr==0.9.0
diff --git a/embed/ansible/kubitect/roles/hosts-setup/tasks/main.yaml b/embed/ansible/kubitect/roles/hosts-setup/tasks/main.yaml
diff --git a/embed/embed_test.go b/embed/embed_test.go
@@ -8,7 +8,7 @@ import (
 )
 
 func TestGetTemplate(t *testing.T) {
-	tpl, err := GetTemplate("etcd.yaml.tpl")
+	tpl, err := GetTemplate("k3s/inventory.yaml")
 	assert.NoError(t, err)
 	assert.NotNil(t, tpl)
 }

diff --git a/embed/templates/hosts.yaml.tpl b/embed/templates/hosts.yaml.tpl
diff --git a/embed/templates/k3s/inventory.yaml b/embed/templates/k3s/inventory.yaml
@@ -0,0 +1,70 @@
+{{- $cfgNodes := .Values.ConfigNodes -}}
+{{- $infNodes := .Values.InfraNodes -}}
+---
+all:
+	hosts:
+	{{- range $infNodes.LoadBalancer.Instances }}
+		{{- $i := $cfgNodes.LoadBalancer.Instances | select "Id" .Id | first }}
+		{{ .Name }}:
+			ansible_host: {{ .IP }}
+			priority: {{ $i.Priority }}
+	{{- end }}
+	{{- range $infNodes.Master.Instances }}
+		{{- $i := $cfgNodes.Master.Instances | select "Id" .Id | first }}
+		{{ .Name }}:
+			ansible_host: {{ .IP }}
+			server_config_yaml: |-
+				---
+				tls-san: {{ $infNodes.LoadBalancer.VIP }}
+				{{- if $i.Labels }}
+				node-label:
+					{{- range $k, $v := $i.Labels }}
+					- "{{ $k }}={{ $v }}"
+					{{- end }}
+				{{- end }}
+				{{- if $i.Taints }}
+				node-taint:
+					{{- range $i.Taints }}
+					- "{{ . }}"
+					{{- end }}
+				{{- end }}
+	{{- end }}
+	{{- range $infNodes.Worker.Instances }}
+		{{- $i := $cfgNodes.Worker.Instances | select "Id" .Id | first }}
+		{{ .Name }}:
+			ansible_host: {{ .IP }}
+			server_config_yaml: |-
+				---
+				{{- if $i.Labels }}
+				node-label:
+					{{- range $k, $v := $i.Labels }}
+					- "{{ $k }}={{ $v }}"
+					{{- end }}
+				{{- end }}
+				{{- if $i.Taints }}
+				node-taint:
+					{{- range $i.Taints }}
+					- "{{ . }}"
+					{{- end }}
+				{{- end }}
+	{{- end }}
+	children:
+		haproxy:
+			hosts:
+			{{- range $infNodes.LoadBalancer.Instances }}
+				{{ .Name }}:
+			{{- end }}
+		k3s_cluster:
+			children:
+				server:
+					hosts:
+					{{- range $infNodes.Master.Instances }}
+						{{ .Name }}:
+					{{- end }}
+				agent:
+					hosts:
+					{{- if $infNodes.Worker.Instances }}
+						{{- range $infNodes.Worker.Instances }}
+						{{ .Name }}:
+						{{- end }}
+					{{- end }}
diff --git a/embed/templates/k3s/inventory_partial.yaml b/embed/templates/k3s/inventory_partial.yaml
@@ -0,0 +1,20 @@
+{{- $nodes := .Values -}}
+---
+k3s_cluster:
+	children:
+		server:
+			hosts:
+			{{ range $name, $node := $nodes }}
+				{{- if eq $node.GetTypeName "master" }}
+				{{ $name }}:
+					ansible_host: {{ $node.IP }}
+				{{- end }}
+			{{- end }}
+		agent:
+			hosts:
+			{{ range $name, $node := $nodes }}
+				{{- if eq $node.GetTypeName "worker" }}
+				{{ $name }}:
+					ansible_host: {{ $node.IP }}
+				{{- end }}
+			{{- end }}
diff --git a/embed/templates/k8s-cluster.yaml.tpl b/embed/templates/k8s-cluster.yaml.tpl
diff --git a/embed/templates/all.yaml.tpl → embed/templates/kubespray/all.yaml b/embed/templates/all.yaml.tpl → embed/templates/kubespray/all.yaml
@@ -3,11 +3,11 @@
 # https://github.com/kubernetes-sigs/kubespray/blob/v2.17.1/inventory/sample/group_vars/all/all.yml
 ##
 ---
-apiserver_loadbalancer_domain_name: "{{ .InfraNodes.LoadBalancer.VIP }}"
+apiserver_loadbalancer_domain_name: "{{ .Values.LoadBalancer.VIP }}"
 deploy_container_engine: true
 etcd_kubeadm_enabled: false
 loadbalancer_apiserver:
-	address: "{{ .InfraNodes.LoadBalancer.VIP }}"
+	address: "{{ .Values.LoadBalancer.VIP }}"
 	port: 6443
 ## Upstream dns servers
 # upstream_dns_servers:

diff --git a/embed/templates/etcd.yaml.tpl → embed/templates/kubespray/etcd.yaml b/embed/templates/etcd.yaml.tpl → embed/templates/kubespray/etcd.yaml
diff --git a/embed/templates/nodes.yaml.tpl → embed/templates/kubespray/inventory.yaml b/embed/templates/nodes.yaml.tpl → embed/templates/kubespray/inventory.yaml
@@ -1,15 +1,16 @@
-{{- $cfgNodes := .ConfigNodes -}}
+{{- $cfgNodes := .Values.ConfigNodes -}}
+{{- $infNodes := .Values.InfraNodes -}}
 all:
 	hosts:
 	{{- /* Load balancers */ -}}
-	{{- range .InfraNodes.LoadBalancer.Instances }}
+	{{- range $infNodes.LoadBalancer.Instances }}
 		{{- $i := $cfgNodes.LoadBalancer.Instances | select "Id" .Id | first }}
 		{{ .Name }}:
 			ansible_host: {{ .IP }}
 			priority: {{ $i.Priority }}
 	{{- end }}
 	{{- /* Master nodes */ -}}
-	{{- range .InfraNodes.Master.Instances }}
+	{{- range $infNodes.Master.Instances }}
 		{{- $i := $cfgNodes.Master.Instances | select "Id" .Id | first }}
 		{{ .Name }}:
 			ansible_host: {{ .IP }}
@@ -27,7 +28,7 @@ all:
 			{{- end }}
 	{{- end }}
 	{{- /* Worker nodes */ -}}
-	{{- range .InfraNodes.Worker.Instances }}
+	{{- range $infNodes.Worker.Instances }}
 		{{- $i := $cfgNodes.Worker.Instances | select "Id" .Id | first }}
 		{{ .Name }}:
 			ansible_host: {{ .IP }}
@@ -47,30 +48,30 @@ all:
 	children:
 		haproxy:
 			hosts:
-			{{- range .InfraNodes.LoadBalancer.Instances }}
+			{{- range $infNodes.LoadBalancer.Instances }}
 				{{ .Name }}:
 			{{- end }}
 		etcd:
 			hosts:
-			{{- range .InfraNodes.Master.Instances }}
+			{{- range $infNodes.Master.Instances }}
 				{{ .Name }}:
 			{{- end }}
 		k8s_cluster:
 			children:
 				kube_control_plane:
 					hosts:
-					{{- range .InfraNodes.Master.Instances }}
+					{{- range $infNodes.Master.Instances }}
 						{{ .Name }}:
 					{{- end }}
 				kube_node:
 					hosts:
-					{{- if .InfraNodes.Worker.Instances }}
-						{{- range .InfraNodes.Worker.Instances }}
+					{{- if $infNodes.Worker.Instances }}
+						{{- range $infNodes.Worker.Instances }}
 						{{ .Name }}:
 						{{- end }}
 					{{- else }}
 						{{- /* No worker nodes -> masters also become workers */ -}}
-						{{- range .InfraNodes.Master.Instances }}
+						{{- range $infNodes.Master.Instances }}
 						{{ .Name }}:
 						{{- end }}
-					{{- end }}
+					{{- end }}
diff --git a/embed/templates/kubespray/k8s-cluster.yaml b/embed/templates/kubespray/k8s-cluster.yaml
@@ -0,0 +1,12 @@
+##
+# Kubesprays's source file (v2.17.1):
+# https://github.com/kubernetes-sigs/kubespray/blob/v2.17.1/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
+##
+---
+auto_renew_certificates: {{ .Values.Kubernetes.Other.AutoRenewCertificates }}
+cluster_name: cluster.local
+dns_mode: {{ .Values.Kubernetes.DnsMode }}
+kube_version: {{ .Values.Kubernetes.Version }}
+kube_network_plugin: {{ .Values.Kubernetes.NetworkPlugin }}
+kube_proxy_strict_arp: true
+resolvconf_mode: host_resolvconf