feat: rollback to k3s

.gitignore

@@ -14,6 +14,7 @@ talosconfig
 .bin
 # Ansible
 .venv*
+authorized_keys
 # Taskfile
 .task
 # Brew

.sops.yaml

@@ -1,12 +1,11 @@
 ---
 creation_rules:
-  - # IMPORTANT: This rule MUST be above the others
-    path_regex: talos/.*\.sops\.ya?ml
+  - path_regex: kubernetes/.*\.sops\.ya?ml
+    encrypted_regex: "^(data|stringData)$"
     key_groups:
       - age:
           - "age1ptththqpxnx0zuzmq0peq9x30vqgdedjsdlsuzxr5gfc36mnwqlsylrpr8"
-  - path_regex: kubernetes/.*\.sops\.ya?ml
-    encrypted_regex: "^(data|stringData)$"
+  - path_regex: ansible/.*\.sops\.ya?ml
     key_groups:
       - age:
           - "age1ptththqpxnx0zuzmq0peq9x30vqgdedjsdlsuzxr5gfc36mnwqlsylrpr8"

.taskfiles/Talos/Taskfile.yaml

@@ -3,7 +3,7 @@
 version: "3"
 
 vars:
-  TALOS_DIR: "{{.KUBERNETES_DIR}}/main/bootstrap/talos"
+  TALOS_DIR: "{{.KUBERNETES_DIR}}/bootstrap/talos"
   TALHELPER_SECRET_FILE: "{{.TALOS_DIR}}/talsecret.sops.yaml"
   TALHELPER_CONFIG_FILE: "{{.TALOS_DIR}}/talconfig.yaml"
 

ansible/.ansible-lint

@@ -0,0 +1,9 @@
+skip_list:
+  - yaml[commas]
+  - yaml[line-length]
+  - var-naming
+warn_list:
+  - command-instead-of-shell
+  - deprecated-command-syntax
+  - experimental
+  - no-changed-when

ansible/inventory/group_vars/controllers/main.yaml

@@ -0,0 +1,24 @@
+---
+k3s_control_node: true
+k3s_server:
+  cluster-cidr: "172.16.0.0/16"
+  service-cidr: "10.96.0.0/16"
+  disable: ["flannel", "local-storage", "metrics-server", "servicelb", "traefik"]
+  disable-cloud-controller: true
+  disable-kube-proxy: true
+  disable-network-policy: true
+  docker: false
+  embedded-registry: true
+  etcd-expose-metrics: true
+  flannel-backend: "none"
+  kube-apiserver-arg:
+    - "anonymous-auth=true"
+  kube-controller-manager-arg:
+    - "bind-address=0.0.0.0"
+  kube-scheduler-arg:
+    - "bind-address=0.0.0.0"
+  node-ip: "{{ ansible_host }}"
+  secrets-encryption: true
+  tls-san:
+    - "10.69.1.154"
+  write-kubeconfig-mode: "644"

ansible/inventory/group_vars/kubernetes/main.yaml

@@ -0,0 +1,23 @@
+---
+k3s_become: true
+k3s_etcd_datastore: true
+k3s_install_hard_links: true
+k3s_registration_address: "10.69.1.154"
+k3s_registries:
+  mirrors:
+    docker.io:
+    gcr.io:
+    ghcr.io:
+    k8s.gcr.io:
+    lscr.io:
+    mcr.microsoft.com:
+    public.ecr.aws:
+    quay.io:
+    registry.k8s.io:
+# renovate: datasource=github-releases depName=k3s-io/k3s
+k3s_release_version: v1.30.0+k3s1
+k3s_server_manifests_templates:
+  - custom-cilium-helmchart.yaml
+  - custom-kube-vip-ds.yaml
+  - custom-kube-vip-rbac.yaml
+k3s_use_unsupported_config: true

ansible/inventory/group_vars/workers/main.yaml

@@ -0,0 +1,4 @@
+---
+k3s_control_node: false
+k3s_agent:
+  node-ip: "{{ ansible_host }}"

ansible/inventory/hosts.yaml

@@ -0,0 +1,23 @@
+---
+kubernetes:
+  children:
+    controllers:
+      hosts:
+        "rkm1":
+          ansible_user: "oscar"
+          ansible_host: "10.69.1.30"
+          ansible_ssh_private_key_file: "./authorized_keys"
+    workers:
+      hosts:
+        "rkw1":
+          ansible_user: "oscar"
+          ansible_host: "10.69.1.31"
+          ansible_ssh_private_key_file: "./authorized_keys"
+        "rkw2":
+          ansible_user: "oscar"
+          ansible_host: "10.69.1.32"
+          ansible_ssh_private_key_file: "./authorized_keys"
+        "rkw3":
+          ansible_user: "oscar"
+          ansible_host: "10.69.1.33"
+          ansible_ssh_private_key_file: "./authorized_keys"

ansible/playbooks/cluster-installation.yaml

@@ -0,0 +1,91 @@
+---
+- name: Cluster Installation
+  hosts: kubernetes
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  pre_tasks:
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Check if cluster is installed
+      check_mode: false
+      ansible.builtin.stat:
+        path: /etc/rancher/k3s/config.yaml
+      register: k3s_installed
+
+    - name: Ignore manifests templates and urls if the cluster is already installed
+      when: k3s_installed.stat.exists
+      ansible.builtin.set_fact:
+        k3s_server_manifests_templates: []
+        k3s_server_manifests_urls: []
+
+    - name: Prevent downgrades
+      when: k3s_installed.stat.exists
+      ansible.builtin.include_tasks: tasks/version-check.yaml
+
+    - name: Ensure that the /opt/cni directory exists
+      ansible.builtin.file:
+        path: /opt/cni
+        mode: '755'
+        state: directory
+    - name: Ensure that the /opt/cni/bin is a link to /var/lib/rancher/k3s/data/current/bin
+      ansible.builtin.file:
+        src: /var/lib/rancher/k3s/data/current/bin
+        dest: /opt/cni/bin
+        follow: false
+        force: true
+        state: link
+
+    - name: Ensure that the /etc/cni directory exists
+      ansible.builtin.file:
+        path: /etc/cni
+        mode: '755'
+        state: directory
+    - name: Ensure that the /var/lib/rancher/k3s/agent/etc/cni/net.d directory exists
+      ansible.builtin.file:
+        path: /var/lib/rancher/k3s/agent/etc/cni/net.d
+        mode: '755'
+        state: directory
+    - name: Ensure that the /etc/cni/net.d is a link to /var/lib/rancher/k3s/agent/etc/cni/net.d
+      ansible.builtin.file:
+        src: /var/lib/rancher/k3s/agent/etc/cni/net.d
+        dest: /etc/cni/net.d
+        force: true
+        state: link
+
+    - name: Install Kubernetes
+      ansible.builtin.include_role:
+        name: xanmanning.k3s
+        public: true
+      vars:
+        k3s_state: installed
+
+    - name: Kubeconfig
+      ansible.builtin.include_tasks: tasks/kubeconfig.yaml
+
+    - name: Wait for custom manifests to rollout
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{{ item.kind }}"
+        name: "{{ item.name }}"
+        namespace: "{{ item.namespace | default('') }}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
+        - { name: kube-vip, kind: DaemonSet, namespace: kube-system }
+
+    - name: Cilium
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cilium.yaml
+
+    - name: Cruft
+      when: k3s_primary_control_node
+      ansible.builtin.include_tasks: tasks/cruft.yaml

ansible/playbooks/cluster-nuke.yaml

@@ -0,0 +1,105 @@
+---
+- name: Cluster Nuke
+  hosts: kubernetes
+  become: true
+  gather_facts: true
+  any_errors_fatal: true
+  vars_prompt:
+    - name: nuke
+      prompt: |-
+        Are you sure you want to nuke this cluster?
+        Type 'YES I WANT TO DESTROY THIS CLUSTER' to proceed
+      default: "n"
+      private: false
+  pre_tasks:
+    - name: Check for confirmation
+      ansible.builtin.fail:
+        msg: Aborted nuking the cluster
+      when: nuke != 'YES I WANT TO DESTROY THIS CLUSTER'
+
+    - name: Pausing for 5 seconds...
+      ansible.builtin.pause:
+        seconds: 5
+  tasks:
+    - name: Stop Kubernetes # noqa: ignore-errors
+      ignore_errors: true
+      block:
+        - name: Stop Kubernetes
+          ansible.builtin.include_role:
+            name: xanmanning.k3s
+            public: true
+          vars:
+            k3s_state: stopped
+
+    # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
+    - name: Networking
+      block:
+        - name: Networking | Delete Cilium links
+          ansible.builtin.command:
+            cmd: "ip link delete {{ item }}"
+            removes: "/sys/class/net/{{ item }}"
+          loop: ["cilium_host", "cilium_net", "cilium_vxlan"]
+        - name: Networking | Flush iptables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Flush ip6tables
+          ansible.builtin.iptables:
+            table: "{{ item }}"
+            flush: true
+            ip_version: ipv6
+          loop: ["filter", "nat", "mangle", "raw"]
+        - name: Networking | Delete CNI bin link
+          ansible.builtin.file:
+            path: /opt/cni/bin
+            state: absent
+        - name: Networking | Delete CNI conf link
+          ansible.builtin.file:
+            path: /etc/cni/net.d
+            state: absent
+
+    - name: Check to see if k3s-killall.sh exits
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s-killall.sh
+      register: check_k3s_killall_script
+
+    - name: Check to see if k3s-uninstall.sh exits
+      ansible.builtin.stat:
+        path: /usr/local/bin/k3s-uninstall.sh
+      register: check_k3s_uninstall_script
+
+    - name: Run k3s-killall.sh
+      when: check_k3s_killall_script.stat.exists
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-killall.sh
+      register: k3s_killall
+      changed_when: k3s_killall.rc == 0
+
+    - name: Run k3s-uninstall.sh
+      when: check_k3s_uninstall_script.stat.exists
+      ansible.builtin.command:
+        cmd: /usr/local/bin/k3s-uninstall.sh
+      args:
+        removes: /usr/local/bin/k3s-uninstall.sh
+      register: k3s_uninstall
+      changed_when: k3s_uninstall.rc == 0
+
+    - name: Ensure hard links are removed
+      when:
+        - k3s_install_hard_links
+        - not ansible_check_mode
+      ansible.builtin.file:
+        path: "{{ k3s_install_dir }}/{{ item }}"
+        state: absent
+      loop: ["kubectl", "crictl", "ctr"]
+
+    - name: Remove local storage path
+      ansible.builtin.file:
+        path: /var/openebs/local
+        state: absent
+
+    - name: Reboot
+      ansible.builtin.reboot:
+        msg: Rebooting hosts
+        reboot_timeout: 3600