[Release-1.24] - Fix rootless node password location

[dereknola]

Backport fix for Fix rootless node password location
Original Issue: #3636

• Fix rootless node password location #7887

[est-suse]

Validated using commit id b016e3d on 1.24 branch

Docs:
https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cpu-cpuset-and-io-delegation
https://docs.k3s.io/advanced#advanced-rootless-configuration

Environment Details

k3s version v1.24.15+k3s-b016e3d5 (b016e3d5)
go version go1.19.10

Node(s) CPU architecture, OS, and version:

$ cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.1 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.1 LTS (Jammy Jellyfish)"

Cluster Configuration:

Single node

Details

$ curl https://get.k3s.io --output install.sh
$ sudo chmod +x install.sh
$ wget https://raw.githubusercontent.com/k3s-io/k3s/master/k3s-rootless.service
$ mkdir -p /home/ubuntu/.config/systemd/user/
$ cp k3s-rootless.service /home/ubuntu/.config/systemd/user/k3s-rootless.service
$ cat /home/ubuntu/.config/systemd/user/k3s-rootless.service
...
[Unit]
Description=k3s (Rootless)

[Service]
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
Environment=K3S_ROOTLESS_CIDR="10.41.0.0/16"
Environment=K3S_ROOTLESS_PORT_DRIVER=slirp4netns
Environment=K3S_ROOTLESS_DISABLE_HOST_LOOPBACK=true
Environment=K3S_ROOTLESS_MTU=1500
...


$ sudo cat /etc/sysctl.conf
net.ipv4.ip_forward=1
net.ipv6.conf.all.forwarding=1

$ sudo sysctl -w net.ipv4.ip_forward=1
$ sudo sysctl -w net.ipv6.conf.all.forwarding=1
$ sudo apt update
$ sudo apt install uidmap
$ sudo cat /etc/default/grub
---
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"
GRUB_CMDLINE_LINUX="systemd.unified_cgroup_hierarchy=1"
---

$ sudo update-grub
$ sudo mkdir -p /etc/systemd/system/user@.service.d
$ cat <<EOF | sudo tee /etc/systemd/system/user@.service.d/delegate.conf
[Service]
Delegate=cpu cpuset io memory pids
EOF

$ sudo systemctl daemon-reload
$ sudo INSTALL_K3S_COMMIT=b016e3d58be0a0358cad3a3f263d5853053754f8 INSTALL_K3S_SKIP_ENABLE=true ./install.sh
$ sudo reboot
$ grep cgroup /proc/mounts
cgroup2 /sys/fs/cgroup cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate,memory_recursiveprot 0 0

$ stat -c %T -f /sys/fs/cgroup
cgroup2fs

$ cat /sys/fs/cgroup/cgroup.controllers
cpuset cpu io memory hugetlb pids rdma misc

$ sudo sysctl -p
net.ipv4.ip_forward = 1
net.ipv6.conf.all.forwarding = 1

$ systemctl --user enable --now k3s-rootless
$ systemctl --user status k3s-rootless

$ systemctl --user status k3s-rootless
● k3s-rootless.service - k3s (Rootless)
     Loaded: loaded (/home/ubuntu/.config/systemd/user/k3s-rootless.service; enabled; vendor >
     Active: active (running) since Tue 2023-07-11 21:36:11 UTC; 16min ago
   Main PID: 711 (k3s-server)
      Tasks: 170
     Memory: 1.1G
        CPU: 1min 25.143s
     CGroup: /user.slice/user-1000.slice/user@1000.service/app.slice/k3s-rootless.service
             ├─k3s
             │ └─849 "k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "">
             ├─k3s_evac
             │ ├─ 711 "/usr/local/bin/k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ">
             │ ├─ 817 "/proc/self/exe init" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ">
             │ ├─ 827 slirp4netns --mtu 1500 -r 3 --disable-host-loopback --cidr 10.41.0.0/16>
             │ ├─ 831 "k3s server" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ">
             │ ├─ 994 "containerd " "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" >
             │ ├─1306 /home/ubuntu/.rancher/k3s/data/1c82efd5e692f5edba8e61ce8202e9bf51713025>
             │ ├─1324 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.tas>
             │ ├─1547 /home/ubuntu/.rancher/k3s/data/1c82efd5e692f5edba8e61ce8202e9bf51713025>
             │ ├─1564 /home/ubuntu/.rancher/k3s/data/1c82efd5e692f5edba8e61ce8202e9bf51713025>
             │ ├─1581 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.tas>
             │ ├─1588 fuse-overlayfs overlay /run/k3s/containerd/io.containerd.runtime.v2.tas>
             │ ├─1674 /home/ubuntu/.rancher/k3s/data/1c82efd5e692f5edba8e61ce8202e9bf51713025>
lines 1-23

kubectl get nodes

NAME               STATUS   ROLES                  AGE   VERSION
ip-172-31-21-229   Ready    control-plane,master   78s   v1.24.15+k3s-b016e3d5

$ kubectl get pods -A

kube-system   local-path-provisioner-597bc7dccd-q2hjr   1/1     Running     0          8m53s
kube-system   coredns-74448699cf-mcc8q                  1/1     Running     0          8m53s
kube-system   helm-install-traefik-crd-sd4ct            0/1     Completed   0          8m53s
kube-system   metrics-server-749b574fc-c8hj9            1/1     Running     0          8m53s
kube-system   svclb-traefik-3c5d4384-vwlkr              2/2     Running     0          8m21s
kube-system   helm-install-traefik-kcm8q                0/1     Completed   2          8m53s
kube-system   traefik-7467b667d9-shlhb                  1/1     Running     0          8m21s

$ journalctl --user -u k3s-rootless|grep "unable to read node password file"
$