Fix deletion of data and kube root dirs

Make sure that we don't have anything mounted under those directories so
we don't delete persistent data.

We do this py parsing /proc/mounts in reverse order as it is listed in
mount order, and then we unmount anything that is under our directories
before we delete them.

Don't umount datadir itself if it is on a separate partition/mount

Fixes #4318

Signed-off-by: Natanael Copa <ncopa@mirantis.com>
(cherry picked from commit 7ab81aa)

docs/reset.md

@@ -10,6 +10,9 @@ following:
 * Processes and containers: Terminates all running k0s processes to ensure that
   there are no active components left. This includes all container processes
   managed by the Container Runtime.
+* Mounts under k0s data directory: In order to prevent persistent data to be
+  deleted, all mount points under k0s' data directory will be unmounted. If an
+  unmount fails, it will be unmounted lazy.
 * Data stored on the node: Deletes the whole k0s data directory, which includes
   * all k0s-related configuration files, including those used for cluster setup
     and node-specific settings,
@@ -23,8 +26,8 @@ following:
   reboot the host after a reset to ensure that there are no k0s remnants in the
   host's network configuration.
 * Registration with the host's init system: Reverts the registration done by
-  `k0s install`. After a reset, k0s won't be automatically started when the host
-  boots.
+  `k0s install`. After a reset, k0s won't be automatically started when the
+  host boots.
 
 After a successful reset, the k0s binary itself remains. It can then be used to
 join another cluster or create a new one.

pkg/cleanup/directories.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 	"k8s.io/mount-utils"
@@ -46,15 +47,39 @@ func (d *directories) Run() error {
 
 	var dataDirMounted bool
 
-	// search and unmount kubelet volume mounts
-	for _, v := range procMounts {
-		if v.Path == filepath.Join(d.Config.dataDir, "kubelet") {
+	// ensure that we don't delete any persistent data volumes that may be
+	// mounted by kubernetes by unmount every mount point under DataDir.
+	//
+	// Unmount in the reverse order it was mounted so we handle recursive
+	// bind mounts and over mounts properly. If we for any reason are not
+	// able to unmount, fall back to lazy unmount and if that also fails
+	// bail out and don't delete anything.
+	//
+	// Note that if there are any shared bind mounts under k0s data
+	// directory, we may end up unmounting stuff outside the k0s DataDir.
+	// If someone has set a bind mount to be shared, we assume that is the
+	// desired behavior. See MS_SHARED and NOTES:
+	//  - https://man7.org/linux/man-pages/man2/mount.2.html
+	//  - https://man7.org/linux/man-pages/man2/umount.2.html#NOTES
+	for i := len(procMounts) - 1; i >= 0; i-- {
+		v := procMounts[i]
+		// avoid unmount datadir if its mounted on separate partition
+		// k0s didn't mount it so leave it alone
+		if v.Path == d.Config.k0sVars.DataDir {
+			dataDirMounted = true
+			continue
+		}
+		if isUnderPath(v.Path, filepath.Join(d.Config.dataDir, "kubelet")) || isUnderPath(v.Path, d.Config.k0sVars.DataDir) {
 			logrus.Debugf("%v is mounted! attempting to unmount...", v.Path)
 			if err = mounter.Unmount(v.Path); err != nil {
-				logrus.Warningf("failed to unmount %v", v.Path)
+				// if we fail to unmount, try lazy unmount so
+				// we don't end up deleting stuff that we
+				// shouldn't
+				logrus.Warningf("lazy unmounting %v", v.Path)
+				if err = UnmountLazy(v.Path); err != nil {
+					return fmt.Errorf("failed unmount %v", v.Path)
+				}
 			}
-		} else if v.Path == d.Config.dataDir {
-			dataDirMounted = true
 		}
 	}
 
@@ -81,7 +106,13 @@ func (d *directories) Run() error {
 	return nil
 }
 
-// this is for checking if the error retrned by os.RemoveAll is due to
+// test if the path is a directory equal to or under base
+func isUnderPath(path, base string) bool {
+	rel, err := filepath.Rel(base, path)
+	return err == nil && !strings.HasPrefix(rel, "..") && !filepath.IsAbs(rel)
+}
+
+// this is for checking if the error returned by os.RemoveAll is due to
 // it being a mount point. if it is, we can ignore the error. this way
 // we can't rely on os.RemoveAll instead of recursively deleting the
 // contents of the directory

pkg/cleanup/unmount_unix.go

@@ -0,0 +1,27 @@
+//go:build unix
+
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func UnmountLazy(path string) error {
+	return unix.Unmount(path, unix.MNT_DETACH)
+}

pkg/cleanup/unmount_windows.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cleanup
+
+import "fmt"
+
+func UnmountLazy(path string) error {
+	return fmt.Errorf("lazy unmount is not supported on Windows for path: %s", path)
+}