Remove dependency on udev -- create our own device tracking symlinks

This patch frees us from being dependent on dmsetup-specific behavior and interactions with udev.
Ignite can now operate in absence of a working udevd equivalent.
This makes ignite easier to run in docker containers and WSL2.

pkg/apis/ignite/helpers.go

@@ -4,7 +4,6 @@ import (
 	"path"
 
 	"github.com/weaveworks/ignite/pkg/constants"
-	"github.com/weaveworks/ignite/pkg/util"
 )
 
 // SetImage populates relevant fields to an Image on the VM object
@@ -19,9 +18,9 @@ func (vm *VM) SetKernel(kernel *Kernel) {
 	vm.Status.Kernel = kernel.Status.OCISource
 }
 
-// SnapshotDev returns the path where the (legacy) DM snapshot exists
-func (vm *VM) SnapshotDev() string {
-	return path.Join("/dev/mapper", util.NewPrefixer().Prefix(vm.GetUID()))
+// SnapshotDevLink returns the symlink to where the (legacy) DM snapshot exists
+func (vm *VM) SnapshotDevLink() string {
+	return path.Join(vm.ObjectPath(), "snapshot-dev")
 }
 
 // Running returns true if the VM is running, otherwise false

pkg/container/firecracker.go

@@ -21,7 +21,7 @@ import (
 
 // ExecuteFirecracker executes the firecracker process using the Go SDK
 func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) {
-	drivePath := vm.SnapshotDev()
+	drivePath := vm.SnapshotDevLink()
 
 	networkInterfaces := make([]firecracker.NetworkInterface, 0, len(dhcpIfaces))
 	for _, dhcpIface := range dhcpIfaces {

pkg/dmlegacy/cleanup/deactivate.go

@@ -1,6 +1,8 @@
 package cleanup
 
 import (
+	"os"
+
 	api "github.com/weaveworks/ignite/pkg/apis/ignite"
 	"github.com/weaveworks/ignite/pkg/util"
 )
@@ -9,7 +11,7 @@ import (
 func DeactivateSnapshot(vm *api.VM) error {
 	dmArgs := []string{
 		"remove",
-		vm.SnapshotDev(),
+		util.NewPrefixer().Prefix(vm.GetUID()),
 	}
 
 	// If the base device is visible in "dmsetup", we should remove it
@@ -21,5 +23,17 @@ func DeactivateSnapshot(vm *api.VM) error {
 	}
 
 	_, err := util.ExecuteCommand("dmsetup", dmArgs...)
-	return err
+	if err != nil {
+		return err
+	}
+
+	// VM's from previous versions of ignite may not have a snapshot-dev symlink
+	// Lstat it first before attempting to remove it
+	if _, err = os.Lstat(vm.SnapshotDevLink()); err == nil {
+		if err = os.Remove(vm.SnapshotDevLink()); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }

pkg/dmlegacy/loopdev.go

@@ -3,9 +3,11 @@ package dmlegacy
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path"
 	"strconv"
+	"strings"
 
 	losetup "github.com/freddierice/go-losetup"
 )
@@ -36,7 +38,11 @@ func (ld *loopDevice) Size512K() (uint64, error) {
 
 // dmsetup uses stdin to read multiline tables, this is a helper function for that
 func runDMSetup(name string, table []byte) error {
-	cmd := exec.Command("dmsetup", "create", name)
+	cmd := exec.Command(
+		"dmsetup", "create",
+		"--noudevsync", // we don't depend on udevd's /dev/mapper/* symlinks, we create our own
+		name,
+	)
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
 		return err
@@ -52,8 +58,45 @@ func runDMSetup(name string, table []byte) error {
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return fmt.Errorf("command %q exited with %q: %v", cmd.Args, out, err)
+		return fmt.Errorf("command %q exited with %q: %w", cmd.Args, out, err)
 	}
 
 	return nil
 }
+
+// GetBlkDevPath returns the device path for a named device without the use of udevd's symlinks.
+// This is useful for creating our own symlinks to track devices and pass to the sandbox container.
+// The device path could be `/dev/<blkdevname>` (ex: `/dev/dm-0`)
+// or `/dev/mapper/<name>` (ex: `/dev/mapper/ignite-47a6421c19b415ef`)
+// depending on `dmsetup`'s udev-fallback related environment-variables and build-flags.
+func GetBlkDevPath(name string) (string, error) {
+	cmd := exec.Command(
+		"dmsetup", "info",
+		"--noudevsync", // we don't depend on udevd's /dev/mapper/* symlinks, we create our own
+		"--columns", "--noheadings", "-o", "blkdevname",
+		name,
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("command %q exited with %q: %w", cmd.Args, string(out), err)
+	}
+	blkdevname := strings.TrimSpace(string(out))
+
+	// if dmsetup is compiled without udev-sync or the DM_DISABLE_UDEV env var is set,
+	// `dmsetup info` will not return the correct blkdevname ("mapper/<name>") -- it still
+	// returns "dm-<minor>" even though the path doesn't exist.
+	// To work around this, we stat the returned blkdevname and try the fallback if it doesn't exist:
+	blkDevPath := path.Join("/dev", blkdevname)
+	if _, blkErr := os.Stat(blkDevPath); blkErr == nil {
+		return blkDevPath, nil
+	} else if !os.IsNotExist(blkErr) {
+		return "", blkErr
+	}
+
+	fallbackDevPath := path.Join("/dev/mapper", name)
+	if _, fallbackErr := os.Stat(fallbackDevPath); fallbackErr == nil {
+		return fallbackDevPath, nil
+	}
+
+	return "", fmt.Errorf("Could not stat a valid block device path for %q or %q", blkDevPath, fallbackDevPath)
+}

pkg/dmlegacy/snapshot.go

@@ -17,13 +17,15 @@ import (
 
 const snapshotLockFileName = "ignite-snapshot.lock"
 
-// ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used
-func ActivateSnapshot(vm *api.VM) (err error) {
+// ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used.
+// A symlink for the device is created in the VM ObjectPath data directory.
+// It returns the non-symlinked path of the bootable snapshot device.
+func ActivateSnapshot(vm *api.VM) (devPath string, err error) {
 	device := util.NewPrefixer().Prefix(vm.GetUID())
-	devicePath := vm.SnapshotDev()
+	deviceSymlink := vm.SnapshotDevLink()
 
 	// Return if the snapshot is already setup
-	if util.FileExists(devicePath) {
+	if util.FileExists(deviceSymlink) {
 		return
 	}
 
@@ -47,7 +49,8 @@ func ActivateSnapshot(vm *api.VM) (err error) {
 	// Create a lockfile and obtain a lock.
 	lock, err := lockfile.New(glpath)
 	if err != nil {
-		return fmt.Errorf("failed to create lock: %v", err)
+		err = fmt.Errorf("failed to create lockfile: %w", err)
+		return
 	}
 	if err = obtainLock(lock); err != nil {
 		return
@@ -99,23 +102,35 @@ func ActivateSnapshot(vm *api.VM) (err error) {
 			return
 		}
 
-		basePath = fmt.Sprintf("/dev/mapper/%s", baseDevice)
+		if basePath, err = GetBlkDevPath(baseDevice); err != nil {
+			return
+		}
 	}
 
-	// "0 8388608 snapshot /dev/{loop0,mapper/ignite-<uid>-base} /dev/loop1 P 8"
+	// "0 8388608 snapshot /dev/{loop0,dm-1,mapper/ignite-<uid>-base} /dev/loop1 P 8"
 	dmTable := []byte(fmt.Sprintf("0 %d snapshot %s %s P 8", overlayLoopSize, basePath, overlayLoop.Path()))
 
+	// setup the main boot device
 	if err = runDMSetup(device, dmTable); err != nil {
 		return
 	}
 
+	// get the boot device's actual path and create a well named symlink in the VM object dir
+	devPath, err = GetBlkDevPath(device)
+	if err != nil {
+		return
+	}
+	if err = os.Symlink(devPath, deviceSymlink); err != nil {
+		return
+	}
+
 	// Repair the filesystem in case it has errors
 	// e2fsck throws an error if the filesystem gets repaired, so just ignore it
-	_, _ = util.ExecuteCommand("e2fsck", "-p", "-f", devicePath)
+	_, _ = util.ExecuteCommand("e2fsck", "-p", "-f", deviceSymlink)
 
 	// If the overlay is larger than the image, call resize2fs to make the filesystem fill the overlay
 	if overlayLoopSize > imageLoopSize {
-		if _, err = util.ExecuteCommand("resize2fs", devicePath); err != nil {
+		if _, err = util.ExecuteCommand("resize2fs", deviceSymlink); err != nil {
 			return
 		}
 	}

pkg/dmlegacy/vm.go

@@ -83,13 +83,13 @@ func AllocateAndPopulateOverlay(vm *api.VM) error {
 }
 
 func copyToOverlay(vm *api.VM) (err error) {
-	err = ActivateSnapshot(vm)
+	_, err = ActivateSnapshot(vm)
 	if err != nil {
 		return
 	}
 	defer util.DeferErr(&err, func() error { return cleanup.DeactivateSnapshot(vm) })
 
-	mp, err := util.Mount(vm.SnapshotDev())
+	mp, err := util.Mount(vm.SnapshotDevLink())
 	if err != nil {
 		return
 	}

pkg/operations/remove.go

@@ -48,8 +48,8 @@ func CleanupVM(vm *api.VM) error {
 	// TODO should this function return a proper error?
 	RemoveVMContainer(inspectResult)
 
-	// After remove the VM container, and the SnapshotDev still there
-	if _, err := os.Stat(vm.SnapshotDev()); err == nil {
+	// After remove the VM container, and the linked Snapshot Device is still there
+	if _, err := os.Stat(vm.SnapshotDevLink()); err == nil {
 		// try remove it again with DeactivateSnapshot
 		if err := cleanup.DeactivateSnapshot(vm); err != nil {
 			return err

pkg/operations/start.go

@@ -25,7 +25,8 @@ func StartVM(vm *api.VM, debug bool) error {
 	RemoveVMContainer(inspectResult)
 
 	// Setup the snapshot overlay filesystem
-	if err := dmlegacy.ActivateSnapshot(vm); err != nil {
+	snapshotDevPath, err := dmlegacy.ActivateSnapshot(vm)
+	if err != nil {
 		return err
 	}
 
@@ -70,7 +71,7 @@ func StartVM(vm *api.VM, debug bool) error {
 			runtime.BindBoth("/dev/mapper/control"), // This enables containerized Ignite to remove its own dm snapshot
 			runtime.BindBoth("/dev/net/tun"),        // Needed for creating TAP adapters
 			runtime.BindBoth("/dev/kvm"),            // Pass through virtualization support
-			runtime.BindBoth(vm.SnapshotDev()),      // The block device to boot from
+			runtime.BindBoth(snapshotDevPath),       // The non-symlinked path for the block device to boot from
 		},
 		StopTimeout:  constants.STOP_TIMEOUT + constants.IGNITE_TIMEOUT,
 		PortBindings: vm.Spec.Network.Ports, // Add the port mappings to Docker