From e550f32b3ae35ed6f79b8f9f17b2418c2cea5873 Mon Sep 17 00:00:00 2001 From: leigh capili Date: Wed, 29 Jul 2020 00:50:17 -0600 Subject: [PATCH] Remove dependency on udev -- create our own device tracking symlinks This patch frees us from being dependent on dmsetup-specific behavior and interactions with udev. Ignite can now operate in absence of a working udevd equivalent. This makes ignite easier to run in docker containers and WSL2. --- pkg/apis/ignite/helpers.go | 7 ++--- pkg/container/firecracker.go | 2 +- pkg/dmlegacy/cleanup/deactivate.go | 18 ++++++++++-- pkg/dmlegacy/loopdev.go | 47 ++++++++++++++++++++++++++++-- pkg/dmlegacy/snapshot.go | 33 +++++++++++++++------ pkg/dmlegacy/vm.go | 4 +-- pkg/operations/remove.go | 4 +-- pkg/operations/start.go | 5 ++-- 8 files changed, 96 insertions(+), 24 deletions(-) diff --git a/pkg/apis/ignite/helpers.go b/pkg/apis/ignite/helpers.go index 577288d37..edbc206ec 100644 --- a/pkg/apis/ignite/helpers.go +++ b/pkg/apis/ignite/helpers.go @@ -4,7 +4,6 @@ import ( "path" "github.com/weaveworks/ignite/pkg/constants" - "github.com/weaveworks/ignite/pkg/util" ) // SetImage populates relevant fields to an Image on the VM object @@ -19,9 +18,9 @@ func (vm *VM) SetKernel(kernel *Kernel) { vm.Status.Kernel = kernel.Status.OCISource } -// SnapshotDev returns the path where the (legacy) DM snapshot exists -func (vm *VM) SnapshotDev() string { - return path.Join("/dev/mapper", util.NewPrefixer().Prefix(vm.GetUID())) +// SnapshotDevLink returns the symlink to where the (legacy) DM snapshot exists +func (vm *VM) SnapshotDevLink() string { + return path.Join(vm.ObjectPath(), "snapshot-dev") } // Running returns true if the VM is running, otherwise false diff --git a/pkg/container/firecracker.go b/pkg/container/firecracker.go index 4f61ecd19..cef527873 100644 --- a/pkg/container/firecracker.go +++ b/pkg/container/firecracker.go @@ -21,7 +21,7 @@ import ( // ExecuteFirecracker executes the firecracker process using the Go SDK func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) { - drivePath := vm.SnapshotDev() + drivePath := vm.SnapshotDevLink() networkInterfaces := make([]firecracker.NetworkInterface, 0, len(dhcpIfaces)) for _, dhcpIface := range dhcpIfaces { diff --git a/pkg/dmlegacy/cleanup/deactivate.go b/pkg/dmlegacy/cleanup/deactivate.go index 54fd828ae..a9e33a7b2 100644 --- a/pkg/dmlegacy/cleanup/deactivate.go +++ b/pkg/dmlegacy/cleanup/deactivate.go @@ -1,6 +1,8 @@ package cleanup import ( + "os" + api "github.com/weaveworks/ignite/pkg/apis/ignite" "github.com/weaveworks/ignite/pkg/util" ) @@ -9,7 +11,7 @@ import ( func DeactivateSnapshot(vm *api.VM) error { dmArgs := []string{ "remove", - vm.SnapshotDev(), + util.NewPrefixer().Prefix(vm.GetUID()), } // If the base device is visible in "dmsetup", we should remove it @@ -21,5 +23,17 @@ func DeactivateSnapshot(vm *api.VM) error { } _, err := util.ExecuteCommand("dmsetup", dmArgs...) - return err + if err != nil { + return err + } + + // VM's from previous versions of ignite may not have a snapshot-dev symlink + // Lstat it first before attempting to remove it + if _, err = os.Lstat(vm.SnapshotDevLink()); err == nil { + if err = os.Remove(vm.SnapshotDevLink()); err != nil { + return err + } + } + + return nil } diff --git a/pkg/dmlegacy/loopdev.go b/pkg/dmlegacy/loopdev.go index 59f80e458..30e6b46aa 100644 --- a/pkg/dmlegacy/loopdev.go +++ b/pkg/dmlegacy/loopdev.go @@ -3,9 +3,11 @@ package dmlegacy import ( "fmt" "io/ioutil" + "os" "os/exec" "path" "strconv" + "strings" losetup "github.com/freddierice/go-losetup" ) @@ -36,7 +38,11 @@ func (ld *loopDevice) Size512K() (uint64, error) { // dmsetup uses stdin to read multiline tables, this is a helper function for that func runDMSetup(name string, table []byte) error { - cmd := exec.Command("dmsetup", "create", name) + cmd := exec.Command( + "dmsetup", "create", + "--noudevsync", // we don't depend on udevd's /dev/mapper/* symlinks, we create our own + name, + ) stdin, err := cmd.StdinPipe() if err != nil { return err @@ -52,8 +58,45 @@ func runDMSetup(name string, table []byte) error { out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("command %q exited with %q: %v", cmd.Args, out, err) + return fmt.Errorf("command %q exited with %q: %w", cmd.Args, out, err) } return nil } + +// GetBlkDevPath returns the device path for a named device without the use of udevd's symlinks. +// This is useful for creating our own symlinks to track devices and pass to the sandbox container. +// The device path could be `/dev/` (ex: `/dev/dm-0`) +// or `/dev/mapper/` (ex: `/dev/mapper/ignite-47a6421c19b415ef`) +// depending on `dmsetup`'s udev-fallback related environment-variables and build-flags. +func GetBlkDevPath(name string) (string, error) { + cmd := exec.Command( + "dmsetup", "info", + "--noudevsync", // we don't depend on udevd's /dev/mapper/* symlinks, we create our own + "--columns", "--noheadings", "-o", "blkdevname", + name, + ) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("command %q exited with %q: %w", cmd.Args, string(out), err) + } + blkdevname := strings.TrimSpace(string(out)) + + // if dmsetup is compiled without udev-sync or the DM_DISABLE_UDEV env var is set, + // `dmsetup info` will not return the correct blkdevname ("mapper/") -- it still + // returns "dm-" even though the path doesn't exist. + // To work around this, we stat the returned blkdevname and try the fallback if it doesn't exist: + blkDevPath := path.Join("/dev", blkdevname) + if _, blkErr := os.Stat(blkDevPath); blkErr == nil { + return blkDevPath, nil + } else if !os.IsNotExist(blkErr) { + return "", blkErr + } + + fallbackDevPath := path.Join("/dev/mapper", name) + if _, fallbackErr := os.Stat(fallbackDevPath); fallbackErr == nil { + return fallbackDevPath, nil + } + + return "", fmt.Errorf("Could not stat a valid block device path for %q or %q", blkDevPath, fallbackDevPath) +} diff --git a/pkg/dmlegacy/snapshot.go b/pkg/dmlegacy/snapshot.go index 86c39fa52..f3f58d4a0 100644 --- a/pkg/dmlegacy/snapshot.go +++ b/pkg/dmlegacy/snapshot.go @@ -17,13 +17,15 @@ import ( const snapshotLockFileName = "ignite-snapshot.lock" -// ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used -func ActivateSnapshot(vm *api.VM) (err error) { +// ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used. +// A symlink for the device is created in the VM ObjectPath data directory. +// It returns the non-symlinked path of the bootable snapshot device. +func ActivateSnapshot(vm *api.VM) (devPath string, err error) { device := util.NewPrefixer().Prefix(vm.GetUID()) - devicePath := vm.SnapshotDev() + deviceSymlink := vm.SnapshotDevLink() // Return if the snapshot is already setup - if util.FileExists(devicePath) { + if util.FileExists(deviceSymlink) { return } @@ -47,7 +49,8 @@ func ActivateSnapshot(vm *api.VM) (err error) { // Create a lockfile and obtain a lock. lock, err := lockfile.New(glpath) if err != nil { - return fmt.Errorf("failed to create lock: %v", err) + err = fmt.Errorf("failed to create lockfile: %w", err) + return } if err = obtainLock(lock); err != nil { return @@ -99,23 +102,35 @@ func ActivateSnapshot(vm *api.VM) (err error) { return } - basePath = fmt.Sprintf("/dev/mapper/%s", baseDevice) + if basePath, err = GetBlkDevPath(baseDevice); err != nil { + return + } } - // "0 8388608 snapshot /dev/{loop0,mapper/ignite--base} /dev/loop1 P 8" + // "0 8388608 snapshot /dev/{loop0,dm-1,mapper/ignite--base} /dev/loop1 P 8" dmTable := []byte(fmt.Sprintf("0 %d snapshot %s %s P 8", overlayLoopSize, basePath, overlayLoop.Path())) + // setup the main boot device if err = runDMSetup(device, dmTable); err != nil { return } + // get the boot device's actual path and create a well named symlink in the VM object dir + devPath, err = GetBlkDevPath(device) + if err != nil { + return + } + if err = os.Symlink(devPath, deviceSymlink); err != nil { + return + } + // Repair the filesystem in case it has errors // e2fsck throws an error if the filesystem gets repaired, so just ignore it - _, _ = util.ExecuteCommand("e2fsck", "-p", "-f", devicePath) + _, _ = util.ExecuteCommand("e2fsck", "-p", "-f", deviceSymlink) // If the overlay is larger than the image, call resize2fs to make the filesystem fill the overlay if overlayLoopSize > imageLoopSize { - if _, err = util.ExecuteCommand("resize2fs", devicePath); err != nil { + if _, err = util.ExecuteCommand("resize2fs", deviceSymlink); err != nil { return } } diff --git a/pkg/dmlegacy/vm.go b/pkg/dmlegacy/vm.go index b4eb368cb..2db1ff474 100644 --- a/pkg/dmlegacy/vm.go +++ b/pkg/dmlegacy/vm.go @@ -83,13 +83,13 @@ func AllocateAndPopulateOverlay(vm *api.VM) error { } func copyToOverlay(vm *api.VM) (err error) { - err = ActivateSnapshot(vm) + _, err = ActivateSnapshot(vm) if err != nil { return } defer util.DeferErr(&err, func() error { return cleanup.DeactivateSnapshot(vm) }) - mp, err := util.Mount(vm.SnapshotDev()) + mp, err := util.Mount(vm.SnapshotDevLink()) if err != nil { return } diff --git a/pkg/operations/remove.go b/pkg/operations/remove.go index ba0d94abe..bd4d61cbb 100644 --- a/pkg/operations/remove.go +++ b/pkg/operations/remove.go @@ -48,8 +48,8 @@ func CleanupVM(vm *api.VM) error { // TODO should this function return a proper error? RemoveVMContainer(inspectResult) - // After remove the VM container, and the SnapshotDev still there - if _, err := os.Stat(vm.SnapshotDev()); err == nil { + // After remove the VM container, and the linked Snapshot Device is still there + if _, err := os.Stat(vm.SnapshotDevLink()); err == nil { // try remove it again with DeactivateSnapshot if err := cleanup.DeactivateSnapshot(vm); err != nil { return err diff --git a/pkg/operations/start.go b/pkg/operations/start.go index 21edde5af..307d9e07d 100644 --- a/pkg/operations/start.go +++ b/pkg/operations/start.go @@ -25,7 +25,8 @@ func StartVM(vm *api.VM, debug bool) error { RemoveVMContainer(inspectResult) // Setup the snapshot overlay filesystem - if err := dmlegacy.ActivateSnapshot(vm); err != nil { + snapshotDevPath, err := dmlegacy.ActivateSnapshot(vm) + if err != nil { return err } @@ -70,7 +71,7 @@ func StartVM(vm *api.VM, debug bool) error { runtime.BindBoth("/dev/mapper/control"), // This enables containerized Ignite to remove its own dm snapshot runtime.BindBoth("/dev/net/tun"), // Needed for creating TAP adapters runtime.BindBoth("/dev/kvm"), // Pass through virtualization support - runtime.BindBoth(vm.SnapshotDev()), // The block device to boot from + runtime.BindBoth(snapshotDevPath), // The non-symlinked path for the block device to boot from }, StopTimeout: constants.STOP_TIMEOUT + constants.IGNITE_TIMEOUT, PortBindings: vm.Spec.Network.Ports, // Add the port mappings to Docker