Fix concurrent VM creation due to busy device error

Add another lock in DeactivateSnapshot().
Move DeactivateSnapshot from pkg/dmlegacy/cleanup to pkg/dmlegacy to
avoid cyclic dependency.

cmd/ignite-spawn/ignite-spawn.go

@@ -10,7 +10,7 @@ import (
 	"github.com/weaveworks/ignite/pkg/apis/ignite/scheme"
 	"github.com/weaveworks/ignite/pkg/constants"
 	"github.com/weaveworks/ignite/pkg/container"
-	dmcleanup "github.com/weaveworks/ignite/pkg/dmlegacy/cleanup"
+	"github.com/weaveworks/ignite/pkg/dmlegacy"
 	"github.com/weaveworks/ignite/pkg/prometheus"
 	"github.com/weaveworks/ignite/pkg/util"
 	patchutil "github.com/weaveworks/libgitops/pkg/util/patch"
@@ -60,7 +60,7 @@ func StartVM(vm *api.VM) (err error) {
 	defer util.DeferErr(&err, func() error { return patchStopped(vm) })
 
 	// Remove the snapshot overlay post-run, which also removes the detached backing loop devices
-	defer util.DeferErr(&err, func() error { return dmcleanup.DeactivateSnapshot(vm) })
+	defer util.DeferErr(&err, func() error { return dmlegacy.DeactivateSnapshot(vm) })
 
 	// Remove the Prometheus socket post-run
 	defer util.DeferErr(&err, func() error { return os.Remove(metricsSocket) })

pkg/dmlegacy/snapshot.go

@@ -15,7 +15,7 @@ import (
 	"github.com/weaveworks/ignite/pkg/util"
 )
 
-const snapshotLockFileName = "ignite-snapshot.lock"
+const SnapshotLockFileName = "ignite-snapshot.lock"
 
 // ActivateSnapshot sets up the snapshot with devicemapper so that it is active and can be used.
 // It returns the path of the bootable snapshot device.
@@ -41,9 +41,13 @@ func ActivateSnapshot(vm *api.VM) (devicePath string, err error) {
 	// Serialize this operation by creating a global lock file when creating a
 	// loop device and release the lock after setting up device mapper using the
 	// loop device.
+	// Also, concurrent interactions with device mapper results in the error:
+	//   Device or resource busy
+	// Serializing the interaction with device mapper helps avoid issues due to
+	// race conditions.
 
 	// Global lock path.
-	glpath := filepath.Join(os.TempDir(), snapshotLockFileName)
+	glpath := filepath.Join(os.TempDir(), SnapshotLockFileName)
 
 	// Create a lockfile and obtain a lock.
 	lock, err := lockfile.New(glpath)

pkg/dmlegacy/vm.go

@@ -10,10 +10,10 @@ import (
 	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
+
 	api "github.com/weaveworks/ignite/pkg/apis/ignite"
 	meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/constants"
-	"github.com/weaveworks/ignite/pkg/dmlegacy/cleanup"
 	"github.com/weaveworks/ignite/pkg/operations/lookup"
 	"github.com/weaveworks/ignite/pkg/providers"
 	"github.com/weaveworks/ignite/pkg/util"
@@ -87,7 +87,7 @@ func copyToOverlay(vm *api.VM) (err error) {
 	if err != nil {
 		return
 	}
-	defer util.DeferErr(&err, func() error { return cleanup.DeactivateSnapshot(vm) })
+	defer util.DeferErr(&err, func() error { return DeactivateSnapshot(vm) })
 
 	mp, err := util.Mount(vm.SnapshotDev())
 	if err != nil {

pkg/operations/remove.go

@@ -8,7 +8,7 @@ import (
 	api "github.com/weaveworks/ignite/pkg/apis/ignite"
 	meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/client"
-	"github.com/weaveworks/ignite/pkg/dmlegacy/cleanup"
+	"github.com/weaveworks/ignite/pkg/dmlegacy"
 	"github.com/weaveworks/ignite/pkg/logs"
 	"github.com/weaveworks/ignite/pkg/providers"
 	"github.com/weaveworks/ignite/pkg/runtime"
@@ -50,7 +50,7 @@ func CleanupVM(vm *api.VM) error {
 	// After removing the VM container, if the Snapshot Device is still there, clean up
 	if _, err := os.Stat(vm.SnapshotDev()); err == nil {
 		// try remove it again with DeactivateSnapshot
-		if err := cleanup.DeactivateSnapshot(vm); err != nil {
+		if err := dmlegacy.DeactivateSnapshot(vm); err != nil {
 			return err
 		}
 	}