[cinder-csi-plugin] retry mount operation with rescan (kubernetes#2610)

If the initial formatting and mounting fails in
NodeStageVolume try to rescan the device and retry the
operation.

This prevents failures if the device information is
reported wrongly which would otherwise be blocking
the mounting.

Signed-off-by: NymanRobin <robin.nyman@est.tech>

pkg/csi/cinder/nodeserver.go

@@ -398,7 +398,7 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 			options = append(options, collectMountOptions(fsType, mountFlags)...)
 		}
 		// Mount
-		err = m.Mounter().FormatAndMount(devicePath, stagingTarget, fsType, options)
+		err = ns.formatAndMountRetry(devicePath, stagingTarget, fsType, options)
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
@@ -426,6 +426,25 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 	return &csi.NodeStageVolumeResponse{}, nil
 }
 
+// formatAndMountRetry attempts to format and mount a device at the given path.
+// If the initial mount fails, it rescans the device and retries the mount operation.
+func (ns *nodeServer) formatAndMountRetry(devicePath, stagingTarget, fsType string, options []string) error {
+	m := ns.Mount
+	err := m.Mounter().FormatAndMount(devicePath, stagingTarget, fsType, options)
+	if err != nil {
+		klog.Infof("Initial format and mount failed: %v. Attempting rescan.", err)
+		// Attempting rescan if the initial mount fails
+		rescanErr := blockdevice.RescanDevice(devicePath)
+		if rescanErr != nil {
+			klog.Infof("Rescan failed: %v. Returning original mount error.", rescanErr)
+			return err
+		}
+		klog.Infof("Rescan succeeded, retrying format and mount")
+		err = m.Mounter().FormatAndMount(devicePath, stagingTarget, fsType, options)
+	}
+	return err
+}
+
 func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
 	klog.V(4).Infof("NodeUnstageVolume: called with args %+v", protosanitizer.StripSecrets(*req))
 

pkg/util/blockdevice/blockdevice_linux.go

@@ -85,6 +85,16 @@ func checkBlockDeviceSize(devicePath string, deviceMountPath string, newSize int
 	return nil
 }
 
+func triggerRescan(blockDeviceRescanPath string) error {
+	klog.V(4).Infof("Rescanning %q block device geometry", blockDeviceRescanPath)
+	err := os.WriteFile(blockDeviceRescanPath, []byte{'1'}, 0666)
+	if err != nil {
+		klog.Errorf("Error rescanning new block device geometry: %v", err)
+		return err
+	}
+	return nil
+}
+
 func RescanBlockDeviceGeometry(devicePath string, deviceMountPath string, newSize int64) error {
 	if newSize == 0 {
 		klog.Error("newSize is empty, skipping the block device rescan")
@@ -106,13 +116,25 @@ func RescanBlockDeviceGeometry(devicePath string, deviceMountPath string, newSiz
 	}
 
 	klog.V(3).Infof("Resolved block device path from %q to %q", devicePath, blockDeviceRescanPath)
-	klog.V(4).Infof("Rescanning %q block device geometry", devicePath)
-	err = os.WriteFile(blockDeviceRescanPath, []byte{'1'}, 0666)
+	err = triggerRescan(blockDeviceRescanPath)
 	if err != nil {
-		klog.Errorf("Error rescanning new block device geometry: %v", err)
 		// no need to run checkBlockDeviceSize second time here, return the saved error
 		return bdSizeErr
 	}
 
 	return checkBlockDeviceSize(devicePath, deviceMountPath, newSize)
 }
+
+func RescanDevice(devicePath string) error {
+	blockDeviceRescanPath, err := findBlockDeviceRescanPath(devicePath)
+	if err != nil {
+		return fmt.Errorf("Device does not have rescan path " + devicePath)
+	}
+
+	klog.V(3).Infof("Resolved block device path from %q to %q", devicePath, blockDeviceRescanPath)
+	err = triggerRescan(blockDeviceRescanPath)
+	if err != nil {
+		return fmt.Errorf("Error rescanning new block device geometry " + devicePath)
+	}
+	return nil
+}