From a164169fd30db4419c9ff04c9e5cc688b41688eb Mon Sep 17 00:00:00 2001 From: j-griffith Date: Wed, 13 Mar 2019 12:19:14 -0600 Subject: [PATCH 1/2] Revert "Add multiNodeWritable option for RBD Volumes" This reverts commit b5b8e4646094d0ec1f3dfe96060a183c863d7d1a. --- Makefile | 2 +- docs/deploy-rbd.md | 15 --- examples/README.md | 164 --------------------------------- examples/rbd/storageclass.yaml | 3 - pkg/rbd/controllerserver.go | 27 +----- pkg/rbd/nodeserver.go | 11 +-- pkg/rbd/rbd.go | 7 +- pkg/rbd/rbd_attach.go | 8 -- pkg/rbd/rbd_util.go | 9 +- 9 files changed, 8 insertions(+), 238 deletions(-) diff --git a/Makefile b/Makefile index 825ed8b666d..734a761b8a1 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ go-test: ./scripts/test-go.sh static-check: - ./scripts/lint-go.sh + ./scripts/lint-go.sh ./scripts/lint-text.sh rbdplugin: diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index acfde7d4663..bfaa46bb968 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -58,21 +58,6 @@ Parameter | Required | Description `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-publish-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | for Kubernetes | namespaces of the above Secret objects `mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images -`fsType` | no | allows setting to `ext3 | ext-4 | xfs`, default is `ext-4` -`multiNodeWritable` | no | if set to `enabled` allows RBD volumes with MultiNode Access Modes to bypass watcher checks. By default multiple attachments of an RBD volume are NOT allowed. Even if this option is set in the StorageClass, it's ignored if a standard SingleNodeWriter Access Mode is requested - -**Warning for multiNodeWritable:** - -*NOTE* the `multiNodeWritable` setting is NOT safe for use by workloads -that are not designed to coordinate access. This does NOT add any sort -of a clustered filesystem or write syncronization, it's specifically for -special workloads that handle access coordination on their own -(ie Active/Passive scenarios). - -Using this mode for general purposes *WILL RESULT IN DATA CORRUPTION*. -We attempt to limit exposure to trouble here but ignoring the Storage Class -setting unless your Volume explicitly asks for multi node access, and assume -you know what you're doing. **Required secrets:** diff --git a/examples/README.md b/examples/README.md index 16238fd983d..d309cdcaf1d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -114,167 +114,3 @@ To restore the snapshot to a new PVC, deploy kubectl create -f pvc-restore.yaml kubectl create -f pod-restore.yaml ``` - -## How to enable multi node attach support for RBD - -*WARNING* This feature is strictly for workloads that know how to deal -with concurrent acces to the Volume (eg Active/Passive applications). -Using RWX modes on non clustered file systems with applications trying -to simultaneously access the Volume will likely result in data corruption! - -### Example process to test the multiNodeWritable feature - -Modify your current storage class, or create a new storage class specifically -for multi node writers by adding the `multiNodeWritable: "enabled"` entry to -your parameters. Here's an example: - -```yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-rbd -provisioner: csi-rbdplugin -parameters: - monitors: rook-ceph-mon-b.rook-ceph.svc.cluster.local:6789 - pool: rbd - imageFormat: "2" - imageFeatures: layering - csiProvisionerSecretName: csi-rbd-secret - csiProvisionerSecretNamespace: default - csiNodePublishSecretName: csi-rbd-secret - csiNodePublishSecretNamespace: default - adminid: admin - userid: admin - fsType: xfs - multiNodeWritable: "enabled" -reclaimPolicy: Delete -``` - -Now, you can request Claims from the configured storage class that include -the `ReadWriteMany` access mode: - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: pvc-1 -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi - storageClassName: csi-rbd -``` - -Create a POD that uses this PVC: - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: test-1 -spec: - containers: - - name: web-server - image: nginx - volumeMounts: - - name: mypvc - mountPath: /var/lib/www/html - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: pvc-1 - readOnly: false -``` - -Wait for the POD to enter Running state, write some data to -`/var/lib/www/html` - -Now, we can create a second POD (ensure the POD is scheduled on a different -node; multiwriter single node works without this feature) that also uses this -PVC at the same time - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: test-2 -spec: - containers: - - name: web-server - image: nginx - volumeMounts: - - name: mypvc - mountPath: /var/lib/www/html - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: pvc-1 - readOnly: false -``` - -If you access the pod you can check that your data is avaialable at -`/var/lib/www/html` - -## Testing Raw Block feature in kubernetes with RBD volumes - -CSI block volume support is feature-gated and turned off by default. To run CSI -with block volume support enabled, a cluster administrator must enable the -feature for each Kubernetes component using the following feature gate flags: - ---feature-gates=BlockVolume=true,CSIBlockVolume=true - -these feature-gates must be enabled on both api-server and kubelet - -### create a raw-block PVC - -```yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: raw-block-pvc -spec: - accessModes: - - ReadWriteOnce - volumeMode: Block - resources: - requests: - storage: 1Gi - storageClassName: csi-rbd -``` - -create raw block pvc - -```console -kubectl create -f raw-block-pvc.yaml -``` - -### create a pod to mount raw-block PVC - -```yaml ---- -apiVersion: v1 -kind: Pod -metadata: - name: pod-with-raw-block-volume -spec: - containers: - - name: fc-container - image: fedora:26 - command: ["/bin/sh", "-c"] - args: [ "tail -f /dev/null" ] - volumeDevices: - - name: data - devicePath: /dev/xvda - volumes: - - name: data - persistentVolumeClaim: - claimName: raw-block-pvc -``` - -Create a POD that uses raw block PVC - -```console -kubectl create -f raw-block-pod.yaml -``` diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index f7de85f615c..320a489a8f1 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -35,7 +35,4 @@ parameters: userid: kubernetes # uncomment the following to use rbd-nbd as mounter on supported nodes # mounter: rbd-nbd - # fsType: xfs - # uncomment the following line to enable multi-attach on RBD volumes - # multiNodeWritable: enabled reclaimPolicy: Delete diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 41195f15c49..ba7cb909205 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -21,7 +21,6 @@ import ( "os/exec" "sort" "strconv" - "strings" "syscall" csicommon "github.com/ceph/ceph-csi/pkg/csi-common" @@ -94,16 +93,7 @@ func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) erro func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { // TODO (sbezverk) Last check for not exceeding total storage capacity - // MultiNodeWriters are accepted but they're only for special cases, and we skip the watcher checks for them which isn't the greatest - // let's make sure we ONLY skip that if the user is requesting a MULTI Node accessible mode - disableMultiWriter := true - for _, am := range req.VolumeCapabilities { - if am.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - disableMultiWriter = false - } - } - - rbdVol, err := getRBDVolumeOptions(req.GetParameters(), disableMultiWriter) + rbdVol, err := getRBDVolumeOptions(req.GetParameters()) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -344,20 +334,11 @@ func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolume // ValidateVolumeCapabilities checks whether the volume capabilities requested // are supported. func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - params := req.GetParameters() - multiWriter := params["multiNodeWritable"] - if strings.ToLower(multiWriter) == "enabled" { - klog.V(3).Info("detected multiNodeWritable parameter in Storage Class, allowing multi-node access modes") - - } else { - for _, cap := range req.VolumeCapabilities { - if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil - } + for _, cap := range req.VolumeCapabilities { + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil } - } - return &csi.ValidateVolumeCapabilitiesResponse{ Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{ VolumeCapabilities: req.VolumeCapabilities, diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 2faed49c1a3..21d7ae8299a 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -70,19 +70,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if !notMnt { return &csi.NodePublishVolumeResponse{}, nil } - - // if our access mode is a simple SINGLE_NODE_WRITER we're going to ignore the SC directive and use the - // watcher still - ignoreMultiWriterEnabled := true - if req.VolumeCapability.AccessMode.Mode != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { - ignoreMultiWriterEnabled = false - } - - volOptions, err := getRBDVolumeOptions(req.GetVolumeContext(), ignoreMultiWriterEnabled) + volOptions, err := getRBDVolumeOptions(req.GetVolumeContext()) if err != nil { return nil, err } - volOptions.VolName = volName // Mapping RBD image devicePath, err := attachRBDImage(volOptions, volOptions.UserID, req.GetSecrets()) diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index c7b2eab89e1..73911aec428 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -102,12 +102,7 @@ func (r *Driver) Run(driverName, nodeID, endpoint string, containerized bool, ca csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, }) - - // TODO: JDG Should also look at remaining modes like MULT_NODE_READER (SINGLE_READER) - r.cd.AddVolumeCapabilityAccessModes( - []csi.VolumeCapability_AccessMode_Mode{ - csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, - csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + r.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) // Create GRPC servers r.ids = NewIdentityServer(r.cd) diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 88834757b53..354554d120c 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -313,16 +313,8 @@ func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } - // In the case of multiattach we want to short circuit the retries when used (so r`if used; return used`) - // otherwise we're setting this to false which translates to !ok, which means backoff and try again - // NOTE: we ONLY do this if an multi-node access mode is requested for this volume - if (strings.ToLower(volOptions.MultiNodeWritable) == "enabled") && (used) { - klog.V(2).Info("detected MultiNodeWritable enabled, ignoring watcher in-use result") - return used, nil - } return !used, nil }) - // return error if rbd image has not become available for the specified timeout if err == wait.ErrWaitTimeout { return fmt.Errorf("rbd image %s is still being used", imagePath) diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 5f7f16c8953..5f821549616 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -51,7 +51,6 @@ type rbdVolume struct { AdminID string `json:"adminId"` UserID string `json:"userId"` Mounter string `json:"mounter"` - MultiNodeWritable string `json:"multiNodeWritable"` } type rbdSnapshot struct { @@ -227,7 +226,7 @@ func execCommand(command string, args []string) ([]byte, error) { return cmd.CombinedOutput() } -func getRBDVolumeOptions(volOptions map[string]string, ignoreMultiNodeWritable bool) (*rbdVolume, error) { +func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { var ok bool rbdVol := &rbdVolume{} rbdVol.Pool, ok = volOptions["pool"] @@ -261,12 +260,6 @@ func getRBDVolumeOptions(volOptions map[string]string, ignoreMultiNodeWritable b } getCredsFromVol(rbdVol, volOptions) - - klog.V(3).Infof("ignoreMultiNodeWritable flag in parse getRBDVolumeOptions is: %v", ignoreMultiNodeWritable) - // If the volume we're working with is NOT requesting multi-node attach then don't treat it special, ignore the setting in the SC and just keep our watcher checks - if !ignoreMultiNodeWritable { - rbdVol.MultiNodeWritable = volOptions["multiNodeWritable"] - } return rbdVol, nil } From 6ec1196f478ebc07dadc87c351b411f4e028de7f Mon Sep 17 00:00:00 2001 From: j-griffith Date: Wed, 13 Mar 2019 18:18:04 -0600 Subject: [PATCH 2/2] Rework multi-node-multi-writer feature This commit reverts the initial implementation of the multi-node-multi-writer feature: commit: b5b8e4646094d0ec1f3dfe96060a183c863d7d1a It replaces that implementation with a more restrictive version that only allows multi-node-multi-writer for volumes of type `block` With this change there are no volume parameters required in the stoarge class, we also fail any attempt to create a file based device with multi-node-multi-write being specified, this way a user doesn't have to wait until they try and do the publish before realizing it doesn't work. --- Makefile | 2 +- examples/README.md | 99 +++++++++++++++++++++++++++++++++++++ pkg/rbd/controllerserver.go | 20 +++++++- pkg/rbd/nodeserver.go | 15 +++++- pkg/rbd/rbd.go | 9 +++- pkg/rbd/rbd_attach.go | 5 ++ pkg/rbd/rbd_util.go | 7 ++- 7 files changed, 152 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 734a761b8a1..825ed8b666d 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ go-test: ./scripts/test-go.sh static-check: - ./scripts/lint-go.sh + ./scripts/lint-go.sh ./scripts/lint-text.sh rbdplugin: diff --git a/examples/README.md b/examples/README.md index d309cdcaf1d..d27f9777a17 100644 --- a/examples/README.md +++ b/examples/README.md @@ -114,3 +114,102 @@ To restore the snapshot to a new PVC, deploy kubectl create -f pvc-restore.yaml kubectl create -f pod-restore.yaml ``` + +## How to test RBD MULTI_NODE_MULTI_WRITER BLOCK feature + +Requires feature-gates: `BlockVolume=true` `CSIBlockVolume=true` + +*NOTE* The MULTI_NODE_MULTI_WRITER capability is only available for +Volumes that are of access_type `block` + +*WARNING* This feature is strictly for workloads that know how to deal +with concurrent access to the Volume (eg Active/Passive applications). +Using RWX modes on non clustered file systems with applications trying +to simultaneously access the Volume will likely result in data corruption! + +Following are examples for issuing a request for a `Block` +`ReadWriteMany` Claim, and using the resultant Claim for a POD + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: block-pvc +spec: + accessModes: + - ReadWriteMany + volumeMode: Block + resources: + requests: + storage: 1Gi + storageClassName: csi-rbd +``` + +Create a POD that uses this PVC: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: my-container + image: debian + command: ["/bin/bash", "-c"] + args: [ "tail -f /dev/null" ] + volumeDevices: + - devicePath: /dev/rbdblock + name: my-volume + imagePullPolicy: IfNotPresent + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: block-pvc + +``` + +Now, we can create a second POD (ensure the POD is scheduled on a different +node; multiwriter single node works without this feature) that also uses this +PVC at the same time, again wait for the pod to enter running state, and verify +the block device is available. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: another-pod +spec: + containers: + - name: my-container + image: debian + command: ["/bin/bash", "-c"] + args: [ "tail -f /dev/null" ] + volumeDevices: + - devicePath: /dev/rbdblock + name: my-volume + imagePullPolicy: IfNotPresent + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: block-pvc +``` + +Wait for the PODs to enter Running state, check that our block device +is available in the container at `/dev/rdbblock` in both containers: + +```bash +$ kubectl exec -it my-pod -- fdisk -l /dev/rbdblock +Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors +Units: sectors of 1 * 512 = 512 bytes +Sector size (logical/physical): 512 bytes / 512 bytes +I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes +``` + +```bash +$ kubectl exec -it another-pod -- fdisk -l /dev/rbdblock +Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors +Units: sectors of 1 * 512 = 512 bytes +Sector size (logical/physical): 512 bytes / 512 bytes +I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes +``` diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index ba7cb909205..49af632064c 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -93,7 +93,25 @@ func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) erro func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { // TODO (sbezverk) Last check for not exceeding total storage capacity - rbdVol, err := getRBDVolumeOptions(req.GetParameters()) + isMultiNode := false + isBlock := false + for _, cap := range req.VolumeCapabilities { + // Only checking SINGLE_NODE_SINGLE_WRITER here because regardless of the other types (MULTI READER) we need to implement the same logic to ignore the in-use response + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + isMultiNode = true + } + if cap.GetBlock() != nil { + isBlock = true + } + } + + // We want to fail early if the user is trying to create a RWX on a non-block type device + if isMultiNode && !isBlock { + return nil, status.Error(codes.InvalidArgument, "multi node access modes are only supported on rbd `block` type volumes") + } + + // if it's NOT SINGLE_NODE_WRITER and it's BLOCK we'll set the parameter to ignore the in-use checks + rbdVol, err := getRBDVolumeOptions(req.GetParameters(), (isMultiNode && isBlock)) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 21d7ae8299a..3a88b456e8a 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -48,6 +48,7 @@ type NodeServer struct { func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { targetPath := req.GetTargetPath() targetPathMutex.LockKey(targetPath) + disableInUseChecks := false defer func() { if err := targetPathMutex.UnlockKey(targetPath); err != nil { @@ -70,7 +71,19 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if !notMnt { return &csi.NodePublishVolumeResponse{}, nil } - volOptions, err := getRBDVolumeOptions(req.GetVolumeContext()) + + // MULTI_NODE_MULTI_WRITER is supported by default for Block access type volumes + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { + if isBlock { + disableInUseChecks = true + } else { + klog.Warningf("MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`, invalid AccessMode for volume: %v", req.GetVolumeId()) + e := fmt.Errorf("rbd: MULTI_NODE_MULTI_WRITER access mode only allowed with BLOCK access type") + return nil, status.Error(codes.InvalidArgument, e.Error()) + } + } + + volOptions, err := getRBDVolumeOptions(req.GetVolumeContext(), disableInUseChecks) if err != nil { return nil, err } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 73911aec428..3fab59e015a 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -102,7 +102,14 @@ func (r *Driver) Run(driverName, nodeID, endpoint string, containerized bool, ca csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, }) - r.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) + + // We only support the multi-writer option when using block, but it's a supported capability for the plugin in general + // In addition, we want to add the remaining modes like MULTI_NODE_READER_ONLY, + // MULTI_NODE_SINGLE_WRITER etc, but need to do some verification of RO modes first + // will work those as follow up features + r.cd.AddVolumeCapabilityAccessModes( + []csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) // Create GRPC servers r.ids = NewIdentityServer(r.cd) diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 354554d120c..613cfa765e2 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -258,6 +258,7 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string Factor: rbdImageWatcherFactor, Steps: rbdImageWatcherSteps, } + err = waitForrbdImage(backoff, volOptions, userID, credentials) if err != nil { @@ -313,6 +314,10 @@ func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } + if (volOptions.DisableInUseChecks) && (used) { + klog.V(2).Info("valid multi-node attach requested, ignoring watcher in-use result") + return used, nil + } return !used, nil }) // return error if rbd image has not become available for the specified timeout diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 5f821549616..aa5b19f7909 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -51,6 +51,7 @@ type rbdVolume struct { AdminID string `json:"adminId"` UserID string `json:"userId"` Mounter string `json:"mounter"` + DisableInUseChecks bool `json:"disableInUseChecks"` } type rbdSnapshot struct { @@ -226,7 +227,7 @@ func execCommand(command string, args []string) ([]byte, error) { return cmd.CombinedOutput() } -func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { +func getRBDVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) { var ok bool rbdVol := &rbdVolume{} rbdVol.Pool, ok = volOptions["pool"] @@ -259,6 +260,10 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { } } + + klog.V(3).Infof("setting disableInUseChecks on rbd volume to: %v", disableInUseChecks) + rbdVol.DisableInUseChecks = disableInUseChecks + getCredsFromVol(rbdVol, volOptions) return rbdVol, nil }