Skip to content

Commit

Permalink
rbd: add support for rbd striping
Browse files Browse the repository at this point in the history 
RBD supports creating rbd images with
object size, strip unit and strip count
to support striping. This PR adds the support
for the same.

More details about strip at
https://docs.ceph.com/en/quincy/man/8/rbd/#striping

fixes: #3124

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
  • Loading branch information
@Madhu-1
Madhu-1 committed May 25, 2022
1 parent 952105e commit e1b64bf
Show file tree
Hide file tree
Showing 6 changed files with 342 additions and 36 deletions.
3 changes: 3 additions & 0 deletions docs/deploy-rbd.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ make image-cephcsi
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** |
| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases |
| `stripeUnit` | no | stripe unit in bytes |
| `stripeCount` | no | objects to stripe over before looping |
| `objectSize` | no | object size in bytes |

**NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)
Expand Down
147 changes: 147 additions & 0 deletions e2e/rbd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4041,6 +4041,153 @@ var _ = Describe("RBD", func() {
})
})

By("validate rbd image stripe", func() {
stripeUnit := 4096
stripeCount := 8
objectSize := 4096
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}

err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
map[string]string{
"stripeUnit": fmt.Sprintf("%d", stripeUnit),
"stripeCount": fmt.Sprintf("%d", stripeCount),
"objectSize": fmt.Sprintf("%d", objectSize),
},
deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
}()

err = createRBDSnapshotClass(f)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteRBDSnapshotClass()
if err != nil {
e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
}
}()

// create PVC and bind it to an app
pvc, err := loadPVC(pvcPath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}

pvc.Namespace = f.UniqueName

err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
err = validateStripe(f, pvc, stripeUnit, stripeCount, objectSize)
if err != nil {
e2elog.Failf("failed to validate stripe: %v", err)
}

snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name

err = createSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to create snapshot: %v", err)
}
// validate created backend rbd images
// parent PVC + snapshot
totalImages := 2
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
pvcClone, err := loadPVC(pvcClonePath)
if err != nil {
e2elog.Failf("failed to load PVC: %v", err)
}

// create clone PVC as ROX
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.AccessModes = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany}
err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC: %v", err)
}
// validate created backend rbd images
// parent pvc + snapshot + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
err = validateStripe(f, pvcClone, stripeUnit, stripeCount, objectSize)
if err != nil {
e2elog.Failf("failed to validate stripe for clone: %v", err)
}
// delete snapshot
err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete snapshot: %v", err)
}
// delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete PVC: %v", err)
}

pvcSmartClone, err := loadPVC(pvcSmartClonePath)
if err != nil {
e2elog.Failf("failed to load pvcSmartClone: %v", err)
}
pvcSmartClone.Namespace = f.UniqueName

err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to create pvc: %v", err)
}
// validate created backend rbd images
// parent pvc + temp clone + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
err = validateStripe(f, pvcClone, stripeUnit, stripeCount, objectSize)
if err != nil {
e2elog.Failf("failed to validate stripe for clone: %v", err)
}
// delete parent pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete PVC: %v", err)
}

// delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete PVC: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})

// Make sure this should be last testcase in this file, because
// it deletes pool
By("Create a PVC and delete PVC when backend pool deleted", func() {
Expand Down
67 changes: 67 additions & 0 deletions e2e/rbd_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,3 +940,70 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)

return err
}

// imageInfo strongly typed JSON spec for image info.
type imageInfo struct {
ObjectUUID string `json:"name"`
Size int64 `json:"size"`
Format int64 `json:"format"`
StripeUnit int `json:"stripe_unit"`
StripeCount int `json:"stripe_count"`
ObjectSize int `json:"object_size"`
}

// getImageInfo queries rbd about the given image and returns its metadata, and returns
// error if provided image is not found.
func getImageInfo(f *framework.Framework, imageName, poolName string) (imageInfo, error) {
// rbd --format=json info [image-spec | snap-spec]
var imgInfo imageInfo

stdOut, stdErr, err := execCommandInToolBoxPod(
f,
fmt.Sprintf("rbd info %s %s --format json", rbdOptions(poolName), imageName),
rookNamespace)
if err != nil {
return imgInfo, fmt.Errorf("failed to get rbd info: %w", err)
}
if stdErr != "" {
return imgInfo, fmt.Errorf("failed to get rbd info: %v", stdErr)
}
err = json.Unmarshal([]byte(stdOut), &imgInfo)
if err != nil {
return imgInfo, fmt.Errorf("unmarshal failed: %w. raw buffer response: %s",
err, stdOut)
}

return imgInfo, nil
}

// validateStripe validate the stripe count, stripe unit and object size of the
// image.
func validateStripe(f *framework.Framework,
pvc *v1.PersistentVolumeClaim,
stripeUnit,
stripeCount,
objectSize int) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}

imgInfo, err := getImageInfo(f, imageData.imageName, defaultRBDPool)
if err != nil {
return err
}

if imgInfo.ObjectSize != objectSize {
return fmt.Errorf("objectSize %d does not match expected %d", imgInfo.ObjectSize, objectSize)
}

if imgInfo.StripeUnit != stripeUnit {
return fmt.Errorf("stripeUnit %d does not match expected %d", imgInfo.StripeUnit, stripeUnit)
}

if imgInfo.StripeCount != stripeCount {
return fmt.Errorf("stripeCount %d does not match expected %d", imgInfo.StripeCount, stripeCount)
}

return nil
}
8 changes: 8 additions & 0 deletions examples/rbd/storageclass.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@ parameters:
# {"domainLabel":"zone","value":"zone1"}]}
# ]

# Image striping, Refer https://docs.ceph.com/en/latest/man/8/rbd/#striping
# For more details
# (optional) stripe unit in bytes.
# stripeUnit: <>
# (optional) objects to stripe over before looping.
# stripeCount: <>
# (optional) The object size in bytes.
# objectSize: <>
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
Expand Down
19 changes: 19 additions & 0 deletions internal/rbd/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,25 @@ func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.Crea
return err
}

err = validateStriping(req.Parameters)
if err != nil {
return status.Error(codes.InvalidArgument, err.Error())
}

return nil
}

func validateStriping(parameters map[string]string) error {
stripeUnit := parameters["stripeUnit"]
stripeCount := parameters["stripeCount"]
if stripeUnit != "" && stripeCount == "" {
return errors.New("stripeCount must be specified when stripeUnit is specified")
}

if stripeUnit == "" && stripeCount != "" {
return errors.New("stripeUnit must be specified when stripeCount is specified")
}

return nil
}

Expand Down
Loading

0 comments on commit e1b64bf

Please sign in to comment.