rbd: add mkfsOptions to the StorageClass and pass them to mkfs

docs/deploy-rbd.md

@@ -61,6 +61,7 @@ make image-cephcsi
 | `volumeNamePrefix`                                                                                  | no                   | Prefix to use for naming RBD images (defaults to `csi-vol-`).                                                                                                                                                                                                                                      |
 | `snapshotNamePrefix`                                                                                | no                   | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`).                                                                                                                                                                                                                            |
 | `imageFeatures`                                                                                     | no                   | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff`, `deep-flatten` features. deep-flatten is added for cloned images. Refer <https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features> for image feature dependencies. |
+| `mkfsOptions`                                                                                       | no                   | Options to pass to the `mkfs` command while creating the filesystem on the RBD device. Check the man-page for the `mkfs` command for the filesystem for more details. When `mkfsOptions` is set here, the defaults will not be used, consider including them in this parameter.                    |
 | `tryOtherMounters`                                                                                  | no                   | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason                                                                                                                                                                                 |
 | `mapOptions`                                                                                        | no                   | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options.                                                                                           |
 | `unmapOptions`                                                                                      | no                   | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options.                                                                                       |

e2e/rbd.go

@@ -419,7 +419,7 @@ var _ = Describe("RBD", func() {
 				By("verify PVC and app binding on helm installation", func() {
 					err := validatePVCAndAppBinding(pvcPath, appPath, f)
 					if err != nil {
-						framework.Failf("failed to validate CephFS pvc and application binding: %v", err)
+						framework.Failf("failed to validate RBD pvc and application binding: %v", err)
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
@@ -1089,6 +1089,45 @@ var _ = Describe("RBD", func() {
 				}
 			})
 
+			By("create a PVC and bind it to an app with ext4 as the FS and 1024 inodes ", func() {
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					framework.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(
+					f.ClientSet,
+					f,
+					defaultSCName,
+					nil,
+					map[string]string{
+						"csi.storage.k8s.io/fstype": "ext4",
+						"mkfsOptions":               "-N1024", // 1024 inodes
+					},
+					deletePolicy)
+				if err != nil {
+					framework.Failf("failed to create storageclass: %v", err)
+				}
+				err = validatePVCAndAppBinding(pvcPath, appPath, f)
+				if err != nil {
+					framework.Failf("failed to validate pvc and application binding: %v", err)
+				}
+				err = validateInodeCount(pvcPath, f, 1024)
+				if err != nil {
+					framework.Failf("failed to validate pvc and application binding: %v", err)
+				}
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					framework.Failf("failed to delete storageclass: %v", err)
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					framework.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
 			By("create a PVC and bind it to an app using rbd-nbd mounter", func() {
 				if !testNBD {
 					framework.Logf("skipping NBD test")
@@ -1207,7 +1246,7 @@ var _ = Describe("RBD", func() {
 						}
 						err = validatePVCAndAppBinding(pvcPath, appPath, f)
 						if err != nil {
-							framework.Failf("failed to validate CephFS pvc and application binding: %v", err)
+							framework.Failf("failed to validate RBD pvc and application binding: %v", err)
 						}
 						// validate created backend rbd images
 						validateRBDImageCount(f, 0, defaultRBDPool)
@@ -1372,7 +1411,7 @@ var _ = Describe("RBD", func() {
 						}
 						err = validatePVCAndAppBinding(pvcPath, appPath, f)
 						if err != nil {
-							framework.Failf("failed to validate CephFS pvc and application binding: %v", err)
+							framework.Failf("failed to validate RBD pvc and application binding: %v", err)
 						}
 						// validate created backend rbd images
 						validateRBDImageCount(f, 0, defaultRBDPool)

e2e/utils.go

@@ -486,6 +486,7 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) e
 	if err != nil {
 		return err
 	}
+
 	err = deletePVCAndApp("", f, pvc, app)
 
 	return err
@@ -508,6 +509,50 @@ func getMountType(selector, mountPath string, f *framework.Framework) (string, e
 }
 
 func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
+	writeTest := func(ns string, opts *metav1.ListOptions) error {
+		_, stdErr, err := execCommandInPod(f, "echo testing > /target/testing", ns, opts)
+		if err != nil {
+			return fmt.Errorf("failed to exec command in pod: %w", err)
+		}
+		if stdErr != "" {
+			return fmt.Errorf("failed to touch a file as non-root user %v", stdErr)
+		}
+
+		return nil
+	}
+
+	return validateNormalUserPVCAccessFunc(pvcPath, f, writeTest)
+}
+
+func validateInodeCount(pvcPath string, f *framework.Framework, inodes int) error {
+	countInodes := func(ns string, opts *metav1.ListOptions) error {
+		stdOut, stdErr, err := execCommandInPod(f, "df --output=itotal /target | tail -n1", ns, opts)
+		if err != nil {
+			return fmt.Errorf("failed to exec command in pod: %w", err)
+		}
+		if stdErr != "" {
+			return fmt.Errorf("failed to list inodes in pod: %v", stdErr)
+		}
+
+		itotal, err := strconv.Atoi(strings.TrimSpace(stdOut))
+		if err != nil {
+			return fmt.Errorf("failed to parse itotal %q to int: %w", strings.TrimSpace(stdOut), err)
+		}
+		if inodes != itotal {
+			return fmt.Errorf("expected inodes (%d) do not match itotal on volume (%d)", inodes, itotal)
+		}
+
+		return nil
+	}
+
+	return validateNormalUserPVCAccessFunc(pvcPath, f, countInodes)
+}
+
+func validateNormalUserPVCAccessFunc(
+	pvcPath string,
+	f *framework.Framework,
+	validate func(ns string, opts *metav1.ListOptions) error,
+) error {
 	pvc, err := loadPVC(pvcPath)
 	if err != nil {
 		return err
@@ -571,12 +616,10 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
 	opt := metav1.ListOptions{
 		LabelSelector: "app=pod-run-as-non-root",
 	}
-	_, stdErr, err := execCommandInPod(f, "echo testing > /target/testing", app.Namespace, &opt)
+
+	err = validate(app.Namespace, &opt)
 	if err != nil {
-		return fmt.Errorf("failed to exec command in pod: %w", err)
-	}
-	if stdErr != "" {
-		return fmt.Errorf("failed to touch a file as non-root user %v", stdErr)
+		return fmt.Errorf("failed to run validation function: %w", err)
 	}
 
 	// metrics for BlockMode was added in Kubernetes 1.22

examples/rbd/storageclass.yaml

@@ -37,6 +37,17 @@ parameters:
    # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff
    imageFeatures: "layering"
 
+   # (optional) Options to pass to the `mkfs` command while creating the
+   # filesystem on the RBD device. Check the man-page for the `mkfs` command
+   # for the filesystem for more details. When `mkfsOptions` is set here, the
+   # defaults will not be used, consider including them in this parameter.
+   #
+   # The default options depend on the csi.storage.k8s.io/fstype setting:
+   # - ext4: "-m0 -Enodiscard,lazy_itable_init=1,lazy_journal_init=1"
+   # - xfs: "-onouuid -K"
+   #
+   # mkfsOptions: "-m0 -Ediscard -i1024"
+
    # (optional) Specifies whether to try other mounters in case if the current
    # mounter fails to mount the rbd image for any reason. True means fallback
    # to next mounter, default is set to false.

internal/rbd/nodeserver.go

@@ -101,6 +101,15 @@ var (
 	// xfsHasReflink is set by xfsSupportsReflink(), use the function when
 	// checking the support for reflink.
 	xfsHasReflink = xfsReflinkUnset
+
+	mkfsDefaultArgs = map[string][]string{
+		"ext4": {"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1"},
+		"xfs":  {"-K"},
+	}
+
+	mountDefaultOpts = map[string][]string{
+		"xfs": {"nouuid"},
+	}
 )
 
 // parseBoolOption checks if parameters contain option and parse it. If it is
@@ -756,7 +765,8 @@ func (ns *NodeServer) mountVolumeToStagePath(
 		return err
 	}
 
-	opt := []string{"_netdev"}
+	opt := mountDefaultOpts[fsType]
+	opt = append(opt, "_netdev")
 	opt = csicommon.ConstructMountOptions(opt, req.GetVolumeCapability())
 	isBlock := req.GetVolumeCapability().GetBlock() != nil
 	rOnly := "ro"
@@ -771,34 +781,43 @@ func (ns *NodeServer) mountVolumeToStagePath(
 		readOnly = true
 	}
 
-	if fsType == "xfs" {
-		opt = append(opt, "nouuid")
-	}
+	if existingFormat == "" && !staticVol && !readOnly && !isBlock {
+		args := mkfsDefaultArgs[fsType]
 
-	if existingFormat == "" && !staticVol && !readOnly {
-		args := []string{}
+		// if the VolumeContext contains "mkfsOptions", use those as args instead
+		volumeCtx := req.GetVolumeContext()
+		if volumeCtx != nil {
+			mkfsOptions := volumeCtx["mkfsOptions"]
+			if mkfsOptions != "" {
+				args = strings.Split(mkfsOptions, " ")
+			}
+		}
+
+		// add extra arguments depending on the filesystem
+		mkfs := "mkfs." + fsType
 		switch fsType {
 		case "ext4":
-			args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1"}
 			if fileEncryption {
 				args = append(args, "-Oencrypt")
 			}
-			args = append(args, devicePath)
 		case "xfs":
-			args = []string{"-K", devicePath}
 			// always disable reflink
 			// TODO: make enabling an option, see ceph/ceph-csi#1256
 			if ns.xfsSupportsReflink() {
 				args = append(args, "-m", "reflink=0")
 			}
+		case "":
+			// no filesystem type specified, just use "mkfs"
+			mkfs = "mkfs"
 		}
-		if len(args) > 0 {
-			cmdOut, cmdErr := diskMounter.Exec.Command("mkfs."+fsType, args...).CombinedOutput()
-			if cmdErr != nil {
-				log.ErrorLog(ctx, "failed to run mkfs error: %v, output: %v", cmdErr, string(cmdOut))
 
-				return cmdErr
-			}
+		// add device as last argument
+		args = append(args, devicePath)
+		cmdOut, cmdErr := diskMounter.Exec.Command(mkfs, args...).CombinedOutput()
+		if cmdErr != nil {
+			log.ErrorLog(ctx, "failed to run mkfs.%s (%v) error: %v, output: %v", fsType, args, cmdErr, string(cmdOut))
+
+			return cmdErr
 		}
 	}