rbd: add additional space for encrypted volumes

issue: when a block-mode pvc is created with encryption enabled
there is some space reserved for the encryption metadata.
Which doesn't allows users to write extact amount of data that
they have requested for.

solution: create pvc with extra space needed for the encryption
metadata.

The extra space is added during the CreateVolume and ExpandVolume
operations. And while returning the response remove the extra space
so the client/user gets the requested size reported.

Signed-off-by: Praveen M <m.praveen@ibm.com>

PendingReleaseNotes.md

@@ -12,5 +12,6 @@
 - deploy: radosNamespaceCephFS can be configured for ceph-csi-cephfs chart in [PR](https://github.com/ceph/ceph-csi/pull/4652)
 - build: update ceph release to squid in [PR](https://github.com/ceph/ceph-csi/pull/4735)
 - build: CentOS Stream 9 is used as OS in the container-images [PR](https://github.com/ceph/ceph-csi/pull/4735)
+- rbd: add additional space for encrypted volumes for Luks2 header in [PR] (https://github.com/ceph/ceph-csi/pull/4582)
 
 ## NOTE

internal/rbd/controllerserver.go

@@ -1235,6 +1235,17 @@ func (cs *ControllerServer) CreateSnapshot(
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
+	err = vol.Connect(cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer vol.Destroy(ctx)
+
+	err = vol.getImageInfo()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	csiSnap, err := vol.toSnapshot().ToCSI(ctx)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -1294,6 +1305,17 @@ func cloneFromSnapshot(
 		}
 	}
 
+	err = rbdSnap.Connect(cr)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer rbdSnap.Destroy(ctx)
+
+	err = rbdSnap.getImageInfo()
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
 	csiSnap, err := rbdSnap.ToCSI(ctx)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())

internal/rbd/encryption.go

@@ -59,6 +59,9 @@ const (
 	metadataDEK    = "rbd.csi.ceph.com/dek"
 	oldMetadataDEK = ".rbd.csi.ceph.com/dek"
 
+	// luks2 header size metadata key.
+	luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize"
+
 	encryptionPassphraseSize = 20
 
 	// rbdDefaultEncryptionType is the default to use when the
@@ -130,6 +133,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error {
 		return err
 	}
 
+	err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(util.Luks2HeaderSize, 10))
+	if err != nil {
+		return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err)
+	}
+
 	err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared)
 	if err != nil {
 		log.ErrorLog(ctx, "failed to save encryption status, deleting "+

internal/rbd/rbd_util.go

@@ -449,8 +449,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er
 		return fmt.Errorf("failed to get IOContext: %w", err)
 	}
 
-	err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName,
-		uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options)
+	size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB)
+	if pOpts.isBlockEncrypted() {
+		// When a block-mode PVC is created with encryption enabled,
+		// some space is reserved for the LUKS2 header.
+		// Add the LUKS2 header size to the image size so that the user has at least
+		// the requested size.
+		size += util.Luks2HeaderSize
+	}
+
+	err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options)
 	if err != nil {
 		return fmt.Errorf("failed to create rbd image: %w", err)
 	}
@@ -1604,6 +1612,26 @@ func (ri *rbdImage) GetCreationTime(ctx context.Context) (*time.Time, error) {
 	return ri.CreatedAt, nil
 }
 
+// getLuks2HeaderSizeSet returns the value of the LUKS2 header size
+// set in the image metadata.
+func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) {
+	value, err := ri.GetMetadata(luks2HeaderSizeKey)
+	if err != nil {
+		if !errors.Is(err, librbd.ErrNotFound) {
+			return 0, err
+		}
+
+		return 0, nil
+	}
+
+	headerSize, parseErr := strconv.ParseUint(value, 10, 64)
+	if parseErr != nil {
+		return 0, parseErr
+	}
+
+	return headerSize, nil
+}
+
 // getImageInfo queries rbd about the given image and returns its metadata, and returns
 // ErrImageNotFound if provided image is not found.
 func (ri *rbdImage) getImageInfo() error {
@@ -1620,6 +1648,14 @@ func (ri *rbdImage) getImageInfo() error {
 	// TODO: can rv.VolSize not be a uint64? Or initialize it to -1?
 	ri.VolSize = int64(imageInfo.Size)
 
+	// If the luks2HeaderSizeKey metadata is set
+	// reduce the extra size of the LUKS header from the image size.
+	headerSize, err := ri.getLuks2HeaderSizeSet()
+	if err != nil {
+		return err
+	}
+	ri.VolSize -= int64(headerSize)
+
 	features, err := image.GetFeatures()
 	if err != nil {
 		return err
@@ -1869,7 +1905,17 @@ func (ri *rbdImage) resize(newSize int64) error {
 	}
 	defer image.Close()
 
-	err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB))
+	size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB)
+
+	// If the luks2HeaderSizeKey metadata is set
+	// add the extra size of the LUKS header to the image size.
+	headerSize, err := ri.getLuks2HeaderSizeSet()
+	if err != nil {
+		return err
+	}
+	size += headerSize
+
+	err = image.Resize(size)
 	if err != nil {
 		return err
 	}

internal/util/cryptsetup.go

@@ -26,10 +26,17 @@ import (
 
 	"github.com/ceph/ceph-csi/internal/util/file"
 	"github.com/ceph/ceph-csi/internal/util/log"
+
+	"k8s.io/cloud-provider/volume/helpers"
 )
 
 // Limit memory used by Argon2i PBKDF to 32 MiB.
-const cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB
+const (
+	cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB
+	luks2MetadataSize          = 32 << 7  // 4096 KiB
+	luks2KeySlotsSize          = 32 << 8  // 8192 KiB
+	Luks2HeaderSize            = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB))
+)
 
 // LuksFormat sets up volume as an encrypted LUKS partition.
 func LuksFormat(devicePath, passphrase string) (string, string, error) {
@@ -41,6 +48,10 @@ func LuksFormat(devicePath, passphrase string) (string, string, error) {
 		"luks2",
 		"--hash",
 		"sha256",
+		"--luks2-metadata-size",
+		strconv.Itoa(luks2MetadataSize)+"k",
+		"--luks2-keyslots-size",
+		strconv.Itoa(luks2KeySlotsSize)+"k",
 		"--pbkdf-memory",
 		strconv.Itoa(cryptsetupPBKDFMemoryLimit),
 		devicePath,