From 4ae6b023f2f8a16fa0c2c6945fa8ba6240fcfbac Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 28 Mar 2024 17:10:11 +0530 Subject: [PATCH 1/2] rbd: add additional space for encrypted volumes issue: when a block-mode pvc is created with encryption enabled there is some space reserved for the encryption metadata. Which doesn't allows users to write extact amount of data that they have requested for. solution: create pvc with extra space needed for the encryption metadata. `GetLuksHeaderSize()` function returns the luks2 encryption metadata(header size). The extra space is added during the CreateVolume and ExpandVolume operations. And while returning the response remove the extra space so the client/user gets the requested size reported. Signed-off-by: Praveen M --- internal/rbd/rbd_util.go | 24 +++++++++++++++++++++--- internal/util/cryptsetup.go | 13 ++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 7b37a70eb61..5dd476fe4c8 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -443,8 +443,14 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to get IOContext: %w", err) } - err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, - uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options) + size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB) + + // add the extra size of the LUKS header to the image size if block encryption is enabled. + if pOpts.isBlockEncrypted() { + size += util.Luks2HeaderSize + } + + err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options) if err != nil { return fmt.Errorf("failed to create rbd image: %w", err) } @@ -1598,6 +1604,11 @@ func (ri *rbdImage) getImageInfo() error { // TODO: can rv.VolSize not be a uint64? Or initialize it to -1? ri.VolSize = int64(imageInfo.Size) + // remove the extra size of the LUKS header, if the volume is block encrypted. + if ri.isBlockEncrypted() { + ri.VolSize -= int64(util.Luks2HeaderSize) + } + features, err := image.GetFeatures() if err != nil { return err @@ -1846,7 +1857,14 @@ func (ri *rbdImage) resize(newSize int64) error { } defer image.Close() - err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) + size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB) + + // add the extra size of the LUKS header to the image size if block encryption is enabled. + if ri.isBlockEncrypted() { + size += util.Luks2HeaderSize + } + + err = image.Resize(size) if err != nil { return err } diff --git a/internal/util/cryptsetup.go b/internal/util/cryptsetup.go index e5669b42533..1db2163db36 100644 --- a/internal/util/cryptsetup.go +++ b/internal/util/cryptsetup.go @@ -22,10 +22,17 @@ import ( "os/exec" "strconv" "strings" + + "k8s.io/cloud-provider/volume/helpers" ) // Limit memory used by Argon2i PBKDF to 32 MiB. -const cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB +const ( + cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB + luks2MetadataSize = 32 << 7 // 4096 KiB + luks2KeySlotsSize = 32 << 8 // 8192 KiB + Luks2HeaderSize = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB)) +) // LuksFormat sets up volume as an encrypted LUKS partition. func LuksFormat(devicePath, passphrase string) (string, string, error) { @@ -37,6 +44,10 @@ func LuksFormat(devicePath, passphrase string) (string, string, error) { "luks2", "--hash", "sha256", + "--luks2-metadata-size", + strconv.Itoa(luks2MetadataSize)+"k", + "--luks2-keyslots-size", + strconv.Itoa(luks2KeySlotsSize)+"k", "--pbkdf-memory", strconv.Itoa(cryptsetupPBKDFMemoryLimit), devicePath, From bc141c1f75753b0c4af875268be535caaf876229 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Mon, 29 Apr 2024 12:07:43 +0530 Subject: [PATCH 2/2] e2e: test to verify the encrypted image size This commit adds testcase for the create/resize/clone/restore operations for a RBD block PVC and validate imageSize and deviceSize. Signed-off-by: Praveen M --- e2e/rbd.go | 152 ++++++++++++++++++++++++++++++++++++++++++++++ e2e/rbd_helper.go | 20 ++++++ 2 files changed, 172 insertions(+) diff --git a/e2e/rbd.go b/e2e/rbd.go index 1710156973a..96560ba5a25 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -27,10 +27,12 @@ import ( . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/cloud-provider/volume/helpers" "k8s.io/kubernetes/test/e2e/framework" e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -1984,6 +1986,156 @@ var _ = Describe("RBD", func() { } }) + By("create/resize/clone/restore a encrypted block pvc and verify the image size", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"encrypted": "true", "encryptionType": util.EncryptionTypeBlock.String()}, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + + err = createRBDSnapshotClass(f) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + framework.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + var ( + imageSize uint64 + resizeImageSize uint64 + sizeInBytes int64 + ) + + //nolint:goconst // The string "1Gi" is used multiple times in rbd.go, so it's not a const value. + pvcSize := "1Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(pvcSize)); err != nil { + framework.Failf("failed to parse pvc size: %v", err) + } + imageSize = uint64(sizeInBytes) + util.Luks2HeaderSize + + pvc, err := loadPVC(rawPvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + + app, err := loadApp(rawAppPath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + labelKey := "app" + labelValue := "rbd-pod-block-encrypted" + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValue), + } + + app.Labels = map[string]string{labelKey: labelValue} + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate created backend rbd images + err = validateImageSize(f, pvc, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(app, f, &opt, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // create clone PVC and validate the image size + labelValueClonePod := "rbd-pod-block-encrypted-clone" + optClonePod := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueClonePod), + } + + pvcClone, err := loadPVC(pvcBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcClone.Namespace = f.UniqueName + + appClone, err := loadApp(appBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Labels = map[string]string{labelKey: labelValueClonePod} + + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + framework.Failf("failed to create clone PVC and application : %v", err) + } + + err = validateImageSize(f, pvcClone, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(appClone, f, &optClonePod, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // resize PVC and validate the image size + resizePVCSize := "2Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(resizePVCSize)); err != nil { + framework.Failf("failed to parse resize pvc size: %v", err) + } + resizeImageSize = uint64(sizeInBytes) + util.Luks2HeaderSize + + err = expandPVCSize(f.ClientSet, pvc, resizePVCSize, deployTimeout) + if err != nil { + framework.Failf("failed to expand pvc size: %v", err) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) + if err != nil { + framework.Failf("timeout waiting for pod to be in running state: %v", err) + } + + err = validateImageSize(f, pvc, resizeImageSize) + if err != nil { + framework.Failf("failed to validate image size after resize: %v", err) + } + err = checkDeviceSize(app, f, &opt, resizePVCSize) + if err != nil { + framework.Failf("failed to validate device size after resize: %v", err) + } + + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + framework.Failf("failed to delete pvc and app: %v", err) + } + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + framework.Failf("failed to delete clone pvc and app: %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + }) + ByFileAndBlockEncryption("create a PVC and bind it to an app using rbd-nbd mounter with encryption", func( validator encryptionValidateFunc, _ validateFunc, encType util.EncryptionType, ) { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 8d86a3ce106..99e4176c2bf 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -1067,6 +1067,7 @@ type imageInfo struct { StripeUnit int `json:"stripe_unit"` StripeCount int `json:"stripe_count"` ObjectSize int `json:"object_size"` + Size uint64 `json:"size"` } // getImageInfo queries rbd about the given image and returns its metadata, and returns @@ -1126,3 +1127,22 @@ func validateStripe(f *framework.Framework, return nil } + +// validateImageSize validates the size of the image. +func validateImageSize(f *framework.Framework, pvc *v1.PersistentVolumeClaim, imageSize uint64) error { + imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) + if err != nil { + return err + } + + imgInfo, err := getImageInfo(f, imageData.imageName, defaultRBDPool) + if err != nil { + return err + } + + if imgInfo.Size != imageSize { + return fmt.Errorf("image %s size %d does not match expected %d", imgInfo.Name, imgInfo.Size, imageSize) + } + + return nil +}