rbd: implement pv key rotation

This patch implements the EncryptionKeyRotation spec for ceph-csi

Signed-off-by: Niraj Yadav <niryadav@redhat.com>

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	github.com/ceph/go-ceph v0.28.0
 	github.com/container-storage-interface/spec v1.10.0
-	github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67
+	github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65
 	github.com/gemalto/kmip-go v0.0.10
 	github.com/golang/protobuf v1.5.4
 	github.com/google/fscrypt v0.3.6-0.20240502174735-068b9f8f5dec

go.sum

@@ -911,8 +911,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67 h1:UAcAhE1pTkWaFBS0kvhHUcUsoEv5fsieD0tl8psQMCs=
-github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
+github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65 h1:i9JGGQTEmRQXSpQQPR96+DV4D4o+V1+gjAWf+bpxQxk=
+github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=

internal/csi-addons/rbd/encryptionkeyrotation.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"errors"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/rbd"
+	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	ekr "github.com/csi-addons/spec/lib/go/encryptionkeyrotation"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type EncryptionKeyRotationServer struct {
+	*ekr.UnimplementedEncryptionKeyRotationControllerServer
+	volLock *util.VolumeLocks
+}
+
+func NewEncryptionKeyRotationServer(volLock *util.VolumeLocks) *EncryptionKeyRotationServer {
+	return &EncryptionKeyRotationServer{volLock: volLock}
+}
+
+func (ekrs *EncryptionKeyRotationServer) RegisterService(svc grpc.ServiceRegistrar) {
+	ekr.RegisterEncryptionKeyRotationControllerServer(svc, ekrs)
+}
+
+func (ekrs *EncryptionKeyRotationServer) EncryptionKeyRotate(
+	ctx context.Context,
+	req *ekr.EncryptionKeyRotateRequest,
+) (*ekr.EncryptionKeyRotateResponse, error) {
+	// Get the volume ID from the request
+	volID := req.GetVolumeId()
+	if volID == "" {
+		return nil, status.Error(codes.InvalidArgument, "empty volume ID in request")
+	}
+
+	// Block key rotation for RWX/ROX volumes
+	_, isMultiNode := csicommon.IsBlockMultiNode([]*csi.VolumeCapability{req.GetVolumeCapability()})
+	if isMultiNode {
+		return nil, status.Error(codes.Unimplemented, "multi-node key rotation is not supported")
+	}
+
+	if acquired := ekrs.volLock.TryAcquire(volID); !acquired {
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ekrs.volLock.Release(volID)
+
+	// Get the credentials required to authenticate
+	// against a ceph cluster
+	creds, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer creds.DeleteCredentials()
+
+	rbdVol, err := rbd.GenVolFromVolID(ctx, volID, creds, req.GetSecrets())
+	if err != nil {
+		switch {
+		case errors.Is(err, rbd.ErrImageNotFound):
+			err = status.Errorf(codes.NotFound, "volume ID %s not found", volID)
+		case errors.Is(err, util.ErrPoolNotFound):
+			log.ErrorLog(ctx, "failed to get backend volume for %s: %v", volID, err)
+			err = status.Errorf(codes.NotFound, err.Error())
+		default:
+			err = status.Errorf(codes.Internal, err.Error())
+		}
+
+		return nil, err
+	}
+	defer rbdVol.Destroy(ctx)
+
+	err = rbdVol.RotateEncryptionKey(ctx)
+	if err != nil {
+		return nil, status.Errorf(
+			codes.Internal, "failed to rotate the key for volume with ID %q: %s", volID, err.Error())
+	}
+
+	// Success
+	return &ekr.EncryptionKeyRotateResponse{}, nil
+}

internal/csi-addons/rbd/identity.go

@@ -133,6 +133,13 @@ func (is *IdentityServer) GetCapabilities(
 						Type: identity.Capability_ReclaimSpace_ONLINE,
 					},
 				},
+			},
+			&identity.Capability{
+				Type: &identity.Capability_EncryptionKeyRotation_{
+					EncryptionKeyRotation: &identity.Capability_EncryptionKeyRotation{
+						Type: identity.Capability_EncryptionKeyRotation_ENCRYPTIONKEYROTATION,
+					},
+				},
 			})
 	}
 

internal/kms/vault.go

@@ -28,6 +28,8 @@ import (
 	"github.com/hashicorp/vault/api"
 	loss "github.com/libopenstorage/secrets"
 	"github.com/libopenstorage/secrets/vault"
+
+	"github.com/ceph/ceph-csi/internal/util/file"
 )
 
 const (
@@ -269,10 +271,12 @@ func (vc *vaultConnection) initCertificates(config map[string]interface{}, secre
 			return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
 		}
 
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
+		tf, err := file.CreateTempFile("vault-ca-cert", caPEM)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
 		}
+		vaultConfig[api.EnvVaultCACert] = tf.Name()
+
 		// update the existing config
 		for key, value := range vaultConfig {
 			vc.vaultConfig[key] = value
@@ -480,31 +484,3 @@ func detectAuthMountPath(path string) (string, error) {
 
 	return authMountPath, nil
 }
-
-// createTempFile writes data to a temporary file that contains the pattern in
-// the filename (see os.CreateTemp for details).
-func createTempFile(pattern string, data []byte) (string, error) {
-	t, err := os.CreateTemp("", pattern)
-	if err != nil {
-		return "", fmt.Errorf("failed to create temporary file: %w", err)
-	}
-
-	// delete the tmpfile on error
-	defer func() {
-		if err != nil {
-			// ignore error on failure to remove tmpfile (gosec complains)
-			_ = os.Remove(t.Name())
-		}
-	}()
-
-	s, err := t.Write(data)
-	if err != nil || s != len(data) {
-		return "", fmt.Errorf("failed to write temporary file: %w", err)
-	}
-	err = t.Close()
-	if err != nil {
-		return "", fmt.Errorf("failed to close temporary file: %w", err)
-	}
-
-	return t.Name(), nil
-}

internal/kms/vault_test.go

@@ -18,7 +18,6 @@ package kms
 
 import (
 	"errors"
-	"os"
 	"testing"
 
 	loss "github.com/libopenstorage/secrets"
@@ -44,23 +43,6 @@ func TestDetectAuthMountPath(t *testing.T) {
 	}
 }
 
-func TestCreateTempFile(t *testing.T) {
-	t.Parallel()
-	data := []byte("Hello World!")
-	tmpfile, err := createTempFile("my-file", data)
-	if err != nil {
-		t.Errorf("createTempFile() failed: %s", err)
-	}
-	if tmpfile == "" {
-		t.Errorf("createTempFile() returned an empty filename")
-	}
-
-	err = os.Remove(tmpfile)
-	if err != nil {
-		t.Errorf("failed to remove tmpfile (%s): %s", tmpfile, err)
-	}
-}
-
 func TestSetConfigString(t *testing.T) {
 	t.Parallel()
 	const defaultValue = "default-value"

internal/kms/vault_tokens.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ceph/ceph-csi/internal/util/file"
 	"github.com/ceph/ceph-csi/internal/util/k8s"
 
 	"github.com/hashicorp/vault/api"
@@ -378,10 +379,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault CA: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultCACert] = cer.Name()
 	}
 
 	vaultClientCertFromSecret := "" // optional
@@ -403,10 +405,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultClientCert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultClientCert] = cer.Name()
 	}
 
 	vaultClientCertKeyFromSecret := "" // optional
@@ -432,10 +435,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultCAFromSecret, err)
 			}
 		}
-		vaultConfig[api.EnvVaultClientKey], err = createTempFile("vault-client-cert-key", []byte(certKey))
+		ckey, err := file.CreateTempFile("vault-client-cert-key", certKey)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault client cert key: %w", err)
 		}
+		vaultConfig[api.EnvVaultClientKey] = ckey.Name()
 	}
 
 	for key, value := range vaultConfig {

internal/rbd/driver/driver.go

@@ -229,6 +229,9 @@ func (r *Driver) setupCSIAddonsServer(conf *util.Config) error {
 	if conf.IsNodeServer {
 		rs := casrbd.NewReclaimSpaceNodeServer()
 		r.cas.RegisterService(rs)
+
+		ekr := casrbd.NewEncryptionKeyRotationServer(r.ns.VolumeLocks)
+		r.cas.RegisterService(ekr)
 	}
 
 	// start the server, this does not block, it runs a new go-routine

internal/rbd/encryption.go

@@ -63,6 +63,10 @@ const (
 	// user did not specify an "encryptionType", but set
 	// "encryption": true.
 	rbdDefaultEncryptionType = util.EncryptionTypeBlock
+
+	// Luks slots.
+	luksSlot0 = "0"
+	luksSlot1 = "1"
 )
 
 // checkRbdImageEncrypted verifies if rbd image was encrypted when created.
@@ -437,3 +441,72 @@ func (ri *rbdImage) RemoveDEK(ctx context.Context, volumeID string) error {
 
 	return nil
 }
+
+// GetEncryptionPassphraseSize returns the value of `encryptionPassphraseSize`.
+func GetEncryptionPassphraseSize() int {
+	return encryptionPassphraseSize
+}
+
+// RotateEncryptionKey processes the key rotation for the RBD Volume.
+func (rv *rbdVolume) RotateEncryptionKey(ctx context.Context) error {
+	if !rv.isBlockEncrypted() {
+		return errors.New("key rotation unsupported for non block encrypted device")
+	}
+
+	// Verify that the underlying device has been setup for encryption
+	currState, err := rv.checkRbdImageEncrypted(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to check encryption state: %w", err)
+	}
+
+	if currState != rbdImageEncrypted {
+		return errors.New("key rotation not supported for unencrypted device")
+	}
+
+	// Get the device path for the underlying image
+	useNbd := rv.Mounter == rbdNbdMounter && hasNBD
+	devicePath, found := waitForPath(ctx, rv.Pool, rv.RadosNamespace, rv.RbdImageName, 1, useNbd)
+	if !found {
+		return fmt.Errorf("failed to get the device path for %q: %w", rv, err)
+	}
+
+	// Step 1: Get the current passphrase
+	oldPassphrase, err := rv.blockEncryption.GetCryptoPassphrase(ctx, rv.VolID)
+	if err != nil {
+		return fmt.Errorf("failed to fetch the current passphrase for %q: %w", rv, err)
+	}
+
+	// Step 2: Add current key to slot 1
+	err = util.LuksAddKey(devicePath, oldPassphrase, oldPassphrase, luksSlot1)
+	if err != nil {
+		return fmt.Errorf("failed to add curr key to luksSlot1: %w", err)
+	}
+
+	// Step 3: Generate new key and add it to slot 0
+	newPassphrase, err := rv.blockEncryption.GetNewCryptoPassphrase(
+		GetEncryptionPassphraseSize())
+	if err != nil {
+		return fmt.Errorf("failed to generate a new passphrase: %w", err)
+	}
+
+	err = util.LuksAddKey(devicePath, oldPassphrase, newPassphrase, luksSlot0)
+	if err != nil {
+		return fmt.Errorf("failed to add the new key to luksSlot0: %w", err)
+	}
+
+	// Step 4: Add the new key to KMS
+	err = rv.blockEncryption.StoreCryptoPassphrase(ctx, rv.VolID, newPassphrase)
+	if err != nil {
+		return fmt.Errorf("failed to update the new key into the KMS: %w", err)
+	}
+
+	// Step 5: Remove the old key from slot 1
+	// We use the newPassphrase to authenticate LUKS here
+	err = util.LuksRemoveKey(devicePath, newPassphrase, luksSlot1)
+	if err != nil {
+		return fmt.Errorf("failed to remove the backup key from luksSlot1: %w", err)
+	}
+
+	// Return error accordingly.
+	return nil
+}

internal/util/crypto.go

@@ -237,6 +237,11 @@ func (ve *VolumeEncryption) GetCryptoPassphrase(ctx context.Context, volumeID st
 	return ve.KMS.DecryptDEK(ctx, volumeID, passphrase)
 }
 
+// GetNewCryptoPassphrase returns a random passphrase of given length.
+func (ve *VolumeEncryption) GetNewCryptoPassphrase(length int) (string, error) {
+	return generateNewEncryptionPassphrase(length)
+}
+
 // generateNewEncryptionPassphrase generates a random passphrase for encryption.
 func generateNewEncryptionPassphrase(length int) (string, error) {
 	bytesPassphrase := make([]byte, length)