From cfea8d756231e7eda5150e33a7f5711cfcf3aafb Mon Sep 17 00:00:00 2001 From: Marcel Lauhoff Date: Fri, 12 Aug 2022 16:30:35 +0200 Subject: [PATCH] fscrypt: fscrypt integration Integrate google/fscrypt into Ceph CSI KMS and encryption setup. Adds dependencies to google/fscrypt and pkg/xattr. Be as generic as possible to support integration with both RBD and Ceph FS. Add the following public functions: InitializeNode: per-node initialization steps. Must be called before Unlock at least once. Unlock: All steps necessary to unlock an encrypted directory including setting it up initially. IsDirectoryUnlocked: Test if directory is really encrypted Signed-off-by: Marcel Lauhoff --- go.mod | 2 + go.sum | 7 + internal/util/fscrypt/fscrypt.go | 382 +++++++++++++++++++++++++++++++ vendor/modules.txt | 14 ++ 4 files changed, 405 insertions(+) create mode 100644 internal/util/fscrypt/fscrypt.go diff --git a/go.mod b/go.mod index e47f1b7d18d..badf4dde544 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/gemalto/kmip-go v0.0.8-0.20220721195433-3fe83e2d3f26 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 + github.com/google/fscrypt v0.3.3 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/vault/api v1.7.2 @@ -23,6 +24,7 @@ require ( github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a github.com/onsi/ginkgo/v2 v2.1.6 github.com/onsi/gomega v1.20.1 + github.com/pkg/xattr v0.4.7 github.com/prometheus/client_golang v1.12.2 github.com/stretchr/testify v1.8.0 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd diff --git a/go.sum b/go.sum index 2552be64479..34b701e0ae0 100644 --- a/go.sum +++ b/go.sum @@ -485,6 +485,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/cadvisor v0.45.0/go.mod h1:vsMT3Uv2XjQ8M7WUtKARV74mU/HN64C4XtM1bJhUKcU= github.com/google/cel-go v0.12.4/go.mod h1:Av7CU6r6X3YmcHR9GXqVDaEJYfEtSxl6wvIjUQTriCw= +github.com/google/fscrypt v0.3.3 h1:qwx9OCR/xZE68VGr/r0/yugFhlGpIOGsH9JHrttP7vc= +github.com/google/fscrypt v0.3.3/go.mod h1:H1JHtH8BVe0dYNhzx1Ztkn3azQ0OBdoOmM828vEWAXc= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -991,6 +993,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/xattr v0.4.7 h1:XoA3KzmFvyPlH4RwX5eMcgtzcaGBaSvgt3IoFQfbrmQ= +github.com/pkg/xattr v0.4.7/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA= @@ -1143,6 +1147,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= @@ -1503,6 +1508,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9w golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210422114643-f5beecf764ed/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1551,6 +1557,7 @@ golang.org/x/tools v0.0.0-20190718200317-82a3ea8a504c/go.mod h1:jcCCGcm9btYwXyDq golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191025023517-2077df36852e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/internal/util/fscrypt/fscrypt.go b/internal/util/fscrypt/fscrypt.go new file mode 100644 index 00000000000..d339258ed3e --- /dev/null +++ b/internal/util/fscrypt/fscrypt.go @@ -0,0 +1,382 @@ +/* +Copyright 2022 The Ceph-CSI Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fscrypt + +/* +#include +*/ +import "C" + +import ( + "context" + "errors" + "fmt" + "os" + "os/user" + "path" + "time" + "unsafe" + + fscryptactions "github.com/google/fscrypt/actions" + fscryptcrypto "github.com/google/fscrypt/crypto" + fscryptfilesystem "github.com/google/fscrypt/filesystem" + fscryptmetadata "github.com/google/fscrypt/metadata" + "github.com/pkg/xattr" + "golang.org/x/sys/unix" + + "github.com/ceph/ceph-csi/internal/kms" + "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" +) + +const ( + FscryptHashingTimeTarget = 1 * time.Second + FscryptProtectorPrefix = "ceph-csi" + FscryptSubdir = "ceph-csi-encrypted" + encryptionPassphraseSize = 64 +) + +func AppendEncyptedSubdirectory(dir string) string { + return path.Join(dir, FscryptSubdir) +} + +// getPassphrase returns the passphrase from the configured Ceph CSI KMS to be used as a protector key in fscrypt. +func getPassphrase(ctx context.Context, encryption util.VolumeEncryption, volID string) (string, error) { + var ( + passphrase string + err error + ) + + switch encryption.KMS.RequiresDEKStore() { + case kms.DEKStoreIntegrated: + passphrase, err = encryption.GetCryptoPassphrase(volID) + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to get passphrase from KMS: %v", err) + + return "", err + } + case kms.DEKStoreMetadata: + passphrase, err = encryption.KMS.GetSecret(volID) + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to GetSecret: %v", err) + + return "", err + } + } + + return passphrase, nil +} + +// createKeyFuncFromVolumeEncryption returns an fscrypt key function returning +// encryption keys form a VolumeEncryption struct. +func createKeyFuncFromVolumeEncryption( + ctx context.Context, + encryption util.VolumeEncryption, + volID string, +) (func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error), error) { + passphrase, err := getPassphrase(ctx, encryption, volID) + if err != nil { + return nil, err + } + + keyFunc := func(info fscryptactions.ProtectorInfo, retry bool) (*fscryptcrypto.Key, error) { + key, err := fscryptcrypto.NewBlankKey(32) + copy(key.Data(), passphrase) + + return key, err + } + + return keyFunc, nil +} + +// unlockExisting tries to unlock an already set up fscrypt directory using keys from Ceph CSI. +func unlockExisting( + ctx context.Context, + fscryptContext *fscryptactions.Context, + encryptedPath string, protectorName string, + keyFn func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error), +) error { + var err error + + policy, err := fscryptactions.GetPolicyFromPath(fscryptContext, encryptedPath) + if err != nil { + log.ErrorLog(ctx, "fscrypt: policy get failed %v", err) + + return err + } + + optionFn := func(policyDescriptor string, options []*fscryptactions.ProtectorOption) (int, error) { + for idx, option := range options { + if option.Name() == protectorName { + return idx, nil + } + } + + return 0, &fscryptactions.ErrNotProtected{PolicyDescriptor: policyDescriptor, ProtectorDescriptor: protectorName} + } + + if err = policy.Unlock(optionFn, keyFn); err != nil { + log.ErrorLog(ctx, "fscrypt: unlock with protector error: %v", err) + + return err + } + + defer func() { + err = policy.Lock() + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to lock policy after use: %v", err) + } + }() + + if err = policy.Provision(); err != nil { + log.ErrorLog(ctx, "fscrypt: provision fail %v", err) + + return err + } + + log.DebugLog(ctx, "fscrypt protector unlock: %s %+v", protectorName, policy) + + return nil +} + +func initializeAndUnlock( + ctx context.Context, + fscryptContext *fscryptactions.Context, + encryptedPath string, protectorName string, + keyFn func(fscryptactions.ProtectorInfo, bool) (*fscryptcrypto.Key, error), +) error { + var owner *user.User + var err error + + if err = os.Mkdir(encryptedPath, 0o755); err != nil { + return err + } + + protector, err := fscryptactions.CreateProtector(fscryptContext, protectorName, keyFn, owner) + if err != nil { + log.ErrorLog(ctx, "fscrypt: protector name=%s create failed: %v. reverting.", protectorName, err) + if revertErr := protector.Revert(); revertErr != nil { + return revertErr + } + + return err + } + + if err = protector.Unlock(keyFn); err != nil { + return err + } + log.DebugLog(ctx, "fscrypt protector unlock: %+v", protector) + + var policy *fscryptactions.Policy + if policy, err = fscryptactions.CreatePolicy(fscryptContext, protector); err != nil { + return err + } + defer func() { + err = policy.Lock() + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to lock policy after init: %w") + err = policy.Revert() + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to revert policy after failed lock: %w") + } + } + }() + + if err = policy.UnlockWithProtector(protector); err != nil { + log.ErrorLog(ctx, "fscrypt: Failed to unlock policy: %v", err) + + return err + } + + if err = policy.Provision(); err != nil { + log.ErrorLog(ctx, "fscrypt: Failed to provision policy: %v", err) + + return err + } + + if err = policy.Apply(encryptedPath); err != nil { + log.ErrorLog(ctx, "fscrypt: Failed to apply protector (see also kernel log): %w", err) + if err = policy.Deprovision(false); err != nil { + log.ErrorLog(ctx, "fscrypt: Policy cleanup response to failing apply failed: %w", err) + } + + return err + } + + return nil +} + +// getInodeEncryptedAttribute returns the inode's encrypt attribute similar to lsattr(1) +func getInodeEncryptedAttribute(p string) (bool, error) { + file, err := os.Open(p) + if err != nil { + return false, err + } + defer file.Close() + + var attr int + _, _, errno := unix.Syscall(unix.SYS_IOCTL, file.Fd(), unix.FS_IOC_GETFLAGS, + uintptr(unsafe.Pointer(&attr))) + if errno != 0 { + return false, fmt.Errorf("error calling ioctl_iflags: %w", errno) + } + + if attr&C.FS_ENCRYPT_FL != 0 { + return true, nil + } + + return false, nil +} + +// IsDirectoryUnlockedFscrypt checks if a directory is an unlocked fscrypted directory. +func IsDirectoryUnlocked(directoryPath, filesystem string) error { + if _, err := fscryptmetadata.GetPolicy(directoryPath); err != nil { + return fmt.Errorf("no fscrypt policy set on directory %q: %w", directoryPath, err) + } + + switch filesystem { + case "ceph": + _, err := xattr.Get(directoryPath, "ceph.fscrypt.auth") + if err != nil { + return fmt.Errorf("error reading ceph.fscrypt.auth xattr on %q: %w", directoryPath, err) + } + default: + encrypted, err := getInodeEncryptedAttribute(directoryPath) + if err != nil { + return err + } + + if !encrypted { + return fmt.Errorf("path %s does not have the encrypted inode flag set. Encryption init must have failed", + directoryPath) + } + } + + return nil +} + +// InitializeNode performs once per nodeserver initialization +// required by the fscrypt library. Creates /etc/fscrypt.conf. +func InitializeNode(ctx context.Context) error { + err := fscryptactions.CreateConfigFile(FscryptHashingTimeTarget, 2) + if err != nil { + existsError := &fscryptactions.ErrConfigFileExists{} + if errors.As(err, &existsError) { + log.ErrorLog(ctx, "fscrypt: config file %q already exists. Skipping fscrypt node setup", + existsError.Path) + + return nil + } + + return fmt.Errorf("fscrypt node init failed to create node configuration (/etc/fscrypt.conf): %w", + err) + } + + return nil +} + +// FscryptUnlock unlocks possilby creating fresh fscrypt metadata +// iff a volume is encrypted. Otherwise return immediately Calling +// this function requires that InitializeFscrypt ran once on this node. +func Unlock( + ctx context.Context, + volEncryption *util.VolumeEncryption, + stagingTargetPath string, volID string, +) error { + fscryptContext, err := fscryptactions.NewContextFromMountpoint(stagingTargetPath, nil) + if err != nil { + log.ErrorLog(ctx, "fscrypt: failed to create context from mountpoint %v: %w", stagingTargetPath) + + return err + } + + fscryptContext.Config.UseFsKeyringForV1Policies = true + + log.DebugLog(ctx, "fscrypt context: %+v", fscryptContext) + + if err = fscryptContext.Mount.CheckSupport(); err != nil { + log.ErrorLog(ctx, "fscrypt: filesystem mount %s does not support fscrypt", fscryptContext.Mount) + + return err + } + + // A proper set up fscrypy directory requires metadata and a kernel policy: + + // 1. Do we have a metadata directory (.fscrypt) set up? + metadataDirExists := false + if err = fscryptContext.Mount.Setup(0o755); err != nil { + alreadySetupErr := &fscryptfilesystem.ErrAlreadySetup{} + if errors.As(err, &alreadySetupErr) { + log.DebugLog(ctx, "fscrypt: metadata directory %q already set up", alreadySetupErr.Mount.Path) + metadataDirExists = true + } else { + log.ErrorLog(ctx, "fscrypt: mount setup failed: %v", err) + + return err + } + } + + encryptedPath := path.Join(stagingTargetPath, FscryptSubdir) + kernelPolicyExists := false + // 2. Ask the kernel if the directory has an fscrypt policy in place. + if _, err = fscryptmetadata.GetPolicy(encryptedPath); err == nil { // encrypted directory already set up + kernelPolicyExists = true + } + + if metadataDirExists != kernelPolicyExists { + return fmt.Errorf("fscrypt: unsupported state metadata=%t kernel_policy=%t", + metadataDirExists, kernelPolicyExists) + } + + keyFn, err := createKeyFuncFromVolumeEncryption(ctx, *volEncryption, volID) + if err != nil { + log.ErrorLog(ctx, "fscrypt: could not create key function: %v", err) + + return err + } + + protectorName := fmt.Sprintf("%s-%s", FscryptProtectorPrefix, volEncryption.GetID()) + + switch volEncryption.KMS.RequiresDEKStore() { + case kms.DEKStoreMetadata: + // Metadata style KMS use the KMS secret as a custom + // passphrase directly in fscrypt, circumenting key + // derivation on the CSI side to allow users to fall + // back on the fscrypt commandline tool easily + fscryptContext.Config.Source = fscryptmetadata.SourceType_custom_passphrase + case kms.DEKStoreIntegrated: + fscryptContext.Config.Source = fscryptmetadata.SourceType_raw_key + } + + if kernelPolicyExists && metadataDirExists { + log.DebugLog(ctx, "fscrypt: Encrypted directory already set up, policy exists") + + return unlockExisting(ctx, fscryptContext, encryptedPath, protectorName, keyFn) + } + + if !kernelPolicyExists && !metadataDirExists { + log.DebugLog(ctx, "fscrypt: Creating new protector and policy") + if volEncryption.KMS.RequiresDEKStore() == kms.DEKStoreIntegrated { + if err := volEncryption.StoreNewCryptoPassphrase(volID, encryptionPassphraseSize); err != nil { + log.ErrorLog(ctx, "fscrypt: store new crypto passphrase failed: %v", err) + + return err + } + } + + return initializeAndUnlock(ctx, fscryptContext, encryptedPath, protectorName, keyFn) + } + + return fmt.Errorf("unsupported") +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0e1d6542ff5..19c46ae61ee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -230,6 +230,15 @@ github.com/golang/protobuf/ptypes/wrappers # github.com/golang/snappy v0.0.4 ## explicit github.com/golang/snappy +# github.com/google/fscrypt v0.3.3 +## explicit; go 1.11 +github.com/google/fscrypt/actions +github.com/google/fscrypt/crypto +github.com/google/fscrypt/filesystem +github.com/google/fscrypt/keyring +github.com/google/fscrypt/metadata +github.com/google/fscrypt/security +github.com/google/fscrypt/util # github.com/google/gnostic v0.5.7-v3refs ## explicit; go 1.12 github.com/google/gnostic/compiler @@ -476,6 +485,9 @@ github.com/pierrec/lz4/internal/xxh32 # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors +# github.com/pkg/xattr v0.4.7 +## explicit; go 1.14 +github.com/pkg/xattr # github.com/pmezard/go-difflib v1.0.0 ## explicit github.com/pmezard/go-difflib/difflib @@ -594,6 +606,7 @@ go.uber.org/zap/internal/exit go.uber.org/zap/zapcore # golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd ## explicit; go 1.17 +golang.org/x/crypto/argon2 golang.org/x/crypto/blake2b golang.org/x/crypto/blowfish golang.org/x/crypto/chacha20 @@ -602,6 +615,7 @@ golang.org/x/crypto/cryptobyte/asn1 golang.org/x/crypto/curve25519 golang.org/x/crypto/curve25519/internal/field golang.org/x/crypto/ed25519 +golang.org/x/crypto/hkdf golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/internal/subtle golang.org/x/crypto/pbkdf2