support encrypting backup data (with AES-GCM) with flag --encryption-config-path

example/01-encryption-config.json

@@ -0,0 +1,3 @@
+{
+  "key": "my-encryption-key-123"
+}

go.mod

@@ -34,6 +34,11 @@ require (
 	sigs.k8s.io/controller-runtime v0.8.3
 )
 
+require (
+	github.com/minio/sio v0.3.0
+	golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000
+)
+
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/BurntSushi/toml v0.3.1 // indirect
@@ -148,16 +153,15 @@ require (
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.opencensus.io v0.22.2 // indirect
 	go.uber.org/multierr v1.5.0 // indirect
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
 	golang.org/x/exp v0.0.0-20191227195350-da58074b4299 // indirect
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/mod v0.3.0 // indirect
-	golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
-	golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 // indirect
+	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
-	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

go.sum

@@ -538,6 +538,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus=
+github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -839,6 +841,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -852,6 +855,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000 h1:SL+8VVnkqyshUSz5iNnXtrBQzvFF2SkROm6t5RczFAE=
+golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -908,6 +913,8 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -968,6 +975,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71 h1:ikCpsnYR+Ew0vu99XlDp55lGgDJdIMx3f4a18jfse/s=
 golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -977,6 +986,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

pkg/dataencryption/readercloser.go

@@ -0,0 +1,18 @@
+package dataencryption
+
+import "io"
+
+type readerCloser struct {
+	r io.Reader
+	c io.Closer
+}
+
+func (r *readerCloser) Read(p []byte) (n int, err error) {
+	return r.r.Read(p)
+}
+
+func (r *readerCloser) Close() error {
+	return r.c.Close()
+}
+
+var _ io.ReadCloser = &readerCloser{}

pkg/dataencryption/snapstore.go

@@ -0,0 +1,105 @@
+package dataencryption
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/minio/sio"
+	"golang.org/x/crypto/hkdf"
+
+	"github.com/gardener/etcd-backup-restore/pkg/types"
+)
+
+type decoratedSnapStore struct {
+	snapstore            types.SnapStore
+	encryptionConfigPath string
+}
+
+func (r *decoratedSnapStore) readEncryptionMasterKey() string {
+	if r.encryptionConfigPath == "" {
+		return ""
+	}
+	content, err := os.ReadFile(r.encryptionConfigPath)
+	if err != nil {
+		panic(fmt.Errorf("readEncryptionMasterKey: failed to read %q as %v", r.encryptionConfigPath, err))
+	}
+
+	cfg := types.SnapstoreEncryptionConfig{}
+	err = json.Unmarshal(content, &cfg)
+	if err != nil {
+		panic(fmt.Errorf("readEncryptionMasterKey: failed to unmarshal %q as %v", r.encryptionConfigPath, err))
+	}
+
+	return cfg.Key
+}
+
+func (r *decoratedSnapStore) deriveEncryptionKey(masterKey string, snapshot types.Snapshot) ([32]byte, error) {
+	var key [32]byte
+	nonce := snapshot.SnapName // use snapName as the nonce
+	kdf := hkdf.New(sha256.New, []byte(masterKey), []byte(nonce), nil)
+	if _, err := io.ReadFull(kdf, key[:]); err != nil {
+		return [32]byte{}, err
+	}
+	return key, nil
+}
+
+func (r *decoratedSnapStore) Fetch(snapshot types.Snapshot) (io.ReadCloser, error) {
+	masterKey := r.readEncryptionMasterKey()
+	if len(masterKey) == 0 {
+		return r.snapstore.Fetch(snapshot)
+	}
+
+	key, err := r.deriveEncryptionKey(masterKey, snapshot)
+	if err != nil {
+		return nil, fmt.Errorf("deriveEncryptionKey failed as %v", err)
+	}
+
+	originalEncryptedDataReader, err := r.snapstore.Fetch(snapshot)
+	decryptedDataReader, err := sio.DecryptReader(originalEncryptedDataReader, sio.Config{
+		Key:          key[:],
+		CipherSuites: []byte{sio.AES_256_GCM},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("sio.DecryptReader failed as %v", err)
+	}
+
+	return &readerCloser{r: decryptedDataReader, c: originalEncryptedDataReader}, err
+}
+
+func (r *decoratedSnapStore) List() (types.SnapList, error) {
+	return r.snapstore.List()
+}
+
+func (r *decoratedSnapStore) Save(snapshot types.Snapshot, originalUnencryptedDataReader io.ReadCloser) error {
+	masterKey := r.readEncryptionMasterKey()
+	if len(masterKey) == 0 {
+		return r.snapstore.Save(snapshot, originalUnencryptedDataReader)
+	}
+
+	key, err := r.deriveEncryptionKey(masterKey, snapshot)
+	if err != nil {
+		return fmt.Errorf("deriveEncryptionKey failed as %v", err)
+	}
+
+	encryptedDataReader, err := sio.EncryptReader(originalUnencryptedDataReader, sio.Config{
+		Key:          key[:],
+		CipherSuites: []byte{sio.AES_256_GCM},
+	})
+	if err != nil {
+		return fmt.Errorf("sio.EncryptReader failed as %v", err)
+	}
+
+	return r.snapstore.Save(snapshot, &readerCloser{r: encryptedDataReader, c: originalUnencryptedDataReader})
+}
+
+func (r *decoratedSnapStore) Delete(snapshot types.Snapshot) error {
+	return r.snapstore.Delete(snapshot)
+}
+
+// DecorateSnapStore returns a decorated SnapStore that will apply encryption and decryption automatically if configured to do so
+func DecorateSnapStore(snapstore types.SnapStore, encryptionConfigPath string) types.SnapStore {
+	return &decoratedSnapStore{snapstore: snapstore, encryptionConfigPath: encryptionConfigPath}
+}

pkg/snapstore/utils.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gardener/etcd-backup-restore/pkg/dataencryption"
 	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/sirupsen/logrus"
 )
@@ -71,31 +72,39 @@ func GetSnapstore(config *brtypes.SnapstoreConfig) (brtypes.SnapStore, error) {
 		config.MaxParallelChunkUploads = 5
 	}
 
-	switch config.Provider {
-	case brtypes.SnapstoreProviderLocal, "":
-		if config.Container == "" {
-			config.Container = defaultLocalStore
+	snapstore, err := (func() (brtypes.SnapStore, error) {
+		switch config.Provider {
+		case brtypes.SnapstoreProviderLocal, "":
+			if config.Container == "" {
+				config.Container = defaultLocalStore
+			}
+			return NewLocalSnapStore(path.Join(config.Container, config.Prefix))
+		case brtypes.SnapstoreProviderS3:
+			return NewS3SnapStore(config)
+		case brtypes.SnapstoreProviderABS:
+			return NewABSSnapStore(config)
+		case brtypes.SnapstoreProviderGCS:
+			return NewGCSSnapStore(config)
+		case brtypes.SnapstoreProviderSwift:
+			return NewSwiftSnapStore(config)
+		case brtypes.SnapstoreProviderOSS:
+			return NewOSSSnapStore(config)
+		case brtypes.SnapstoreProviderECS:
+			return NewECSSnapStore(config)
+		case brtypes.SnapstoreProviderOCS:
+			return NewOCSSnapStore(config)
+		case brtypes.SnapstoreProviderFakeFailed:
+			return NewFailedSnapStore(), nil
+		default:
+			return nil, fmt.Errorf("unsupported storage provider : %s", config.Provider)
 		}
-		return NewLocalSnapStore(path.Join(config.Container, config.Prefix))
-	case brtypes.SnapstoreProviderS3:
-		return NewS3SnapStore(config)
-	case brtypes.SnapstoreProviderABS:
-		return NewABSSnapStore(config)
-	case brtypes.SnapstoreProviderGCS:
-		return NewGCSSnapStore(config)
-	case brtypes.SnapstoreProviderSwift:
-		return NewSwiftSnapStore(config)
-	case brtypes.SnapstoreProviderOSS:
-		return NewOSSSnapStore(config)
-	case brtypes.SnapstoreProviderECS:
-		return NewECSSnapStore(config)
-	case brtypes.SnapstoreProviderOCS:
-		return NewOCSSnapStore(config)
-	case brtypes.SnapstoreProviderFakeFailed:
-		return NewFailedSnapStore(), nil
-	default:
-		return nil, fmt.Errorf("unsupported storage provider : %s", config.Provider)
+	})()
+
+	if err != nil {
+		return snapstore, err
 	}
+
+	return dataencryption.DecorateSnapStore(snapstore, config.EncryptionConfigPath), err
 }
 
 // GetEnvVarOrError returns the value of specified environment variable or terminates if it's not defined.

pkg/types/snapstore.go

@@ -172,6 +172,8 @@ type SnapstoreConfig struct {
 	TempDir string `json:"tempDir,omitempty"`
 	// IsSource determines if this SnapStore is the source for a copy operation
 	IsSource bool `json:"isSource,omitempty"`
+	// EncryptionConfigPath is the path to a JSON file containing SnapstoreEncryptionConfig
+	EncryptionConfigPath string `json:"encryptionConfigPath,omitempty"`
 }
 
 // AddFlags adds the flags to flagset.
@@ -190,6 +192,7 @@ func (c *SnapstoreConfig) addFlags(fs *flag.FlagSet, parameterPrefix string) {
 	fs.StringVar(&c.Prefix, parameterPrefix+"store-prefix", c.Prefix, "prefix or directory inside container under which snapstore is created")
 	fs.UintVar(&c.MaxParallelChunkUploads, parameterPrefix+"max-parallel-chunk-uploads", c.MaxParallelChunkUploads, "maximum number of parallel chunk uploads allowed ")
 	fs.StringVar(&c.TempDir, parameterPrefix+"snapstore-temp-directory", c.TempDir, "temporary directory for processing")
+	fs.StringVar(&c.EncryptionConfigPath, parameterPrefix+"encryption-config-path", c.EncryptionConfigPath, "optional path to a JSON file containing SnapstoreEncryptionConfig ( e.g. '{ \"key\": \"my-encryption-key-123\" }' )")
 }
 
 // Validate validates the config.
@@ -219,3 +222,10 @@ func (c *SnapstoreConfig) MergeWith(other *SnapstoreConfig) {
 		c.TempDir = other.TempDir
 	}
 }
+
+// SnapstoreEncryptionConfig structure holds configuration for backup data encryption
+type SnapstoreEncryptionConfig struct {
+	// Key is the key (any string, e.g. "my-encryption-key-123") to be used in symmetric encryption of the backup data.
+	// If `Key` is an empty string, backup data encryption is disabled.
+	Key string `json:"key"`
+}