From 70ac68a82f539812e01bc8e03314981f25ba093c Mon Sep 17 00:00:00 2001 From: James Bach Date: Sun, 23 Aug 2020 15:39:47 +0100 Subject: [PATCH 01/13] Adds SSE config to S3 Objstore Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 40 ++++++++++++++++++++++++++++++++++---- pkg/objstore/s3/s3_test.go | 12 ++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index ad05670d0e..fea4b10fd1 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -9,6 +9,7 @@ import ( "crypto/tls" "fmt" "io" + "io/ioutil" "net" "net/http" "os" @@ -53,13 +54,22 @@ type Config struct { AccessKey string `yaml:"access_key"` Insecure bool `yaml:"insecure"` SignatureV2 bool `yaml:"signature_version2"` - SSEEncryption bool `yaml:"encrypt_sse"` SecretKey string `yaml:"secret_key"` PutUserMetadata map[string]string `yaml:"put_user_metadata"` HTTPConfig HTTPConfig `yaml:"http_config"` TraceConfig TraceConfig `yaml:"trace"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. - PartSize uint64 `yaml:"part_size"` + PartSize uint64 `yaml:"part_size"` + SSEConfig SSEConfig `yaml:"sse_config"` +} + +// SSEConfig deals with the configuration of SSE for Minio. The following options are valid: +// kmsencryptioncontext == https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context +type SSEConfig struct { + Enable bool `yaml:"enabled"` + KMSKeyID string `yaml:"kms_key_id"` + KMSEncryptionContext map[string]string `yaml:"kms_encryption_context"` + EncryptionKey string `yaml:"encryption_key"` } type TraceConfig struct { @@ -173,8 +183,26 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B client.SetAppInfo(fmt.Sprintf("thanos-%s", component), fmt.Sprintf("%s (%s)", version.Version, runtime.Version())) var sse encrypt.ServerSide - if config.SSEEncryption { - sse = encrypt.NewSSE() + if config.SSEConfig.Enable { + switch { + case config.SSEConfig.KMSKeyID != "": + sse, err = encrypt.NewSSEKMS(config.SSEConfig.KMSKeyID, config.SSEConfig.KMSEncryptionContext) + if err != nil { + return nil, errors.Wrap(err, "initialize s3 client SSE-KMS") + } + case config.SSEConfig.EncryptionKey != "": + key, err := ioutil.ReadFile(config.SSEConfig.EncryptionKey) + if err != nil { + return nil, err + } + + sse, err = encrypt.NewSSEC(key) + if err != nil { + return nil, errors.Wrap(err, "initialize s3 client SSE-C") + } + default: + sse = encrypt.NewSSE() + } } if config.TraceConfig.Enable { @@ -211,6 +239,10 @@ func validate(conf Config) error { if conf.AccessKey != "" && conf.SecretKey == "" { return errors.New("no s3 secret_key specified while access_key is present in config file; either both should be present in config or envvars/IAM should be used.") } + + if conf.SSEConfig.EncryptionKey != "" && conf.SSEConfig.KMSKeyID != "" { + return errors.New("sse_encryption_key AND sse_kms_key_id set in sse_config. You can set one or the other, but NOT both.") + } return nil } diff --git a/pkg/objstore/s3/s3_test.go b/pkg/objstore/s3/s3_test.go index 3ef3ea19e2..22bf36f241 100644 --- a/pkg/objstore/s3/s3_test.go +++ b/pkg/objstore/s3/s3_test.go @@ -24,6 +24,18 @@ insecure: false`) } } +func TestParseConfig_SSEConfig(t *testing.T) { + input := []byte(`sse_config: + enabled: true`) + + cfg, err := parseConfig(input) + testutil.Ok(t, err) + + if !cfg.SSEConfig.Enable { + t.Errorf("parsing of sse_config failed: got %v, expected %v", cfg.SSEConfig.Enable, true) + } +} + func TestParseConfig_DefaultHTTPConfig(t *testing.T) { input := []byte(`bucket: abcd insecure: false`) From 4b70610a030ea40db92c213fc8ad6fa858c8ca48 Mon Sep 17 00:00:00 2001 From: James Bach Date: Sun, 23 Aug 2020 15:59:21 +0100 Subject: [PATCH 02/13] Document sse_config block in storage.md Signed-off-by: James Bach --- docs/storage.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/storage.md b/docs/storage.md index 26dac3509b..c71fa23013 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -86,6 +86,11 @@ config: encrypt_sse: false secret_key: "" put_user_metadata: {} + sse_config: + enabled: false + kms_key_id: "" + kms_encryption_context: {} + encryption_key: "" http_config: idle_conn_timeout: 1m30s response_header_timeout: 2m @@ -115,6 +120,18 @@ For debug and testing purposes you can set * `trace.enable: true` to enable the minio client's verbose logging. Each request and response will be logged into the debug logger, so debug level logging must be enabled for this functionality. +#### S3 Server-Side Encryption + +SSE can be configued using the `sse_config`. [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html), [SSE-KMS](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html), and [SSE-C](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html) are supported. + +The following combinations are allowed: +* If `enabled` is set to `true` but nothing else is set, we default to using [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html). +* If `kms_key_id` is set, [SSE-KMS](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) is used, and the objects that Thanos uploads will be encrypted using that key. +* If `kms_encryption_context` is set with `kms_key_id`, you will add an [encryption context](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context) that provides a layer of integrity checks. Note that you do not have to set this as AWS will provide a default one for you. +* If `encryption_key` is set, [SSE-C](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html) is set up using the key provided. + +Thanos will throw an error if `encryption_key` AND `kms_key_id` is set. + #### Credentials By default Thanos will try to retrieve credentials from the following sources: From 666be0890b88b4f585785a9ce4edf34a9bf7cd05 Mon Sep 17 00:00:00 2001 From: James Bach Date: Sun, 23 Aug 2020 15:24:13 +0000 Subject: [PATCH 03/13] run make docs Signed-off-by: James Bach --- docs/storage.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index c71fa23013..a33b3e49b5 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -83,14 +83,8 @@ config: access_key: "" insecure: false signature_version2: false - encrypt_sse: false secret_key: "" put_user_metadata: {} - sse_config: - enabled: false - kms_key_id: "" - kms_encryption_context: {} - encryption_key: "" http_config: idle_conn_timeout: 1m30s response_header_timeout: 2m @@ -98,6 +92,11 @@ config: trace: enable: false part_size: 134217728 + sse_config: + enabled: false + kms_key_id: "" + kms_encryption_context: {} + encryption_key: "" ``` At a minimum, you will need to provide a value for the `bucket`, `endpoint`, `access_key`, and `secret_key` keys. The rest of the keys are optional. From 16c2832e18e94554ec4a8d28f663b3e1780b5eae Mon Sep 17 00:00:00 2001 From: James Bach Date: Sun, 23 Aug 2020 16:30:07 +0100 Subject: [PATCH 04/13] Add SSE config options to changelog Signed-off-by: James Bach --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81f5002132..558f2eca5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2976](https://github.com/thanos-io/thanos/pull/2976) Query: Better rounding for incoming query timestamps. - [#2929](https://github.com/thanos-io/thanos/pull/2929) Mixin: Fix expression for 'unhealthy sidecar' alert and also increase the timeout for 10 minutes. - [#3024](https://github.com/thanos-io/thanos/pull/3024) Query: consider group name and file for deduplication +- [#3064](https://github.com/thanos-io/thanos/pull/3064) s3: Add SSE/SSE-KMS/SSE-C configuration ### Added From 104c7a4cb1c58f74c895379a02760bbe8652c6f0 Mon Sep 17 00:00:00 2001 From: James Bach Date: Sun, 23 Aug 2020 16:22:13 +0000 Subject: [PATCH 05/13] Update changelog Signed-off-by: James Bach --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 558f2eca5c..cae9cf71fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2976](https://github.com/thanos-io/thanos/pull/2976) Query: Better rounding for incoming query timestamps. - [#2929](https://github.com/thanos-io/thanos/pull/2929) Mixin: Fix expression for 'unhealthy sidecar' alert and also increase the timeout for 10 minutes. - [#3024](https://github.com/thanos-io/thanos/pull/3024) Query: consider group name and file for deduplication -- [#3064](https://github.com/thanos-io/thanos/pull/3064) s3: Add SSE/SSE-KMS/SSE-C configuration +- [#3064](https://github.com/thanos-io/thanos/pull/3064) s3: Add SSE/SSE-KMS/SSE-C configuration ### Added From 9608dc1e7e59713a41745bcdf1768979db21bcfb Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 09:10:26 +0100 Subject: [PATCH 06/13] Define explicit SSE 'Type' Signed-off-by: James Bach --- CHANGELOG.md | 12 +++++- docs/storage.md | 14 +++---- pkg/objstore/s3/s3.go | 25 ++++++++---- pkg/objstore/s3/s3_test.go | 83 +++++++++++++++++++++++++++++++++++--- 4 files changed, 114 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cae9cf71fe..0c80b412e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,16 @@ We use *breaking* word for marking changes that are not backward compatible (rel :warning: **WARNING** :warning: Thanos Rule's `/api/v1/rules` endpoint no longer returns the old, deprecated `partial_response_strategy`. The old, deprecated value has been fixed to `WARN` for quite some time. _Please_ use `partialResponseStrategy`. +:warning: **WARNING** :warning: The `sse_encryption` value is now deprecated in favour of `sse_config`. If you used `sse_encryption`, the migration strategy is to set up the following block: + +```yaml + +--- +sse_config: + type: SSE-S3 +``` + + ### Fixed - [#2937](https://github.com/thanos-io/thanos/pull/2937) Receive: Fixing auto-configuration of --receive.local-endpoint @@ -29,7 +39,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2976](https://github.com/thanos-io/thanos/pull/2976) Query: Better rounding for incoming query timestamps. - [#2929](https://github.com/thanos-io/thanos/pull/2929) Mixin: Fix expression for 'unhealthy sidecar' alert and also increase the timeout for 10 minutes. - [#3024](https://github.com/thanos-io/thanos/pull/3024) Query: consider group name and file for deduplication -- [#3064](https://github.com/thanos-io/thanos/pull/3064) s3: Add SSE/SSE-KMS/SSE-C configuration +- [#3064](https://github.com/thanos-io/thanos/pull/3064) s3: Add SSE/SSE-KMS/SSE-C configuration. ### Added diff --git a/docs/storage.md b/docs/storage.md index a33b3e49b5..bafbff24e7 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -93,7 +93,7 @@ config: enable: false part_size: 134217728 sse_config: - enabled: false + type: SSE-S3|SSE-KMS|SSE-C kms_key_id: "" kms_encryption_context: {} encryption_key: "" @@ -123,13 +123,13 @@ For debug and testing purposes you can set SSE can be configued using the `sse_config`. [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html), [SSE-KMS](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html), and [SSE-C](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html) are supported. -The following combinations are allowed: -* If `enabled` is set to `true` but nothing else is set, we default to using [SSE-S3](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html). -* If `kms_key_id` is set, [SSE-KMS](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html) is used, and the objects that Thanos uploads will be encrypted using that key. -* If `kms_encryption_context` is set with `kms_key_id`, you will add an [encryption context](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context) that provides a layer of integrity checks. Note that you do not have to set this as AWS will provide a default one for you. -* If `encryption_key` is set, [SSE-C](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html) is set up using the key provided. +* If type is set to `SSE-S3` you do not need to configure other options. -Thanos will throw an error if `encryption_key` AND `kms_key_id` is set. +* If type is set to `SSE-KMS` you must set `kms_key_id`. The `kms_encryption_context` is optional, as [AWS provides a default encryption context](https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context). + +* If type is set to `SSE-C` you must provide a path to the encryption key using `encryption_key`. + +If the SSE Config block is set but the `type` is not one of `SSE-S3`, `SSE-KMS`, or `SSE-C`, an error is raised. #### Credentials diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index fea4b10fd1..783ffe2e42 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -66,7 +66,7 @@ type Config struct { // SSEConfig deals with the configuration of SSE for Minio. The following options are valid: // kmsencryptioncontext == https://docs.aws.amazon.com/kms/latest/developerguide/services-s3.html#s3-encryption-context type SSEConfig struct { - Enable bool `yaml:"enabled"` + Type string `yaml:"type"` KMSKeyID string `yaml:"kms_key_id"` KMSEncryptionContext map[string]string `yaml:"kms_encryption_context"` EncryptionKey string `yaml:"encryption_key"` @@ -183,14 +183,15 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B client.SetAppInfo(fmt.Sprintf("thanos-%s", component), fmt.Sprintf("%s (%s)", version.Version, runtime.Version())) var sse encrypt.ServerSide - if config.SSEConfig.Enable { + if config.SSEConfig.Type != "" { switch { - case config.SSEConfig.KMSKeyID != "": + case config.SSEConfig.Type != "SSE-KMS": sse, err = encrypt.NewSSEKMS(config.SSEConfig.KMSKeyID, config.SSEConfig.KMSEncryptionContext) if err != nil { return nil, errors.Wrap(err, "initialize s3 client SSE-KMS") } - case config.SSEConfig.EncryptionKey != "": + + case config.SSEConfig.Type == "SSE-C": key, err := ioutil.ReadFile(config.SSEConfig.EncryptionKey) if err != nil { return nil, err @@ -200,8 +201,13 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B if err != nil { return nil, errors.Wrap(err, "initialize s3 client SSE-C") } - default: + + case config.SSEConfig.Type == "SSE-S3": sse = encrypt.NewSSE() + + default: + sseErrMsg := errors.New("A type of SSE-S3, SSE-C, or SSE-KMS was not provided in sse_config") + return nil, errors.Wrap(sseErrMsg, "Initialize s3 client SSE Config") } } @@ -240,9 +246,14 @@ func validate(conf Config) error { return errors.New("no s3 secret_key specified while access_key is present in config file; either both should be present in config or envvars/IAM should be used.") } - if conf.SSEConfig.EncryptionKey != "" && conf.SSEConfig.KMSKeyID != "" { - return errors.New("sse_encryption_key AND sse_kms_key_id set in sse_config. You can set one or the other, but NOT both.") + if conf.SSEConfig.Type == "SSE-C" && conf.SSEConfig.EncryptionKey == "" { + return errors.New("encryption_key must be set if sse_config.type is set to 'SSE-C'") + } + + if conf.SSEConfig.Type == "SSE-KMS" && conf.SSEConfig.KMSKeyID == "" { + return errors.New("kms_key_id must be set if sse_config.type is set to 'SSE-KMS'") } + return nil } diff --git a/pkg/objstore/s3/s3_test.go b/pkg/objstore/s3/s3_test.go index 22bf36f241..8ce2595bc0 100644 --- a/pkg/objstore/s3/s3_test.go +++ b/pkg/objstore/s3/s3_test.go @@ -25,15 +25,88 @@ insecure: false`) } func TestParseConfig_SSEConfig(t *testing.T) { - input := []byte(`sse_config: - enabled: true`) + input := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-S3`) cfg, err := parseConfig(input) testutil.Ok(t, err) + testutil.Ok(t, validate(cfg)) - if !cfg.SSEConfig.Enable { - t.Errorf("parsing of sse_config failed: got %v, expected %v", cfg.SSEConfig.Enable, true) - } + input2 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-C`) + + cfg, err = parseConfig(input2) + testutil.Ok(t, err) + testutil.NotOk(t, validate(cfg)) + + input3 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-C + kms_key_id: qweasd`) + + cfg, err = parseConfig(input3) + testutil.Ok(t, err) + testutil.NotOk(t, validate(cfg)) + + input4 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-C + encryption_key: /some/file`) + + cfg, err = parseConfig(input4) + testutil.Ok(t, err) + testutil.Ok(t, validate(cfg)) + + input5 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-KMS`) + + cfg, err = parseConfig(input5) + testutil.Ok(t, err) + testutil.NotOk(t, validate(cfg)) + + input6 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-KMS + kms_key_id: abcd1234-ab12-cd34-1234567890ab`) + + cfg, err = parseConfig(input6) + testutil.Ok(t, err) + testutil.Ok(t, validate(cfg)) + + input7 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-KMS + kms_key_id: abcd1234-ab12-cd34-1234567890ab + kms_encryption_context: + key: value + something: else + a: b`) + + cfg, err = parseConfig(input7) + testutil.Ok(t, err) + testutil.Ok(t, validate(cfg)) + + input8 := []byte(`bucket: abdd +endpoint: "s3-endpoint" +sse_config: + type: SSE-MagicKey + kms_key_id: abcd1234-ab12-cd34-1234567890ab + encryption_key: /some/file`) + + cfg, err = parseConfig(input8) + testutil.Ok(t, err) + // Since the error handling for "proper type" if done as we're setting up the bucket. + testutil.Ok(t, validate(cfg)) } func TestParseConfig_DefaultHTTPConfig(t *testing.T) { From 370466e1a188d9f8f4894230a499ce5eb23558bb Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 08:17:17 +0000 Subject: [PATCH 07/13] Update docs again for real Signed-off-by: James Bach --- docs/storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage.md b/docs/storage.md index bafbff24e7..329c0f8486 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -93,7 +93,7 @@ config: enable: false part_size: 134217728 sse_config: - type: SSE-S3|SSE-KMS|SSE-C + type: "" kms_key_id: "" kms_encryption_context: {} encryption_key: "" From e7a54087132a925c6b40e3ab7b262b23f73fe08e Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 10:56:53 +0100 Subject: [PATCH 08/13] Fix bad logic for SSE-KMS Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 783ffe2e42..069be1e388 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -185,7 +185,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B var sse encrypt.ServerSide if config.SSEConfig.Type != "" { switch { - case config.SSEConfig.Type != "SSE-KMS": + case config.SSEConfig.Type == "SSE-KMS": sse, err = encrypt.NewSSEKMS(config.SSEConfig.KMSKeyID, config.SSEConfig.KMSEncryptionContext) if err != nil { return nil, errors.Wrap(err, "initialize s3 client SSE-KMS") From 81d03e8bab3b7562ccf4ff0b4b96a03c99e4f8ae Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 12:45:58 +0000 Subject: [PATCH 09/13] Adds example policy for KMS documentation Signed-off-by: James Bach --- docs/storage.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/storage.md b/docs/storage.md index 329c0f8486..715fb4212c 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -131,6 +131,26 @@ SSE can be configued using the `sse_config`. [SSE-S3](https://docs.aws.amazon.co If the SSE Config block is set but the `type` is not one of `SSE-S3`, `SSE-KMS`, or `SSE-C`, an error is raised. +You will also need to apply the following AWS IAM policy for the user to access the KMS key: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "KMSAccess", + "Effect": "Allow", + "Action": [ + "kms:GenerateDataKey", + "kms:Encrypt", + "kms:Decrypt" + ], + "Resource": "arn:aws:kms:::key/" + } + ] +} +``` + #### Credentials By default Thanos will try to retrieve credentials from the following sources: From 1c8efe2a45b05923a28c095ea23f42217956ab5f Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 13:53:27 +0100 Subject: [PATCH 10/13] Changes SSE strings into constants Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 069be1e388..2f1efda094 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -35,6 +35,10 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +const SSEKMS = "SSE-KMS" +const SSEC = "SSE-C" +const SSES3 = "SSE-S3" + var DefaultConfig = Config{ PutUserMetadata: map[string]string{}, HTTPConfig: HTTPConfig{ @@ -184,14 +188,14 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B var sse encrypt.ServerSide if config.SSEConfig.Type != "" { - switch { - case config.SSEConfig.Type == "SSE-KMS": + switch config.SSEConfig.Type { + case SSEKMS: sse, err = encrypt.NewSSEKMS(config.SSEConfig.KMSKeyID, config.SSEConfig.KMSEncryptionContext) if err != nil { return nil, errors.Wrap(err, "initialize s3 client SSE-KMS") } - case config.SSEConfig.Type == "SSE-C": + case SSEC: key, err := ioutil.ReadFile(config.SSEConfig.EncryptionKey) if err != nil { return nil, err @@ -202,7 +206,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B return nil, errors.Wrap(err, "initialize s3 client SSE-C") } - case config.SSEConfig.Type == "SSE-S3": + case SSES3: sse = encrypt.NewSSE() default: @@ -246,11 +250,11 @@ func validate(conf Config) error { return errors.New("no s3 secret_key specified while access_key is present in config file; either both should be present in config or envvars/IAM should be used.") } - if conf.SSEConfig.Type == "SSE-C" && conf.SSEConfig.EncryptionKey == "" { + if conf.SSEConfig.Type == SSEC && conf.SSEConfig.EncryptionKey == "" { return errors.New("encryption_key must be set if sse_config.type is set to 'SSE-C'") } - if conf.SSEConfig.Type == "SSE-KMS" && conf.SSEConfig.KMSKeyID == "" { + if conf.SSEConfig.Type == SSEKMS && conf.SSEConfig.KMSKeyID == "" { return errors.New("kms_key_id must be set if sse_config.type is set to 'SSE-KMS'") } From 0b2d3a395ba6a86234f8961c16082c6001a62ebe Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 13:54:15 +0100 Subject: [PATCH 11/13] Make SSE error message prettier Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 2f1efda094..fe21d606a8 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -210,7 +210,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B sse = encrypt.NewSSE() default: - sseErrMsg := errors.New("A type of SSE-S3, SSE-C, or SSE-KMS was not provided in sse_config") + sseErrMsg := errors.Errorf("Unsupported type %q was provided. Supported types are SSE-S3, SSE-KMS, SSE-C", config.SSEConfig.Type) return nil, errors.Wrap(sseErrMsg, "Initialize s3 client SSE Config") } } From 9079289460f063a90e1939d22b615a1ff0c074e0 Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 13:56:21 +0100 Subject: [PATCH 12/13] Export consant with comments Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index fe21d606a8..5544ff1657 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -35,8 +35,13 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" +// SSEKMS is the name of the SSE-KMS method for objectstore encryption. const SSEKMS = "SSE-KMS" + +// SSEC is the name of the SSE-C method for objstore encryption. const SSEC = "SSE-C" + +// SSES3 is the name of the SSE-S3 method for objstore encryption. const SSES3 = "SSE-S3" var DefaultConfig = Config{ From 2c52cc2e8132c950d8597029d7dcd8a971e4134a Mon Sep 17 00:00:00 2001 From: James Bach Date: Mon, 24 Aug 2020 14:32:38 +0100 Subject: [PATCH 13/13] Group consts in pkg/s3 Signed-off-by: James Bach --- pkg/objstore/s3/s3.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 5544ff1657..789e629010 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -32,17 +32,19 @@ import ( "gopkg.in/yaml.v2" ) -// DirDelim is the delimiter used to model a directory structure in an object store bucket. -const DirDelim = "/" +const ( + // DirDelim is the delimiter used to model a directory structure in an object store bucket. + DirDelim = "/" -// SSEKMS is the name of the SSE-KMS method for objectstore encryption. -const SSEKMS = "SSE-KMS" + // SSEKMS is the name of the SSE-KMS method for objectstore encryption. + SSEKMS = "SSE-KMS" -// SSEC is the name of the SSE-C method for objstore encryption. -const SSEC = "SSE-C" + // SSEC is the name of the SSE-C method for objstore encryption. + SSEC = "SSE-C" -// SSES3 is the name of the SSE-S3 method for objstore encryption. -const SSES3 = "SSE-S3" + // SSES3 is the name of the SSE-S3 method for objstore encryption. + SSES3 = "SSE-S3" +) var DefaultConfig = Config{ PutUserMetadata: map[string]string{},