Add S3 send_content_md5 config and default to use MD5 (#5870)

* change s3 bucket client to put objects using md5 checksum algorithm

Signed-off-by: Ben Ye <benye@amazon.com>

* update docs

Signed-off-by: Ben Ye <benye@amazon.com>

* changelog

Signed-off-by: Ben Ye <benye@amazon.com>

* fix tests

Signed-off-by: Ben Ye <benye@amazon.com>

---------

Signed-off-by: Ben Ye <benye@amazon.com>

CHANGELOG.md

@@ -11,6 +11,7 @@
 * [CHANGE] Querier: Mark `-querier.ingester-streaming` flag as deprecated. Now query ingester streaming is always enabled. #5817
 * [CHANGE] Compactor/Bucket Store: Added `-blocks-storage.bucket-store.block-discovery-strategy` to configure different block listing strategy. Reverted the current recursive block listing mechanism and use the strategy `Concurrent` as in 1.15. #5828
 * [CHANGE] Compactor: Don't halt compactor when overlapped source blocks detected. #5854
+* [CHANGE] S3 Bucket Client: Expose `-blocks-storage.s3.send-content-md5` flag and set default checksum algorithm to MD5. #5870
 * [FEATURE] OTLP ingestion experimental. #5813
 * [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477
 * [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605

docs/blocks-storage/querier.md

@@ -288,6 +288,12 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.bucket-lookup-type
     [bucket_lookup_type: <string> | default = "auto"]
 
+    # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+    # algorithm to verify the provided digest. If false, use CRC32C algorithm
+    # instead.
+    # CLI flag: -blocks-storage.s3.send-content-md5
+    [send_content_md5: <boolean> | default = true]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]

docs/blocks-storage/store-gateway.md

@@ -397,6 +397,12 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.bucket-lookup-type
     [bucket_lookup_type: <string> | default = "auto"]
 
+    # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+    # algorithm to verify the provided digest. If false, use CRC32C algorithm
+    # instead.
+    # CLI flag: -blocks-storage.s3.send-content-md5
+    [send_content_md5: <boolean> | default = true]
+
     # The s3_sse_config configures the S3 server-side encryption.
     # The CLI flags prefix for this block config is: blocks-storage
     [sse: <s3_sse_config>]

docs/configuration/config-file-reference.md

@@ -559,6 +559,12 @@ s3:
   # CLI flag: -alertmanager-storage.s3.bucket-lookup-type
   [bucket_lookup_type: <string> | default = "auto"]
 
+  # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+  # algorithm to verify the provided digest. If false, use CRC32C algorithm
+  # instead.
+  # CLI flag: -alertmanager-storage.s3.send-content-md5
+  [send_content_md5: <boolean> | default = true]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: alertmanager-storage
   [sse: <s3_sse_config>]
@@ -832,6 +838,12 @@ s3:
   # CLI flag: -blocks-storage.s3.bucket-lookup-type
   [bucket_lookup_type: <string> | default = "auto"]
 
+  # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+  # algorithm to verify the provided digest. If false, use CRC32C algorithm
+  # instead.
+  # CLI flag: -blocks-storage.s3.send-content-md5
+  [send_content_md5: <boolean> | default = true]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: blocks-storage
   [sse: <s3_sse_config>]
@@ -4326,6 +4338,12 @@ s3:
   # CLI flag: -ruler-storage.s3.bucket-lookup-type
   [bucket_lookup_type: <string> | default = "auto"]
 
+  # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+  # algorithm to verify the provided digest. If false, use CRC32C algorithm
+  # instead.
+  # CLI flag: -ruler-storage.s3.send-content-md5
+  [send_content_md5: <boolean> | default = true]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: ruler-storage
   [sse: <s3_sse_config>]
@@ -4607,6 +4625,12 @@ s3:
   # CLI flag: -runtime-config.s3.bucket-lookup-type
   [bucket_lookup_type: <string> | default = "auto"]
 
+  # If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum
+  # algorithm to verify the provided digest. If false, use CRC32C algorithm
+  # instead.
+  # CLI flag: -runtime-config.s3.send-content-md5
+  [send_content_md5: <boolean> | default = true]
+
   # The s3_sse_config configures the S3 server-side encryption.
   # The CLI flags prefix for this block config is: runtime-config
   [sse: <s3_sse_config>]

pkg/storage/bucket/s3/bucket_client.go

@@ -81,13 +81,14 @@ func newS3Config(cfg Config) (s3.Config, error) {
 	}
 
 	return s3.Config{
-		Bucket:    cfg.BucketName,
-		Endpoint:  cfg.Endpoint,
-		Region:    cfg.Region,
-		AccessKey: cfg.AccessKeyID,
-		SecretKey: cfg.SecretAccessKey.Value,
-		Insecure:  cfg.Insecure,
-		SSEConfig: sseCfg,
+		Bucket:         cfg.BucketName,
+		Endpoint:       cfg.Endpoint,
+		Region:         cfg.Region,
+		AccessKey:      cfg.AccessKeyID,
+		SecretKey:      cfg.SecretAccessKey.Value,
+		Insecure:       cfg.Insecure,
+		SSEConfig:      sseCfg,
+		SendContentMd5: cfg.SendContentMd5,
 		HTTPConfig: s3.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),

pkg/storage/bucket/s3/config.go

@@ -66,6 +66,7 @@ type Config struct {
 	Insecure         bool           `yaml:"insecure"`
 	SignatureVersion string         `yaml:"signature_version"`
 	BucketLookupType string         `yaml:"bucket_lookup_type"`
+	SendContentMd5   bool           `yaml:"send_content_md5"`
 
 	SSE  SSEConfig  `yaml:"sse"`
 	HTTP HTTPConfig `yaml:"http"`
@@ -86,6 +87,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.")
 	f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
 	f.StringVar(&cfg.BucketLookupType, prefix+"s3.bucket-lookup-type", BucketAutoLookup, fmt.Sprintf("The s3 bucket lookup style. Supported values are: %s.", strings.Join(supportedBucketLookupTypes, ", ")))
+	f.BoolVar(&cfg.SendContentMd5, prefix+"s3.send-content-md5", true, "If true, attach MD5 checksum when upload objects and S3 uses MD5 checksum algorithm to verify the provided digest. If false, use CRC32C algorithm instead.")
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
 	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
 }

pkg/storage/bucket/s3/config_test.go

@@ -18,6 +18,7 @@ import (
 var defaultConfig = Config{
 	SignatureVersion: SignatureVersionV4,
 	BucketLookupType: BucketAutoLookup,
+	SendContentMd5:   true,
 	HTTP: HTTPConfig{
 		Config: bucket_http.Config{
 			IdleConnTimeout:       90 * time.Second,
@@ -78,6 +79,7 @@ http:
 				Insecure:         true,
 				SignatureVersion: "test-signature-version",
 				BucketLookupType: BucketVirtualHostLookup,
+				SendContentMd5:   true,
 				SSE: SSEConfig{
 					Type:                 "test-type",
 					KMSKeyID:             "test-kms-key-id",