New S3 Flag to configure ListObjectsVersion

This patch make it possible to configure the S3 ListObjectsVersion, which is
already configurable in the [thanos-io S3
Client](https://github.com/thanos-io/objstore/blob/23ebe2eacadd89cf23bf0fbd931352112b4c846d/providers/s3/s3.go#LL137C47-L137C67).

Signed-off-by: Sven Haardiek <sven@haardiek.de>

CHANGELOG.md

@@ -10,6 +10,7 @@
 * [FEATURE] Query-frontend: added experimental support to cache cardinality query responses. The cache will be used when `-query-frontend.cache-results` is enabled and `-query-frontend.results-cache-ttl-for-cardinality-query` set to a value greater than 0. The following metrics have been added to track the query results cache hit ratio per `request_type`: #5212 #5235
   * `cortex_frontend_query_result_cache_requests_total{request_type="query_range|cardinality"}`
   * `cortex_frontend_query_result_cache_hits_total{request_type="query_range|cardinality"}`
+* [FEATURE] Added `-<prefix>.s3.list-objects-version` flag to configure the S3 list objects version.
 * [ENHANCEMENT] Cardinality API: When zone aware replication is enabled, the label values cardinality API can now tolerate single zone failure #5178
 * [ENHANCEMENT] Distributor: optimize sending requests to ingesters when incoming requests don't need to be modified. For now this feature can be disabled by setting `-timeseries-unmarshal-caching-optimization-enabled=false`. #5137
 * [ENHANCEMENT] Add advanced CLI flags to control gRPC client behaviour: #5161

docs/sources/mimir/references/configuration-parameters/index.md

@@ -4173,6 +4173,10 @@ The s3_backend block configures the connection to Amazon S3 object storage backe
 # CLI flag: -<prefix>.s3.signature-version
 [signature_version: <string> | default = "v4"]
 
+# (advanced) S3 list object version
+# CLI flag: -<prefix>.s3.list-objects-version
+[list_objects_version: <string> | default = ""]
+
 # (experimental) The S3 storage class to use, not set by default. Details can be
 # found at https://aws.amazon.com/s3/storage-classes/. Supported values are:
 # STANDARD, REDUCED_REDUNDANCY, GLACIER, STANDARD_IA, ONEZONE_IA,

pkg/storage/bucket/s3/bucket_client.go

@@ -50,14 +50,15 @@ func newS3Config(cfg Config) (s3.Config, error) {
 	}
 
 	return s3.Config{
-		Bucket:          cfg.BucketName,
-		Endpoint:        cfg.Endpoint,
-		Region:          cfg.Region,
-		AccessKey:       cfg.AccessKeyID,
-		SecretKey:       cfg.SecretAccessKey.String(),
-		Insecure:        cfg.Insecure,
-		PutUserMetadata: putUserMetadata,
-		SSEConfig:       sseCfg,
+		Bucket:             cfg.BucketName,
+		Endpoint:           cfg.Endpoint,
+		Region:             cfg.Region,
+		AccessKey:          cfg.AccessKeyID,
+		SecretKey:          cfg.SecretAccessKey.String(),
+		Insecure:           cfg.Insecure,
+		PutUserMetadata:    putUserMetadata,
+		SSEConfig:          sseCfg,
+		ListObjectsVersion: cfg.ListObjectsVersion,
 		HTTPConfig: s3.HTTPConfig{
 			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
 			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),

pkg/storage/bucket/s3/config.go

@@ -75,14 +75,15 @@ func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 
 // Config holds the config options for an S3 backend
 type Config struct {
-	Endpoint         string         `yaml:"endpoint"`
-	Region           string         `yaml:"region"`
-	BucketName       string         `yaml:"bucket_name"`
-	SecretAccessKey  flagext.Secret `yaml:"secret_access_key"`
-	AccessKeyID      string         `yaml:"access_key_id"`
-	Insecure         bool           `yaml:"insecure" category:"advanced"`
-	SignatureVersion string         `yaml:"signature_version" category:"advanced"`
-	StorageClass     string         `yaml:"storage_class" category:"experimental"`
+	Endpoint           string         `yaml:"endpoint"`
+	Region             string         `yaml:"region"`
+	BucketName         string         `yaml:"bucket_name"`
+	SecretAccessKey    flagext.Secret `yaml:"secret_access_key"`
+	AccessKeyID        string         `yaml:"access_key_id"`
+	Insecure           bool           `yaml:"insecure" category:"advanced"`
+	SignatureVersion   string         `yaml:"signature_version" category:"advanced"`
+	ListObjectsVersion string         `yaml:"list_objects_version" category:"advanced"`
+	StorageClass       string         `yaml:"storage_class" category:"experimental"`
 
 	SSE  SSEConfig  `yaml:"sse"`
 	HTTP HTTPConfig `yaml:"http"`
@@ -102,6 +103,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.Endpoint, prefix+"s3.endpoint", "", "The S3 bucket endpoint. It could be an AWS S3 endpoint listed at https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an S3-compatible service in hostname:port format.")
 	f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.")
 	f.StringVar(&cfg.SignatureVersion, prefix+"s3.signature-version", SignatureVersionV4, fmt.Sprintf("The signature version to use for authenticating against S3. Supported values are: %s.", strings.Join(supportedSignatureVersions, ", ")))
+	f.StringVar(&cfg.ListObjectsVersion, prefix+"s3.list-objects-version", "", "S3 list object version")
 	f.StringVar(&cfg.StorageClass, prefix+"s3.storage-class", "", "The S3 storage class to use, not set by default. Details can be found at https://aws.amazon.com/s3/storage-classes/. Supported values are: "+strings.Join(supportedStorageClasses, ", "))
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
 	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)