Feat: add Baidu Cloud BOS as storage backends for Loki #4788

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Main
 * [5899](https://github.com/grafana/loki/pull/5899) **simonswine**: Update go image to 1.17.9.
 * [5888](https://github.com/grafana/loki/pull/5888) **Papawy** Fix common config net interface name overwritten by ring common config
+* [5848](https://github.com/grafana/loki/pull/5848) **arcosx**: Add Baidu AI Cloud as a storage backend choice.
 * [5799](https://github.com/grafana/loki/pull/5799) **cyriltovena** Fix deduping issues when multiple entries with the same timestamp exist.
 * [5799](https://github.com/grafana/loki/pull/5799) **cyriltovena** Fixes deduping issues when multiple entries exists with the same timestamp.
 * [5780](https://github.com/grafana/loki/pull/5780) **simonswine**: Update alpine image to 3.15.4.

docs/sources/configuration/_index.md

@@ -469,7 +469,7 @@ ruler_client:
 [poll_interval: <duration> | default = 1m]
 
 storage:
-  # Method to use for backend rule storage (azure, gcs, s3, swift, local).
+  # Method to use for backend rule storage (azure, gcs, s3, swift, local, bos).
   # CLI flag: -ruler.storage.type
   [type: <string> ]
 
@@ -488,6 +488,9 @@ storage:
   # Configures backend rule storage for a local file system directory.
   [local: <local_storage_config>]
 
+  # Configures backend rule storage for Baidu Object Storage (BOS).
+  [bos: <bos_storage_config>]
+
   # The `hedging` block configures how to hedge storage requests.
   [hedging: <hedging>]
 
@@ -725,6 +728,25 @@ ring:
   [num_tokens: <int> | default = 128]
 ```
 
+## bos_storage_config
+
+The `bos_storage_config` block configures Baidu Object Storage (BOS)  as general storage for data generated by Loki.
+
+```yaml
+# Name of BOS bucket.
+# CLI flag: <prefix>.baidubce.bucket-name
+[ bucket_name: <string> | default = "" ]
+# BOS endpoint to connect to.
+# CLI flag: <prefix>.baidubce.endpoint
+[ endpoint: <string> | default = "bj.bcebos.com" ]
+# Baidu Cloud Engine (BCE) Access Key ID
+# CLI flag: <prefix>.baidubce.access-key-id
+[ access_key_id: <string> | default = "" ]
+# BCE Secret Access Key
+# CLI flag: <prefix>.baidubce.secret-access-key
+[ secret_access_key: <string> | default = "" ]
+```
+
 ## azure_storage_config
 
 The `azure_storage_config` configures Azure as a general storage for different data generated by Loki.
@@ -2009,7 +2031,7 @@ compacts index shards to more performant forms.
 [working_directory: <string>]
 
 # The shared store used for storing boltdb files.
-# Supported types: gcs, s3, azure, swift, filesystem.
+# Supported types: gcs, s3, azure, swift, filesystem, bos.
 # CLI flag: -boltdb.shipper.compactor.shared-store
 [shared_store: <string>]
 
@@ -2563,8 +2585,12 @@ If any specific configuration for an object storage client have been provided el
 # Configures a (local) file system as the common storage.
 [filesystem: <filesystem>]
 
+# Configures Baidu Object Storage (BOS) as the common storage.
+[bos: <bos_storage_config>]
+
 # The `hedging_config` configures how to hedge requests for the storage.
 [hedging: <hedging_config>]
+
 ```
 
 ### filesystem

docs/sources/configuration/examples.md

@@ -124,6 +124,37 @@ storage_config:
 ```
 
 
+## bos-config.yaml
+
+```yaml
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: boltdb-shipper
+      object_store: bos
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+    shared_store: bos
+
+  bos:
+    bucket_name: bucket_name_1
+    endpoint: bj.bcebos.com
+    access_key_id: access_key_id
+    secret_access_key: secret_access_key
+
+compactor:
+  working_directory: /tmp/loki/compactor
+  shared_store: bos
+```
+
+
 ## cassandra-index.yaml
 
 ```yaml

docs/sources/configuration/examples/bos-config.yaml

@@ -0,0 +1,25 @@
+schema_config:
+  configs:
+    - from: 2020-05-15
+      store: boltdb-shipper
+      object_store: bos
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/index_cache
+    shared_store: bos
+
+  bos:
+    bucket_name: bucket_name_1
+    endpoint: bj.bcebos.com
+    access_key_id: access_key_id
+    secret_access_key: secret_access_key
+
+compactor:
+  working_directory: /tmp/loki/compactor
+  shared_store: bos

docs/sources/operations/storage/_index.md

@@ -43,6 +43,7 @@ The following are supported for the chunks:
 - [Amazon S3](https://aws.amazon.com/s3)
 - [Google Cloud Storage](https://cloud.google.com/storage/)
 - [Filesystem](filesystem/) (please read more about the filesystem to understand the pros/cons before using with production data)
+- [Baidu Object Storage](https://cloud.baidu.com/product/bos.html)
 
 ## Cloud Storage Permissions
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/Workiva/go-datastructures v1.0.53
 	github.com/alicebob/miniredis/v2 v2.14.3
 	github.com/aws/aws-sdk-go v1.43.10
+	github.com/baidubce/bce-sdk-go v0.9.81
 	github.com/bmatcuk/doublestar v1.2.2
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/buger/jsonparser v1.1.1

pkg/loki/common/common.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/loki/pkg/storage/chunk/client/aws"
 	"github.com/grafana/loki/pkg/storage/chunk/client/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/client/baidubce"
 	"github.com/grafana/loki/pkg/storage/chunk/client/gcp"
 	"github.com/grafana/loki/pkg/storage/chunk/client/hedging"
 	"github.com/grafana/loki/pkg/storage/chunk/client/openstack"
@@ -54,19 +55,21 @@ func (c *Config) RegisterFlags(_ *flag.FlagSet) {
 }
 
 type Storage struct {
-	S3       aws.S3Config            `yaml:"s3"`
-	GCS      gcp.GCSConfig           `yaml:"gcs"`
-	Azure    azure.BlobStorageConfig `yaml:"azure"`
-	Swift    openstack.SwiftConfig   `yaml:"swift"`
-	FSConfig FilesystemConfig        `yaml:"filesystem"`
-	Hedging  hedging.Config          `yaml:"hedging"`
+	S3       aws.S3Config              `yaml:"s3"`
+	GCS      gcp.GCSConfig             `yaml:"gcs"`
+	Azure    azure.BlobStorageConfig   `yaml:"azure"`
+	BOS      baidubce.BOSStorageConfig `yaml:"bos"`
+	Swift    openstack.SwiftConfig     `yaml:"swift"`
+	FSConfig FilesystemConfig          `yaml:"filesystem"`
+	Hedging  hedging.Config            `yaml:"hedging"`
 }
 
 func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	s.S3.RegisterFlagsWithPrefix(prefix+".s3", f)
 	s.GCS.RegisterFlagsWithPrefix(prefix+".gcs", f)
 	s.Azure.RegisterFlagsWithPrefix(prefix+".azure", f)
 	s.Swift.RegisterFlagsWithPrefix(prefix+".swift", f)
+	s.BOS.RegisterFlagsWithPrefix(prefix+".bos", f)
 	s.FSConfig.RegisterFlagsWithPrefix(prefix+".filesystem", f)
 	s.Hedging.RegisterFlagsWithPrefix(prefix, f)
 }

pkg/loki/config_wrapper.go

@@ -449,6 +449,16 @@ func applyStorageConfig(cfg, defaults *ConfigWrapper) error {
 		}
 	}
 
+	if !reflect.DeepEqual(cfg.Common.Storage.BOS, defaults.StorageConfig.BOSStorageConfig) {
+		configsFound++
+		applyConfig = func(r *ConfigWrapper) {
+			r.Ruler.StoreConfig.Type = "bos"
+			r.Ruler.StoreConfig.BOS = r.Common.Storage.BOS
+			r.StorageConfig.BOSStorageConfig = r.Common.Storage.BOS
+			r.CompactorConfig.SharedStoreType = config.StorageTypeBOS
+		}
+	}
+
 	if !reflect.DeepEqual(cfg.Common.Storage.Swift, defaults.StorageConfig.Swift) {
 		configsFound++
 

pkg/loki/config_wrapper_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/loki/pkg/storage/bucket/swift"
 	"github.com/grafana/loki/pkg/storage/chunk/client/aws"
 	"github.com/grafana/loki/pkg/storage/chunk/client/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/client/baidubce"
 	"github.com/grafana/loki/pkg/storage/chunk/client/gcp"
 	"github.com/grafana/loki/pkg/storage/config"
 	"github.com/grafana/loki/pkg/util"
@@ -201,6 +202,7 @@ memberlist:
 		//    s3: aws.S3Config
 		//    swift: openstack.SwiftConfig
 		//    filesystem: Filesystem
+		//    bos: baidubce.BOSStorageConfig
 
 		t.Run("does not automatically configure cloud object storage", func(t *testing.T) {
 			config, defaults := testContext(emptyConfigString, nil)
@@ -211,12 +213,13 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
-
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
 			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig, config.StorageConfig.AWSStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
 			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
 			assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
 		})
 
 		t.Run("when multiple configs are provided, an error is returned", func(t *testing.T) {
@@ -285,12 +288,13 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
-
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
 			// should remain empty
 			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
 			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
 			assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
 		})
 
 		t.Run("when common gcs storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -320,11 +324,13 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
 			// should remain empty
 			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
 			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
 			assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
 		})
 
 		t.Run("when common azure storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -370,8 +376,48 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
+
+			// should remain empty
+			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
+			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
+			assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
+		})
+
+		t.Run("when common bos storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
+			bosConfig := `common:
+  storage:
+    bos:
+      bucket_name: arcosx
+      endpoint: bj.bcebos.com
+      access_key_id: baidu
+      secret_access_key: bce`
+
+			config, defaults := testContext(bosConfig, nil)
+
+			assert.Equal(t, "bos", config.Ruler.StoreConfig.Type)
+
+			for _, actual := range []baidubce.BOSStorageConfig{
+				config.Ruler.StoreConfig.BOS,
+				config.StorageConfig.BOSStorageConfig,
+			} {
+				assert.Equal(t, "arcosx", actual.BucketName)
+				assert.Equal(t, "bj.bcebos.com", actual.Endpoint)
+				assert.Equal(t, "baidu", actual.AccessKeyID)
+				assert.Equal(t, "bce", actual.SecretAccessKey)
+			}
+
+			// should remain empty
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
 
 			// should remain empty
+			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
 			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
 			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
@@ -435,12 +481,13 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
-
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
 			// should remain empty
 			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
 			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
 			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
 		})
 
 		t.Run("when common filesystem/local config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -462,12 +509,13 @@ memberlist:
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
 			assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
-
+			assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
 			// should remain empty
 			assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
 			assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
 			assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
 			assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
+			assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
 		})
 
 		t.Run("explicit ruler storage object storage configuration provided via config file is preserved", func(t *testing.T) {

pkg/ruler/base/storage.go

@@ -21,6 +21,7 @@ import (
 	"github.com/grafana/loki/pkg/storage/chunk/client"
 	"github.com/grafana/loki/pkg/storage/chunk/client/aws"
 	"github.com/grafana/loki/pkg/storage/chunk/client/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/client/baidubce"
 	"github.com/grafana/loki/pkg/storage/chunk/client/gcp"
 	"github.com/grafana/loki/pkg/storage/chunk/client/hedging"
 	"github.com/grafana/loki/pkg/storage/chunk/client/openstack"
@@ -33,11 +34,12 @@ type RuleStoreConfig struct {
 	ConfigDB configClient.Config `yaml:"configdb"`
 
 	// Object Storage Configs
-	Azure azure.BlobStorageConfig `yaml:"azure"`
-	GCS   gcp.GCSConfig           `yaml:"gcs"`
-	S3    aws.S3Config            `yaml:"s3"`
-	Swift openstack.SwiftConfig   `yaml:"swift"`
-	Local local.Config            `yaml:"local"`
+	Azure azure.BlobStorageConfig   `yaml:"azure"`
+	GCS   gcp.GCSConfig             `yaml:"gcs"`
+	S3    aws.S3Config              `yaml:"s3"`
+	BOS   baidubce.BOSStorageConfig `yaml:"bos"`
+	Swift openstack.SwiftConfig     `yaml:"swift"`
+	Local local.Config              `yaml:"local"`
 
 	mock rulestore.RuleStore `yaml:"-"`
 }
@@ -50,7 +52,7 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) {
 	cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.Local.RegisterFlagsWithPrefix("ruler.storage.", f)
-
+	cfg.BOS.RegisterFlagsWithPrefix("ruler.storage.", f)
 	f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3, swift, local)")
 }
 
@@ -101,6 +103,8 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr
 		client, err = gcp.NewGCSObjectClient(context.Background(), cfg.GCS, hedgeCfg)
 	case "s3":
 		client, err = aws.NewS3ObjectClient(cfg.S3, hedgeCfg)
+	case "bos":
+		client, err = baidubce.NewBOSObjectStorage(&cfg.BOS)
 	case "swift":
 		client, err = openstack.NewSwiftObjectClient(cfg.Swift, hedgeCfg)
 	case "local":