feat: add baidu bce bos storage support

Signed-off-by: arcosx <arcosx@outlook.com>
grafana · Feb 15, 2022 · 9d58e8f · 9d58e8f
1 parent da9b4c9
commit 9d58e8f
Show file tree

Hide file tree

Showing 31 changed files with 8,677 additions and 22 deletions.
diff --git a/go.mod b/go.mod
@@ -104,6 +104,8 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.13.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.0.0-00010101000000-000000000000
+	github.com/baidubce/bce-sdk-go v0.9.106
+	github.com/fsnotify/fsnotify v1.5.1
 	github.com/google/renameio/v2 v2.0.0
 	github.com/google/uuid v1.2.0
 	github.com/mattn/go-ieproxy v0.0.1
@@ -163,7 +165,6 @@ require (
 	github.com/envoyproxy/go-control-plane v0.10.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-openapi/analysis v0.20.0 // indirect

diff --git a/go.sum b/go.sum
@@ -293,6 +293,8 @@ github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnO
 github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
 github.com/baidubce/bce-sdk-go v0.9.81 h1:n8KfThLG9fvGv3A+RtTt/jKhg/FPPRpo+iNnS2r+iPI=
 github.com/baidubce/bce-sdk-go v0.9.81/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
+github.com/baidubce/bce-sdk-go v0.9.106 h1:yC04NJdGkjsjLkZZ/BEmetcY5WVnWmAK7gr0LXCWpG8=
+github.com/baidubce/bce-sdk-go v0.9.106/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg=
 github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=

diff --git a/pkg/loki/common/common.go b/pkg/loki/common/common.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/grafana/loki/pkg/storage/chunk/aws"
 	"github.com/grafana/loki/pkg/storage/chunk/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/baidubce"
 	"github.com/grafana/loki/pkg/storage/chunk/gcp"
 	"github.com/grafana/loki/pkg/storage/chunk/hedging"
 	"github.com/grafana/loki/pkg/storage/chunk/openstack"
@@ -50,19 +51,21 @@ func (c *Config) RegisterFlags(_ *flag.FlagSet) {
 }
 
 type Storage struct {
-	S3       aws.S3Config            `yaml:"s3"`
-	GCS      gcp.GCSConfig           `yaml:"gcs"`
-	Azure    azure.BlobStorageConfig `yaml:"azure"`
-	Swift    openstack.SwiftConfig   `yaml:"swift"`
-	FSConfig FilesystemConfig        `yaml:"filesystem"`
-	Hedging  hedging.Config          `yaml:"hedging"`
+	S3       aws.S3Config              `yaml:"s3"`
+	GCS      gcp.GCSConfig             `yaml:"gcs"`
+	Azure    azure.BlobStorageConfig   `yaml:"azure"`
+	Bos      baidubce.BosStorageConfig `yaml:"bos"`
+	Swift    openstack.SwiftConfig     `yaml:"swift"`
+	FSConfig FilesystemConfig          `yaml:"filesystem"`
+	Hedging  hedging.Config            `yaml:"hedging"`
 }
 
 func (s *Storage) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	s.S3.RegisterFlagsWithPrefix(prefix+".s3", f)
 	s.GCS.RegisterFlagsWithPrefix(prefix+".gcs", f)
 	s.Azure.RegisterFlagsWithPrefix(prefix+".azure", f)
 	s.Swift.RegisterFlagsWithPrefix(prefix+".swift", f)
+	s.Bos.RegisterFlagsWithPrefix(prefix+".bos", f)
 	s.FSConfig.RegisterFlagsWithPrefix(prefix+".filesystem", f)
 	s.Hedging.RegisterFlagsWithPrefix(prefix, f)
 }

diff --git a/pkg/ruler/base/storage.go b/pkg/ruler/base/storage.go
@@ -20,6 +20,7 @@ import (
 	"github.com/grafana/loki/pkg/storage/chunk"
 	"github.com/grafana/loki/pkg/storage/chunk/aws"
 	"github.com/grafana/loki/pkg/storage/chunk/azure"
+	"github.com/grafana/loki/pkg/storage/chunk/baidubce"
 	"github.com/grafana/loki/pkg/storage/chunk/gcp"
 	"github.com/grafana/loki/pkg/storage/chunk/hedging"
 	"github.com/grafana/loki/pkg/storage/chunk/openstack"
@@ -33,11 +34,12 @@ type RuleStoreConfig struct {
 	ConfigDB configClient.Config `yaml:"configdb"`
 
 	// Object Storage Configs
-	Azure azure.BlobStorageConfig `yaml:"azure"`
-	GCS   gcp.GCSConfig           `yaml:"gcs"`
-	S3    aws.S3Config            `yaml:"s3"`
-	Swift openstack.SwiftConfig   `yaml:"swift"`
-	Local local.Config            `yaml:"local"`
+	Azure azure.BlobStorageConfig   `yaml:"azure"`
+	GCS   gcp.GCSConfig             `yaml:"gcs"`
+	S3    aws.S3Config              `yaml:"s3"`
+	BOS   baidubce.BosStorageConfig `yaml:"bos"`
+	Swift openstack.SwiftConfig     `yaml:"swift"`
+	Local local.Config              `yaml:"local"`
 
 	mock rulestore.RuleStore `yaml:"-"`
 }
@@ -50,7 +52,7 @@ func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) {
 	cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.Swift.RegisterFlagsWithPrefix("ruler.storage.", f)
 	cfg.Local.RegisterFlagsWithPrefix("ruler.storage.", f)
-
+	cfg.BOS.RegisterFlagsWithPrefix("ruler.storage.", f)
 	f.StringVar(&cfg.Type, "ruler.storage.type", "configdb", "Method to use for backend rule storage (configdb, azure, gcs, s3, swift, local)")
 }
 
@@ -101,6 +103,8 @@ func NewLegacyRuleStore(cfg RuleStoreConfig, hedgeCfg hedging.Config, clientMetr
 		client, err = gcp.NewGCSObjectClient(context.Background(), cfg.GCS, hedgeCfg)
 	case "s3":
 		client, err = aws.NewS3ObjectClient(cfg.S3, hedgeCfg)
+	case "bos":
+		client, err = baidubce.NewBosObjectStorage(&cfg.BOS)
 	case "swift":
 		client, err = openstack.NewSwiftObjectClient(cfg.Swift, hedgeCfg)
 	case "local":