Support azure storage of azurechina,azuregerman,azureusgov (grafana#2988

) * Support azure storage of azurechina,azuregerman,azureusgov Signed-off-by: Binbin Zou <binbin.zou@BinbinZous-MacBook-Pro.local> * change pr number Signed-off-by: Binbin Zou <binbin.zou@BinbinZous-MacBook-Pro.local> * Optimize the code Signed-off-by: Binbin Zou <binbin.zou@BinbinZous-MacBook-Pro.local> Co-authored-by: Binbin Zou <binbin.zou@BinbinZous-MacBook-Pro.local>
cyriltovena · Aug 11, 2020 · 40df163 · 40df163
1 parent d68fc89
commit 40df163
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 4 deletions.
diff --git a/azure/blob_storage_client.go b/azure/blob_storage_client.go
@@ -17,11 +17,39 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
-const blobURLFmt = "https://%s.blob.core.windows.net/%s/%s"
-const containerURLFmt = "https://%s.blob.core.windows.net/%s"
+const (
+	// Environment
+	azureGlobal       = "AzureGlobal"
+	azureChinaCloud   = "AzureChinaCloud"
+	azureGermanCloud  = "AzureGermanCloud"
+	azureUSGovernment = "AzureUSGovernment"
+)
+
+var (
+	supportedEnvironments = []string{azureGlobal, azureChinaCloud, azureGermanCloud, azureUSGovernment}
+	endpoints             = map[string]struct{ blobURLFmt, containerURLFmt string }{
+		azureGlobal: {
+			"https://%s.blob.core.windows.net/%s/%s",
+			"https://%s.blob.core.windows.net/%s",
+		},
+		azureChinaCloud: {
+			"https://%s.blob.core.chinacloudapi.cn/%s/%s",
+			"https://%s.blob.core.chinacloudapi.cn/%s",
+		},
+		azureGermanCloud: {
+			"https://%s.blob.core.cloudapi.de/%s/%s",
+			"https://%s.blob.core.cloudapi.de/%s",
+		},
+		azureUSGovernment: {
+			"https://%s.blob.core.usgovcloudapi.net/%s/%s",
+			"https://%s.blob.core.usgovcloudapi.net/%s",
+		},
+	}
+)
 
 // BlobStorageConfig defines the configurable flags that can be defined when using azure blob storage.
 type BlobStorageConfig struct {
+	Environment        string         `yaml:"environment"`
 	ContainerName      string         `yaml:"container_name"`
 	AccountName        string         `yaml:"account_name"`
 	AccountKey         flagext.Secret `yaml:"account_key"`
@@ -41,6 +69,7 @@ func (c *BlobStorageConfig) RegisterFlags(f *flag.FlagSet) {
 
 // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet
 func (c *BlobStorageConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&c.Environment, prefix+"azure.environment", azureGlobal, fmt.Sprintf("Azure Cloud environment. Supported values are: %s.", strings.Join(supportedEnvironments, ", ")))
 	f.StringVar(&c.ContainerName, prefix+"azure.container-name", "cortex", "Name of the blob container used to store chunks. This container must be created before running cortex.")
 	f.StringVar(&c.AccountName, prefix+"azure.account-name", "", "The Microsoft Azure account name to be used")
 	f.Var(&c.AccountKey, prefix+"azure.account-key", "The Microsoft Azure account key to use.")
@@ -123,7 +152,7 @@ func (b *BlobStorage) getBlobURL(blobID string) (azblob.BlockBlobURL, error) {
 	blobID = strings.Replace(blobID, ":", "-", -1)
 
 	//generate url for new chunk blob
-	u, err := url.Parse(fmt.Sprintf(blobURLFmt, b.cfg.AccountName, b.cfg.ContainerName, blobID))
+	u, err := url.Parse(fmt.Sprintf(b.selectBlobURLFmt(), b.cfg.AccountName, b.cfg.ContainerName, blobID))
 	if err != nil {
 		return azblob.BlockBlobURL{}, err
 	}
@@ -137,7 +166,7 @@ func (b *BlobStorage) getBlobURL(blobID string) (azblob.BlockBlobURL, error) {
 }
 
 func (b *BlobStorage) buildContainerURL() (azblob.ContainerURL, error) {
-	u, err := url.Parse(fmt.Sprintf(containerURLFmt, b.cfg.AccountName, b.cfg.ContainerName))
+	u, err := url.Parse(fmt.Sprintf(b.selectContainerURLFmt(), b.cfg.AccountName, b.cfg.ContainerName))
 	if err != nil {
 		return azblob.ContainerURL{}, err
 	}
@@ -214,3 +243,19 @@ func (b *BlobStorage) DeleteObject(ctx context.Context, blobID string) error {
 func (b *BlobStorage) PathSeparator() string {
 	return b.delimiter
 }
+
+// Validate the config.
+func (c *BlobStorageConfig) Validate() error {
+	if !util.StringsContain(supportedEnvironments, c.Environment) {
+		return fmt.Errorf("unsupported Azure blob storage environment: %s, please select one of: %s ", c.Environment, strings.Join(supportedEnvironments, ", "))
+	}
+	return nil
+}
+
+func (b *BlobStorage) selectBlobURLFmt() string {
+	return endpoints[b.cfg.Environment].blobURLFmt
+}
+
+func (b *BlobStorage) selectContainerURLFmt() string {
+	return endpoints[b.cfg.Environment].containerURLFmt
+}
diff --git a/storage/factory.go b/storage/factory.go
@@ -114,6 +114,9 @@ func (cfg *Config) Validate() error {
 	if err := cfg.IndexQueriesCacheConfig.Validate(); err != nil {
 		return errors.Wrap(err, "invalid Index Queries Cache config")
 	}
+	if err := cfg.AzureStorageConfig.Validate(); err != nil {
+		return errors.Wrap(err, "invalid Azure Storage config")
+	}
 	return nil
 }