From 40df1635e60fff50298a377ec4bff067d6cb1fb2 Mon Sep 17 00:00:00 2001 From: Binbin Zou <42572980+zbbkeepgoing@users.noreply.github.com> Date: Tue, 11 Aug 2020 13:58:28 +0800 Subject: [PATCH] Support azure storage of azurechina,azuregerman,azureusgov (#2988) * Support azure storage of azurechina,azuregerman,azureusgov Signed-off-by: Binbin Zou * change pr number Signed-off-by: Binbin Zou * Optimize the code Signed-off-by: Binbin Zou Co-authored-by: Binbin Zou --- azure/blob_storage_client.go | 53 +++++++++++++++++++++++++++++++++--- storage/factory.go | 3 ++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/azure/blob_storage_client.go b/azure/blob_storage_client.go index 45d34c2748e8..ba45008e80c1 100644 --- a/azure/blob_storage_client.go +++ b/azure/blob_storage_client.go @@ -17,11 +17,39 @@ import ( "github.com/cortexproject/cortex/pkg/util/flagext" ) -const blobURLFmt = "https://%s.blob.core.windows.net/%s/%s" -const containerURLFmt = "https://%s.blob.core.windows.net/%s" +const ( + // Environment + azureGlobal = "AzureGlobal" + azureChinaCloud = "AzureChinaCloud" + azureGermanCloud = "AzureGermanCloud" + azureUSGovernment = "AzureUSGovernment" +) + +var ( + supportedEnvironments = []string{azureGlobal, azureChinaCloud, azureGermanCloud, azureUSGovernment} + endpoints = map[string]struct{ blobURLFmt, containerURLFmt string }{ + azureGlobal: { + "https://%s.blob.core.windows.net/%s/%s", + "https://%s.blob.core.windows.net/%s", + }, + azureChinaCloud: { + "https://%s.blob.core.chinacloudapi.cn/%s/%s", + "https://%s.blob.core.chinacloudapi.cn/%s", + }, + azureGermanCloud: { + "https://%s.blob.core.cloudapi.de/%s/%s", + "https://%s.blob.core.cloudapi.de/%s", + }, + azureUSGovernment: { + "https://%s.blob.core.usgovcloudapi.net/%s/%s", + "https://%s.blob.core.usgovcloudapi.net/%s", + }, + } +) // BlobStorageConfig defines the configurable flags that can be defined when using azure blob storage. type BlobStorageConfig struct { + Environment string `yaml:"environment"` ContainerName string `yaml:"container_name"` AccountName string `yaml:"account_name"` AccountKey flagext.Secret `yaml:"account_key"` @@ -41,6 +69,7 @@ func (c *BlobStorageConfig) RegisterFlags(f *flag.FlagSet) { // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet func (c *BlobStorageConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&c.Environment, prefix+"azure.environment", azureGlobal, fmt.Sprintf("Azure Cloud environment. Supported values are: %s.", strings.Join(supportedEnvironments, ", "))) f.StringVar(&c.ContainerName, prefix+"azure.container-name", "cortex", "Name of the blob container used to store chunks. This container must be created before running cortex.") f.StringVar(&c.AccountName, prefix+"azure.account-name", "", "The Microsoft Azure account name to be used") f.Var(&c.AccountKey, prefix+"azure.account-key", "The Microsoft Azure account key to use.") @@ -123,7 +152,7 @@ func (b *BlobStorage) getBlobURL(blobID string) (azblob.BlockBlobURL, error) { blobID = strings.Replace(blobID, ":", "-", -1) //generate url for new chunk blob - u, err := url.Parse(fmt.Sprintf(blobURLFmt, b.cfg.AccountName, b.cfg.ContainerName, blobID)) + u, err := url.Parse(fmt.Sprintf(b.selectBlobURLFmt(), b.cfg.AccountName, b.cfg.ContainerName, blobID)) if err != nil { return azblob.BlockBlobURL{}, err } @@ -137,7 +166,7 @@ func (b *BlobStorage) getBlobURL(blobID string) (azblob.BlockBlobURL, error) { } func (b *BlobStorage) buildContainerURL() (azblob.ContainerURL, error) { - u, err := url.Parse(fmt.Sprintf(containerURLFmt, b.cfg.AccountName, b.cfg.ContainerName)) + u, err := url.Parse(fmt.Sprintf(b.selectContainerURLFmt(), b.cfg.AccountName, b.cfg.ContainerName)) if err != nil { return azblob.ContainerURL{}, err } @@ -214,3 +243,19 @@ func (b *BlobStorage) DeleteObject(ctx context.Context, blobID string) error { func (b *BlobStorage) PathSeparator() string { return b.delimiter } + +// Validate the config. +func (c *BlobStorageConfig) Validate() error { + if !util.StringsContain(supportedEnvironments, c.Environment) { + return fmt.Errorf("unsupported Azure blob storage environment: %s, please select one of: %s ", c.Environment, strings.Join(supportedEnvironments, ", ")) + } + return nil +} + +func (b *BlobStorage) selectBlobURLFmt() string { + return endpoints[b.cfg.Environment].blobURLFmt +} + +func (b *BlobStorage) selectContainerURLFmt() string { + return endpoints[b.cfg.Environment].containerURLFmt +} diff --git a/storage/factory.go b/storage/factory.go index d63b1d958b1f..32d51ad1fc47 100644 --- a/storage/factory.go +++ b/storage/factory.go @@ -114,6 +114,9 @@ func (cfg *Config) Validate() error { if err := cfg.IndexQueriesCacheConfig.Validate(); err != nil { return errors.Wrap(err, "invalid Index Queries Cache config") } + if err := cfg.AzureStorageConfig.Validate(); err != nil { + return errors.Wrap(err, "invalid Azure Storage config") + } return nil }