objectstore/cos: add endpoint field for config vpc internal endpoint (

#4999)

* objectstore/cos: add `endpoint` field for config vpc internal endpoint

Signed-off-by: Jimmie Han <hanjinming@outlook.com>

* add changelog

Signed-off-by: Jimmie Han <hanjinming@outlook.com>

* fix docs

Signed-off-by: Jimmie Han <hanjinming@outlook.com>

* fix typo

Signed-off-by: Jimmie Han <hanjinming@outlook.com>

CHANGELOG.md

@@ -37,6 +37,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4888](https://github.com/thanos-io/thanos/pull/4888) Cache: support redis cache backend.
 - [#4946](https://github.com/thanos-io/thanos/pull/4946) Store: Support tls_config configuration for the s3 minio client.
 - [#4974](https://github.com/thanos-io/thanos/pull/4974) Store: Support tls_config configuration for connecting with Azure storage.
+- [#4999](https://github.com/thanos-io/thanos/pull/4999) COS: Support `endpoint` configuration for vpc internal endpoint.
 
 ### Fixed
 

docs/storage.md

@@ -407,6 +407,7 @@ config:
   bucket: ""
   region: ""
   app_id: ""
+  endpoint: ""
   secret_key: ""
   secret_id: ""
   http_config:
@@ -419,6 +420,10 @@ config:
     max_conns_per_host: 0
 ```
 
+The `secret_key` and `secret_id` field is required. The `http_config` field is optional for optimize HTTP transport settings. There are two ways to configure the required bucket information:
+1. Provide the values of `bucket`, `region` and `app_id` keys.
+2. Provide the values of `endpoint` key with url format when you want to specify vpc internal endpoint. Please refer to the document of [endpoint](https://intl.cloud.tencent.com/document/product/436/6224) for more detail.
+
 Set the flags `--objstore.config-file` to reference to the configuration file.
 
 #### AliYun OSS

pkg/objstore/cos/cos.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
@@ -54,13 +55,24 @@ type Config struct {
 	Bucket     string     `yaml:"bucket"`
 	Region     string     `yaml:"region"`
 	AppId      string     `yaml:"app_id"`
+	Endpoint   string     `yaml:"endpoint"`
 	SecretKey  string     `yaml:"secret_key"`
 	SecretId   string     `yaml:"secret_id"`
 	HTTPConfig HTTPConfig `yaml:"http_config"`
 }
 
 // Validate checks to see if mandatory cos config options are set.
 func (conf *Config) validate() error {
+	if conf.Endpoint != "" {
+		if _, err := url.Parse(conf.Endpoint); err != nil {
+			return errors.Wrap(err, "parse endpoint")
+		}
+		if conf.SecretId == "" ||
+			conf.SecretKey == "" {
+			return errors.New("secret_id or secret_key is empty")
+		}
+		return nil
+	}
 	if conf.Bucket == "" ||
 		conf.AppId == "" ||
 		conf.Region == "" ||
@@ -119,7 +131,15 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error
 		return nil, errors.Wrap(err, "validate cos configuration")
 	}
 
-	bucketURL := cos.NewBucketURL(fmt.Sprintf("%s-%s", config.Bucket, config.AppId), config.Region, true)
+	var bucketURL *url.URL
+	if config.Endpoint != "" {
+		bucketURL, err = url.Parse(config.Endpoint)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse endpoint")
+		}
+	} else {
+		bucketURL = cos.NewBucketURL(fmt.Sprintf("%s-%s", config.Bucket, config.AppId), config.Region, true)
+	}
 	b := &cos.BaseURL{BucketURL: bucketURL}
 	client := cos.NewClient(b, &http.Client{
 		Transport: &cos.AuthorizationTransport{
@@ -355,6 +375,7 @@ func configFromEnv() Config {
 		Bucket:    os.Getenv("COS_BUCKET"),
 		AppId:     os.Getenv("COS_APP_ID"),
 		Region:    os.Getenv("COS_REGION"),
+		Endpoint:  os.Getenv("COS_ENDPOINT"),
 		SecretId:  os.Getenv("COS_SECRET_ID"),
 		SecretKey: os.Getenv("COS_SECRET_KEY"),
 	}
@@ -424,6 +445,16 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) {
 }
 
 func validateForTest(conf Config) error {
+	if conf.Endpoint != "" {
+		if _, err := url.Parse(conf.Endpoint); err != nil {
+			return errors.Wrap(err, "parse endpoint")
+		}
+		if conf.SecretId == "" ||
+			conf.SecretKey == "" {
+			return errors.New("secret_id or secret_key is empty")
+		}
+		return nil
+	}
 	if conf.AppId == "" ||
 		conf.Region == "" ||
 		conf.SecretId == "" ||

pkg/objstore/cos/cos_test.go

@@ -62,3 +62,76 @@ http_config:
 		})
 	}
 }
+
+func TestConfig_validate(t *testing.T) {
+	type fields struct {
+		Bucket     string
+		Region     string
+		AppId      string
+		Endpoint   string
+		SecretKey  string
+		SecretId   string
+		HTTPConfig HTTPConfig
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantErr bool
+	}{
+		{
+			name: "ok endpoint",
+			fields: fields{
+				Endpoint:  "http://bucket-123.cos.ap-beijing.myqcloud.com",
+				SecretId:  "sid",
+				SecretKey: "skey",
+			},
+			wantErr: false,
+		},
+		{
+			name: "ok bucket-appid-region",
+			fields: fields{
+				Bucket:    "bucket",
+				AppId:     "123",
+				Region:    "ap-beijing",
+				SecretId:  "sid",
+				SecretKey: "skey",
+			},
+			wantErr: false,
+		},
+		{
+			name: "missing skey",
+			fields: fields{
+				Bucket: "bucket",
+				AppId:  "123",
+				Region: "ap-beijing",
+			},
+			wantErr: true,
+		},
+		{
+			name: "missing bucket",
+			fields: fields{
+				AppId:     "123",
+				Region:    "ap-beijing",
+				SecretId:  "sid",
+				SecretKey: "skey",
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			conf := &Config{
+				Bucket:     tt.fields.Bucket,
+				Region:     tt.fields.Region,
+				AppId:      tt.fields.AppId,
+				Endpoint:   tt.fields.Endpoint,
+				SecretKey:  tt.fields.SecretKey,
+				SecretId:   tt.fields.SecretId,
+				HTTPConfig: tt.fields.HTTPConfig,
+			}
+			if err := conf.validate(); (err != nil) != tt.wantErr {
+				t.Errorf("validate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}