Allow to disable http2 for GCS.

CHANGELOG.md

@@ -20,6 +20,7 @@
 * [4892](https://github.com/grafana/loki/pull/4892) **cristaloleg**: Loki: upgrade cristalhq/hedgedhttp from v0.6.0 to v0.7.0
 * [4902](https://github.com/grafana/loki/pull/4902) **cyriltovena**: Fixes 500 when query is outside of max_query_lookback.
 * [4904](https://github.com/grafana/loki/pull/4904) **bboreham**: Fixes rare race condition that could crash an ingester.
+* [4942](https://github.com/grafana/loki/pull/4942) **cyriltovena**: Allow to disable http2 for GCS.
 
 # 2.4.1 (2021/11/07)
 

docs/sources/configuration/_index.md

@@ -730,6 +730,10 @@ The `gcs_storage_config` configures GCS as a general storage for different data
 # The duration after which the requests to GCS should be timed out.
 # CLI flag: -<prefix>.gcs.request-timeout
 [request_timeout: <duration> | default = 0s]
+
+# Enable http2 when connecting to GCS.
+# CLI flag: -<prefix>.gcs.enable-http2
+[enable_http2: <bool> | default = true]
 ```
 
 ## s3_storage_config

pkg/storage/chunk/aws/s3_storage_client.go

@@ -299,9 +299,9 @@ func buildS3Client(cfg S3Config, hedgingCfg hedging.Config, hedging bool) (*s3.S
 			KeepAlive: 30 * time.Second,
 			DualStack: true,
 		}).DialContext,
-		MaxIdleConns:          100,
+		MaxIdleConns:          512,
 		IdleConnTimeout:       cfg.HTTPConfig.IdleConnTimeout,
-		MaxIdleConnsPerHost:   100,
+		MaxIdleConnsPerHost:   256,
 		TLSHandshakeTimeout:   3 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 		ResponseHeaderTimeout: cfg.HTTPConfig.ResponseHeaderTimeout,

pkg/storage/chunk/azure/blob_storage_client.go

@@ -68,8 +68,8 @@ var (
 					KeepAlive: 30 * time.Second,
 					DualStack: true,
 				}).Dial,
-				MaxIdleConns:           0,
-				MaxIdleConnsPerHost:    100,
+				MaxIdleConns:           512,
+				MaxIdleConnsPerHost:    256,
 				IdleConnTimeout:        90 * time.Second,
 				TLSHandshakeTimeout:    10 * time.Second,
 				ExpectContinueTimeout:  1 * time.Second,
@@ -292,7 +292,6 @@ func (b *BlobStorage) newPipeline(hedgingCfg hedging.Config, hedging bool) (pipe
 	}
 
 	return azblob.NewPipeline(*tokenCredential, opts), nil
-
 }
 
 func (b *BlobStorage) getOAuthToken() (*azblob.TokenCredential, error) {
@@ -331,7 +330,6 @@ func (b *BlobStorage) fetchMSIToken() (*adal.ServicePrincipalToken, error) {
 
 	// both can be empty, systemAssignedMSI scenario
 	spt, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, "https://storage.azure.com/")
-
 	if err != nil {
 		return nil, err
 	}

pkg/storage/chunk/gcp/gcs_object_client.go

@@ -33,6 +33,7 @@ type GCSConfig struct {
 	ChunkBufferSize  int           `yaml:"chunk_buffer_size"`
 	RequestTimeout   time.Duration `yaml:"request_timeout"`
 	EnableOpenCensus bool          `yaml:"enable_opencensus"`
+	EnableHTTP2      bool          `yaml:"enable_http2"`
 
 	Insecure bool `yaml:"-"`
 }
@@ -48,6 +49,7 @@ func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&cfg.ChunkBufferSize, prefix+"gcs.chunk-buffer-size", 0, "The size of the buffer that GCS client for each PUT request. 0 to disable buffering.")
 	f.DurationVar(&cfg.RequestTimeout, prefix+"gcs.request-timeout", 0, "The duration after which the requests to GCS should be timed out.")
 	f.BoolVar(&cfg.EnableOpenCensus, prefix+"gcs.enable-opencensus", true, "Enabled OpenCensus (OC) instrumentation for all requests.")
+	f.BoolVar(&cfg.EnableHTTP2, prefix+"gcs.enable-http2", true, "Enabled HTTP2 connections.")
 }
 
 func (cfg *GCSConfig) ToCortexGCSConfig() cortex_gcp.GCSConfig {
@@ -82,7 +84,7 @@ func newGCSObjectClient(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.C
 
 func newBucketHandle(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.Config, hedging bool, clientFactory ClientFactory) (*storage.BucketHandle, error) {
 	var opts []option.ClientOption
-	httpClient, err := gcsInstrumentation(ctx, storage.ScopeReadWrite, cfg.Insecure)
+	httpClient, err := gcsInstrumentation(ctx, storage.ScopeReadWrite, cfg.Insecure, cfg.EnableHTTP2)
 	if err != nil {
 		return nil, err
 	}

pkg/storage/chunk/gcp/instrumentation.go

@@ -50,9 +50,15 @@ func bigtableInstrumentation() ([]grpc.UnaryClientInterceptor, []grpc.StreamClie
 		}
 }
 
-func gcsInstrumentation(ctx context.Context, scope string, insecure bool) (*http.Client, error) {
+func gcsInstrumentation(ctx context.Context, scope string, insecure bool, http2 bool) (*http.Client, error) {
 	// start with default transport
 	customTransport := http.DefaultTransport.(*http.Transport).Clone()
+	customTransport.MaxIdleConnsPerHost = 256
+	customTransport.MaxIdleConns = 512
+	if !http2 {
+		// disable HTTP/2 by setting TLSNextProto to non-nil empty map, as per the net/http documentation.
+		customTransport.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper)
+	}
 	if insecure {
 		customTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}

pkg/storage/chunk/openstack/swift_object_client.go

@@ -24,7 +24,8 @@ import (
 
 var defaultTransport http.RoundTripper = &http.Transport{
 	Proxy:                 http.ProxyFromEnvironment,
-	MaxIdleConnsPerHost:   512,
+	MaxIdleConnsPerHost:   256,
+	MaxIdleConns:          512,
 	ExpectContinueTimeout: 5 * time.Second,
 }
 

pkg/storage/chunk/util/parallel_chunk_fetch.go

@@ -11,7 +11,7 @@ import (
 	"github.com/grafana/loki/pkg/storage/chunk"
 )
 
-const maxParallel = 1000
+const maxParallel = 150
 
 var decodeContextPool = sync.Pool{
 	New: func() interface{} {