diff --git a/config/config.go b/config/config.go index 2f302cc6328..943d18a95de 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,7 @@ type Configuration struct { Host string `mapstructure:"host"` Port int `mapstructure:"port"` Client HTTPClient `mapstructure:"http_client"` + CacheClient HTTPClient `mapstructure:"http_client_cache"` AdminPort int `mapstructure:"admin_port"` EnableGzip bool `mapstructure:"enable_gzip"` // StatusResponse is the string which will be returned by the /status endpoint when things are OK. @@ -583,6 +584,9 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("http_client.max_idle_connections", 400) v.SetDefault("http_client.max_idle_connections_per_host", 10) v.SetDefault("http_client.idle_connection_timeout_seconds", 60) + v.SetDefault("http_client_cache.max_idle_connections", 10) + v.SetDefault("http_client_cache.max_idle_connections_per_host", 2) + v.SetDefault("http_client_cache.idle_connection_timeout_seconds", 60) // no metrics configured by default (metrics{host|database|username|password}) v.SetDefault("metrics.disabled_metrics.account_adapter_details", false) v.SetDefault("metrics.influxdb.host", "") diff --git a/config/config_test.go b/config/config_test.go index 182a46eef50..78630e071d9 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -68,6 +68,10 @@ http_client: max_idle_connections: 500 max_idle_connections_per_host: 20 idle_connection_timeout_seconds: 30 +http_client_cache: + max_idle_connections: 1 + max_idle_connections_per_host: 2 + idle_connection_timeout_seconds: 3 currency_converter: fetch_url: https://currency.prebid.org fetch_interval_seconds: 1800 @@ -214,6 +218,9 @@ func TestFullConfig(t *testing.T) { cmpInts(t, "http_client.max_idle_connections", cfg.Client.MaxIdleConns, 500) cmpInts(t, "http_client.max_idle_connections_per_host", cfg.Client.MaxIdleConnsPerHost, 20) cmpInts(t, "http_client.idle_connection_timeout_seconds", cfg.Client.IdleConnTimeout, 30) + cmpInts(t, "http_client_cache.max_idle_connections", cfg.CacheClient.MaxIdleConns, 1) + cmpInts(t, "http_client_cache.max_idle_connections_per_host", cfg.CacheClient.MaxIdleConnsPerHost, 2) + cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", cfg.CacheClient.IdleConnTimeout, 3) cmpInts(t, "gdpr.host_vendor_id", cfg.GDPR.HostVendorID, 15) cmpBools(t, "gdpr.usersync_if_ambiguous", cfg.GDPR.UsersyncIfAmbiguous, true) diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 31dddae4c74..b8a3ae0eae2 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -170,7 +170,7 @@ func TestGetBidCacheInfo(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(handlerNoBidServer)) defer server.Close() - e := NewExchange(server.Client(), pbc.NewClient(&cfg.CacheURL, &cfg.ExtCacheURL, testEngine), cfg, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), adapters.ParseBidderInfos(cfg.Adapters, "../static/bidder-info", openrtb_ext.BidderList()), gdpr.AlwaysAllow{}, currencies.NewRateConverterDefault()).(*exchange) + e := NewExchange(server.Client(), pbc.NewClient(&http.Client{}, &cfg.CacheURL, &cfg.ExtCacheURL, testEngine), cfg, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), adapters.ParseBidderInfos(cfg.Adapters, "../static/bidder-info", openrtb_ext.BidderList()), gdpr.AlwaysAllow{}, currencies.NewRateConverterDefault()).(*exchange) /* 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs */ liveAdapters := []openrtb_ext.BidderName{bidderName} diff --git a/prebid_cache_client/client.go b/prebid_cache_client/client.go index 58e2734ed25..a5730ce7914 100644 --- a/prebid_cache_client/client.go +++ b/prebid_cache_client/client.go @@ -47,14 +47,9 @@ type Cacheable struct { Key string } -func NewClient(conf *config.Cache, extCache *config.ExternalCache, metrics pbsmetrics.MetricsEngine) Client { +func NewClient(httpClient *http.Client, conf *config.Cache, extCache *config.ExternalCache, metrics pbsmetrics.MetricsEngine) Client { return &clientImpl{ - httpClient: &http.Client{ - Transport: &http.Transport{ - MaxIdleConns: 10, - IdleConnTimeout: 65, - }, - }, + httpClient: httpClient, putUrl: conf.GetBaseURL() + "/cache", externalCacheHost: extCache.Host, externalCachePath: extCache.Path, diff --git a/prebid_cache_client/client_test.go b/prebid_cache_client/client_test.go index 1b5b4e38967..5840d4ea564 100644 --- a/prebid_cache_client/client_test.go +++ b/prebid_cache_client/client_test.go @@ -233,11 +233,9 @@ func TestStripCacheHostAndPath(t *testing.T) { }, } for _, test := range testInput { - //start client - cacheClient := NewClient(&inCacheURL, &test.inExtCacheURL, &metricsConf.DummyMetricsEngine{}) + cacheClient := NewClient(&http.Client{}, &inCacheURL, &test.inExtCacheURL, &metricsConf.DummyMetricsEngine{}) cHost, cPath := cacheClient.GetExtCacheData() - //assert assert.Equal(t, test.expectedHost, cHost) assert.Equal(t, test.expectedPath, cPath) } diff --git a/router/router.go b/router/router.go index 1994639110c..449ab65a448 100644 --- a/router/router.go +++ b/router/router.go @@ -183,7 +183,7 @@ func New(cfg *config.Configuration, rateConvertor *currencies.RateConverter) (r glog.Infof("Could not read certificates file: %s \n", readCertErr.Error()) } - theClient := &http.Client{ + generalHttpClient := &http.Client{ Transport: &http.Transport{ MaxIdleConns: cfg.Client.MaxIdleConns, MaxIdleConnsPerHost: cfg.Client.MaxIdleConnsPerHost, @@ -191,13 +191,22 @@ func New(cfg *config.Configuration, rateConvertor *currencies.RateConverter) (r TLSClientConfig: &tls.Config{RootCAs: certPool}, }, } + + cacheHttpClient := &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: cfg.CacheClient.MaxIdleConns, + MaxIdleConnsPerHost: cfg.CacheClient.MaxIdleConnsPerHost, + IdleConnTimeout: time.Duration(cfg.CacheClient.IdleConnTimeout) * time.Second, + }, + } + // Hack because of how legacy handles districtm legacyBidderList := openrtb_ext.BidderList() legacyBidderList = append(legacyBidderList, openrtb_ext.BidderName("districtm")) // Metrics engine r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList) - db, shutdown, fetcher, ampFetcher, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, theClient, r.Router) + db, shutdown, fetcher, ampFetcher, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) // todo(zachbadgett): better shutdown r.Shutdown = shutdown @@ -223,10 +232,11 @@ func New(cfg *config.Configuration, rateConvertor *currencies.RateConverter) (r defaultAliases, defReqJSON := readDefaultRequest(cfg.DefReqConfig) syncers := usersyncers.NewSyncerMap(cfg) - gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, adapters.GDPRAwareSyncerIDs(syncers), theClient) + gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, adapters.GDPRAwareSyncerIDs(syncers), generalHttpClient) exchanges = newExchangeMap(cfg) - theExchange := exchange.NewExchange(theClient, pbc.NewClient(&cfg.CacheURL, &cfg.ExtCacheURL, r.MetricsEngine), cfg, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor) + cacheClient := pbc.NewClient(cacheHttpClient, &cfg.CacheURL, &cfg.ExtCacheURL, r.MetricsEngine) + theExchange := exchange.NewExchange(generalHttpClient, cacheClient, cfg, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor) openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, fetcher, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap)