From 72be0a4a8556287c283653b7e909f89e0cd5ba48 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 20 Apr 2021 17:03:11 -0400 Subject: [PATCH 01/50] Implement New Bidder Chooser --- adapters/adform/adform_test.go | 2 +- adapters/appnexus/appnexus_test.go | 2 +- adapters/conversant/cnvr_legacy_test.go | 2 +- adapters/lifestreet/lifestreet_test.go | 2 +- adapters/openrtb_util_test.go | 2 +- adapters/pubmatic/pubmatic_test.go | 2 +- adapters/pulsepoint/pulsepoint_test.go | 2 +- adapters/rubicon/rubicon_test.go | 20 +- adapters/sovrn/sovrn_test.go | 2 +- config/usersync.go | 6 + endpoints/cookie_sync.go | 4 +- endpoints/cookie_sync_test.go | 11 +- endpoints/getuids.go | 2 +- endpoints/openrtb2/amp_auction.go | 2 +- endpoints/openrtb2/auction.go | 2 +- endpoints/openrtb2/video_auction.go | 2 +- endpoints/setuid.go | 2 +- endpoints/setuid_test.go | 22 +- exchange/legacy.go | 2 +- pbs/pbsrequest.go | 14 +- pbs/usersync.go | 2 +- usersync/bidderchooser.go | 58 +++++ usersync/bidderchooser_test.go | 270 ++++++++++++++++++++++++ usersync/cookie.go | 58 ++--- usersync/cookie_test.go | 48 ++--- usersync/shuffler.go | 13 ++ usersync/shuffler_test.go | 42 ++++ 27 files changed, 486 insertions(+), 110 deletions(-) create mode 100644 config/usersync.go create mode 100644 usersync/bidderchooser.go create mode 100644 usersync/bidderchooser_test.go create mode 100644 usersync/shuffler.go create mode 100644 usersync/shuffler_test.go diff --git a/adapters/adform/adform_test.go b/adapters/adform/adform_test.go index 558453e4030..90d958c0f23 100644 --- a/adapters/adform/adform_test.go +++ b/adapters/adform/adform_test.go @@ -222,7 +222,7 @@ func preparePrebidRequest(serverUrl string, t *testing.T) *pbs.PBSRequest { prebidHttpRequest.Header.Add("Referer", adformTestData.referrer) prebidHttpRequest.Header.Add("X-Real-IP", adformTestData.deviceIP) - pbsCookie := usersync.ParsePBSCookieFromRequest(prebidHttpRequest, &config.HostCookie{}) + pbsCookie := usersync.ParseCookieFromRequest(prebidHttpRequest, &config.HostCookie{}) pbsCookie.TrySync("adform", adformTestData.buyerUID) fakeWriter := httptest.NewRecorder() diff --git a/adapters/appnexus/appnexus_test.go b/adapters/appnexus/appnexus_test.go index d453a779854..967ad767f27 100644 --- a/adapters/appnexus/appnexus_test.go +++ b/adapters/appnexus/appnexus_test.go @@ -610,7 +610,7 @@ func TestAppNexusLegacyBasicResponse(t *testing.T) { req.Header.Add("User-Agent", andata.deviceUA) req.Header.Add("X-Real-IP", andata.deviceIP) - pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(req, &config.HostCookie{}) pc.TrySync("adnxs", andata.buyerUID) fakewriter := httptest.NewRecorder() diff --git a/adapters/conversant/cnvr_legacy_test.go b/adapters/conversant/cnvr_legacy_test.go index 712f85f4404..8b7188c8390 100644 --- a/adapters/conversant/cnvr_legacy_test.go +++ b/adapters/conversant/cnvr_legacy_test.go @@ -691,7 +691,7 @@ func ParseRequest(req *pbs.PBSRequest) (*pbs.PBSRequest, error) { // Need to pass the conversant user id thru uid cookie httpReq := httptest.NewRequest("POST", "/foo", body) - cookie := usersync.NewPBSCookie() + cookie := usersync.NewCookie() _ = cookie.TrySync("conversant", ExpectedBuyerUID) httpReq.Header.Set("Cookie", cookie.ToHTTPCookie(90*24*time.Hour).String()) httpReq.Header.Add("Referer", "http://example.com") diff --git a/adapters/lifestreet/lifestreet_test.go b/adapters/lifestreet/lifestreet_test.go index 412333c3714..19f18a4c401 100644 --- a/adapters/lifestreet/lifestreet_test.go +++ b/adapters/lifestreet/lifestreet_test.go @@ -225,7 +225,7 @@ func TestLifestreetBasicResponse(t *testing.T) { req.Header.Add("Referer", lsdata.referrer) req.Header.Add("X-Real-IP", lsdata.deviceIP) - pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(req, &config.HostCookie{}) fakewriter := httptest.NewRecorder() pc.SetCookieOnResponse(fakewriter, false, &config.HostCookie{Domain: ""}, 90*24*time.Hour) diff --git a/adapters/openrtb_util_test.go b/adapters/openrtb_util_test.go index 2cf67c22537..0bbadb55a85 100644 --- a/adapters/openrtb_util_test.go +++ b/adapters/openrtb_util_test.go @@ -341,7 +341,7 @@ func TestOpenRTBEmptyUser(t *testing.T) { } func TestOpenRTBUserWithCookie(t *testing.T) { - pbsCookie := usersync.NewPBSCookie() + pbsCookie := usersync.NewCookie() pbsCookie.TrySync("test", "abcde") pbReq := pbs.PBSRequest{ User: &openrtb.User{}, diff --git a/adapters/pubmatic/pubmatic_test.go b/adapters/pubmatic/pubmatic_test.go index 8ed2ccd391c..2608e92f93c 100644 --- a/adapters/pubmatic/pubmatic_test.go +++ b/adapters/pubmatic/pubmatic_test.go @@ -661,7 +661,7 @@ func TestPubmaticSampleRequest(t *testing.T) { httpReq := httptest.NewRequest("POST", server.URL, body) httpReq.Header.Add("Referer", "http://test.com/sports") - pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("pubmatic", "12345") fakewriter := httptest.NewRecorder() diff --git a/adapters/pulsepoint/pulsepoint_test.go b/adapters/pulsepoint/pulsepoint_test.go index 33023d0500a..61504b1c25f 100644 --- a/adapters/pulsepoint/pulsepoint_test.go +++ b/adapters/pulsepoint/pulsepoint_test.go @@ -239,7 +239,7 @@ func SampleRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { // setup a http request httpReq := httptest.NewRequest("POST", CreateService(adapterstest.BidOnTags("")).Server.URL, body) httpReq.Header.Add("Referer", "http://news.pub/topnews") - pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("pulsepoint", "pulsepointUser123") fakewriter := httptest.NewRecorder() diff --git a/adapters/rubicon/rubicon_test.go b/adapters/rubicon/rubicon_test.go index 41e37f41126..23fb28abc62 100644 --- a/adapters/rubicon/rubicon_test.go +++ b/adapters/rubicon/rubicon_test.go @@ -4,28 +4,24 @@ import ( "bytes" "context" "encoding/json" - "github.com/prebid/prebid-server/errortypes" + "fmt" "io/ioutil" "net/http" "net/http/httptest" "strconv" + "strings" "testing" "time" - "github.com/prebid/prebid-server/adapters/adapterstest" - "github.com/prebid/prebid-server/cache/dummycache" - "github.com/prebid/prebid-server/pbs" - "github.com/prebid/prebid-server/usersync" - - "fmt" - - "strings" - "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/adapters/adapterstest" + "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - + "github.com/prebid/prebid-server/pbs" + "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" ) @@ -905,7 +901,7 @@ func CreatePrebidRequest(server *httptest.Server, t *testing.T) (an *RubiconAdap req.Header.Add("User-Agent", rubidata.deviceUA) req.Header.Add("X-Real-IP", rubidata.deviceIP) - pc := usersync.ParsePBSCookieFromRequest(req, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(req, &config.HostCookie{}) pc.TrySync("rubicon", rubidata.buyerUID) fakewriter := httptest.NewRecorder() diff --git a/adapters/sovrn/sovrn_test.go b/adapters/sovrn/sovrn_test.go index aa1b98f00f5..69a8c0d0f7c 100644 --- a/adapters/sovrn/sovrn_test.go +++ b/adapters/sovrn/sovrn_test.go @@ -192,7 +192,7 @@ func SampleSovrnRequest(numberOfImpressions int, t *testing.T) *pbs.PBSRequest { httpReq.Header.Add("Referer", testUrl) httpReq.Header.Add("User-Agent", testUserAgent) httpReq.Header.Add("X-Forwarded-For", testIp) - pc := usersync.ParsePBSCookieFromRequest(httpReq, &config.HostCookie{}) + pc := usersync.ParseCookieFromRequest(httpReq, &config.HostCookie{}) pc.TrySync("sovrn", testSovrnUserId) fakewriter := httptest.NewRecorder() diff --git a/config/usersync.go b/config/usersync.go new file mode 100644 index 00000000000..7940c5eb885 --- /dev/null +++ b/config/usersync.go @@ -0,0 +1,6 @@ +package config + +type UserSyncCooperative struct { + Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"` + PriorityGroups [][]string `mapstructure:"priorityGroups" json:"priorityGroups,omitempty"` +} diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index bf3935f0535..cf04176ddd9 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -72,7 +72,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h defer deps.pbsAnalytics.LogCookieSyncObject(&co) deps.metrics.RecordCookieSync() - userSyncCookie := usersync.ParsePBSCookieFromRequest(r, deps.hostCookie) + userSyncCookie := usersync.ParseCookieFromRequest(r, deps.hostCookie) if !userSyncCookie.AllowSyncs() { http.Error(w, "User has opted out", http.StatusUnauthorized) co.Status = http.StatusUnauthorized @@ -233,7 +233,7 @@ type cookieSyncRequest struct { Limit int `json:"limit"` } -func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderName]usersync.Usersyncer, cookie *usersync.PBSCookie, needSyncupForSameSite bool) { +func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderName]usersync.Usersyncer, cookie *usersync.Cookie, needSyncupForSameSite bool) { for i := 0; i < len(req.Bidders); i++ { thisBidder := req.Bidders[i] if syncer, isValid := valid[openrtb_ext.BidderName(thisBidder)]; !isValid || (cookie.HasLiveSync(syncer.FamilyName()) && !needSyncupForSameSite) { diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index a6352387786..201317da381 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -140,15 +140,6 @@ func TestCookieSyncEmptyBidders(t *testing.T) { assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) } -// Make sure that all syncs are returned if "bidders" isn't a key -func TestCookieSyncNoBidders(t *testing.T) { - rr := doPost("{}", nil, true, syncersForTest()) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"appnexus", "audienceNetwork", "lifestreet", "pubmatic"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} - func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) { rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true}, config.CCPA{}) assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") @@ -183,7 +174,7 @@ func doConfigurablePost(body string, existingSyncs map[string]string, gdprHostCo req, _ := http.NewRequest("POST", "/cookie_sync", strings.NewReader(body)) if len(existingSyncs) > 0 { - pcs := usersync.NewPBSCookie() + pcs := usersync.NewCookie() for bidder, uid := range existingSyncs { pcs.TrySync(bidder, uid) } diff --git a/endpoints/getuids.go b/endpoints/getuids.go index 859c0e7288c..ad984a8df00 100644 --- a/endpoints/getuids.go +++ b/endpoints/getuids.go @@ -18,7 +18,7 @@ type userSyncs struct { // returns all the existing syncs for the user func NewGetUIDsEndpoint(cfg config.HostCookie) httprouter.Handle { return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - pc := usersync.ParsePBSCookieFromRequest(r, &cfg) + pc := usersync.ParseCookieFromRequest(r, &cfg) userSyncs := new(userSyncs) userSyncs.BuyerUIDs = pc.GetUIDs() json.NewEncoder(w).Encode(userSyncs) diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 1f3d622b80d..ce1eaa1c33d 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -152,7 +152,7 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h } defer cancel() - usersyncs := usersync.ParsePBSCookieFromRequest(r, &(deps.cfg.HostCookie)) + usersyncs := usersync.ParseCookieFromRequest(r, &(deps.cfg.HostCookie)) if usersyncs.LiveSyncCount() == 0 { labels.CookieFlag = metrics.CookieFlagNo } else { diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index d26623da348..52be86c4596 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -150,7 +150,7 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http defer cancel() } - usersyncs := usersync.ParsePBSCookieFromRequest(r, &(deps.cfg.HostCookie)) + usersyncs := usersync.ParseCookieFromRequest(r, &(deps.cfg.HostCookie)) if req.App != nil { labels.Source = metrics.DemandApp labels.RType = metrics.ReqTypeORTB2App diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index 7735d886730..b4b21d7d1b9 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -253,7 +253,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re defer cancel() } - usersyncs := usersync.ParsePBSCookieFromRequest(r, &(deps.cfg.HostCookie)) + usersyncs := usersync.ParseCookieFromRequest(r, &(deps.cfg.HostCookie)) if bidReq.App != nil { labels.Source = metrics.DemandApp labels.PubID = getAccountID(bidReq.App.Publisher) diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 4bff02acf37..0074ed63e3f 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -42,7 +42,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[openrtb_ext.BidderName defer pbsanalytics.LogSetUIDObject(&so) - pc := usersync.ParsePBSCookieFromRequest(r, &cfg) + pc := usersync.ParseCookieFromRequest(r, &cfg) if !pc.AllowSyncs() { w.WriteHeader(http.StatusUnauthorized) metricsEngine.RecordUserIDSet(metrics.UserLabels{ diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 87c98f23ad1..54e3c915dba 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -203,7 +203,7 @@ func TestSetUIDEndpoint(t *testing.T) { func TestSetUIDEndpointMetrics(t *testing.T) { testCases := []struct { uri string - cookies []*usersync.PBSCookie + cookies []*usersync.Cookie validFamilyNames []string gdprAllowsHostCookies bool expectedMetricAction metrics.RequestAction @@ -213,7 +213,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }{ { uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.PBSCookie{}, + cookies: []*usersync.Cookie{}, validFamilyNames: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionSet, @@ -223,7 +223,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }, { uri: "/setuid?bidder=pubmatic&uid=", - cookies: []*usersync.PBSCookie{}, + cookies: []*usersync.Cookie{}, validFamilyNames: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionSet, @@ -233,7 +233,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }, { uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.PBSCookie{usersync.NewPBSCookieWithOptOut()}, + cookies: []*usersync.Cookie{usersync.NewPBSCookieWithOptOut()}, validFamilyNames: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionOptOut, @@ -242,7 +242,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }, { uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.PBSCookie{}, + cookies: []*usersync.Cookie{}, validFamilyNames: []string{}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionErr, @@ -251,7 +251,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1", - cookies: []*usersync.PBSCookie{}, + cookies: []*usersync.Cookie{}, validFamilyNames: []string{"pubmatic"}, gdprAllowsHostCookies: false, expectedMetricAction: metrics.RequestActionGDPR, @@ -282,7 +282,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { func TestOptedOut(t *testing.T) { request := httptest.NewRequest("GET", "/setuid?bidder=pubmatic&uid=123", nil) - cookie := usersync.NewPBSCookie() + cookie := usersync.NewCookie() cookie.SetPreference(false) addCookie(request, cookie) validFamilyNames := []string{"pubmatic"} @@ -377,7 +377,7 @@ func assertHasSyncs(t *testing.T, testCase string, resp *httptest.ResponseRecord func makeRequest(uri string, existingSyncs map[string]string) *http.Request { request := httptest.NewRequest("GET", uri, nil) if len(existingSyncs) > 0 { - pbsCookie := usersync.NewPBSCookie() + pbsCookie := usersync.NewCookie() for family, value := range existingSyncs { pbsCookie.TrySync(family, value) } @@ -405,11 +405,11 @@ func doRequest(req *http.Request, metrics metrics.MetricsEngine, validFamilyName return response } -func addCookie(req *http.Request, cookie *usersync.PBSCookie) { +func addCookie(req *http.Request, cookie *usersync.Cookie) { req.AddCookie(cookie.ToHTTPCookie(time.Duration(1) * time.Hour)) } -func parseCookieString(t *testing.T, response *httptest.ResponseRecorder) *usersync.PBSCookie { +func parseCookieString(t *testing.T, response *httptest.ResponseRecorder) *usersync.Cookie { cookieString := response.Header().Get("Set-Cookie") parser := regexp.MustCompile("uids=(.*?);") res := parser.FindStringSubmatch(cookieString) @@ -418,7 +418,7 @@ func parseCookieString(t *testing.T, response *httptest.ResponseRecorder) *users Name: "uids", Value: res[1], } - return usersync.ParsePBSCookie(&httpCookie) + return usersync.ParseCookie(&httpCookie) } type mockPermsSetUID struct { diff --git a/exchange/legacy.go b/exchange/legacy.go index b4845b76c69..8626f7c1762 100644 --- a/exchange/legacy.go +++ b/exchange/legacy.go @@ -100,7 +100,7 @@ func (bidder *adaptedAdapter) toLegacyRequest(req *openrtb.BidRequest) (*pbs.PBS domain = req.Site.Domain } - cookie := usersync.NewPBSCookie() + cookie := usersync.NewCookie() if req.User != nil { if req.User.BuyerUID != "" { cookie.TrySync(bidder.adapter.Name(), req.User.BuyerUID) diff --git a/pbs/pbsrequest.go b/pbs/pbsrequest.go index 30f8bd25c0d..8da6c2f51a8 100644 --- a/pbs/pbsrequest.go +++ b/pbs/pbsrequest.go @@ -168,12 +168,12 @@ type PBSRequest struct { SDK *SDK `json:"sdk"` // internal - Bidders []*PBSBidder `json:"-"` - User *openrtb.User `json:"-"` - Cookie *usersync.PBSCookie `json:"-"` - Url string `json:"-"` - Domain string `json:"-"` - Regs *openrtb.Regs `json:"-"` + Bidders []*PBSBidder `json:"-"` + User *openrtb.User `json:"-"` + Cookie *usersync.Cookie `json:"-"` + Url string `json:"-"` + Domain string `json:"-"` + Regs *openrtb.Regs `json:"-"` Start time.Time } @@ -264,7 +264,7 @@ func ParsePBSRequest(r *http.Request, cfg *config.AuctionTimeouts, cache cache.C // use client-side data for web requests if pbsReq.App == nil { - pbsReq.Cookie = usersync.ParsePBSCookieFromRequest(r, hostCookieConfig) + pbsReq.Cookie = usersync.ParseCookieFromRequest(r, hostCookieConfig) pbsReq.Device.UA = r.Header.Get("User-Agent") diff --git a/pbs/usersync.go b/pbs/usersync.go index 4cac3544804..2af0708bd3d 100644 --- a/pbs/usersync.go +++ b/pbs/usersync.go @@ -92,7 +92,7 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr return } - pc := usersync.ParsePBSCookieFromRequest(r, deps.HostCookieConfig) + pc := usersync.ParseCookieFromRequest(r, deps.HostCookieConfig) pc.SetPreference(optout == "") pc.SetCookieOnResponse(w, false, deps.HostCookieConfig, deps.HostCookieConfig.TTLDuration()) diff --git a/usersync/bidderchooser.go b/usersync/bidderchooser.go new file mode 100644 index 00000000000..1bed3256703 --- /dev/null +++ b/usersync/bidderchooser.go @@ -0,0 +1,58 @@ +package usersync + +import "github.com/prebid/prebid-server/config" + +type bidderChooser interface { + choose(requested, available []string, cooperative config.UserSyncCooperative) []string +} + +type randomBidderChooser struct { + shuffler shuffler +} + +func (c randomBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { + if requested == nil { + return c.shuffledCopy(available) + } + + if cooperative.Enabled { + return c.chooseCooperative(requested, available, cooperative.PriorityGroups) + } + + return c.shuffledCopy(requested) +} + +func (c randomBidderChooser) chooseCooperative(requested, available []string, priorityGroups [][]string) []string { + biddersCapacity := int(float64(len(available)) * 1.5) + bidders := make([]string, 0, biddersCapacity) + + // requested + bidders = c.shuffledAppend(bidders, requested) + + // priority groups + for _, group := range priorityGroups { + bidders = c.shuffledAppend(bidders, group) + } + + // available + bidders = c.shuffledAppend(bidders, available) + + return bidders +} + +func (c randomBidderChooser) shuffledCopy(a []string) []string { + if a == nil { + return nil + } + aCopy := make([]string, len(a)) + copy(aCopy, a) + c.shuffler.shuffle(aCopy) + return aCopy +} + +func (c randomBidderChooser) shuffledAppend(a, b []string) []string { + startIndex := len(a) + a = append(a, b...) + c.shuffler.shuffle(a[startIndex:]) + return a +} diff --git a/usersync/bidderchooser_test.go b/usersync/bidderchooser_test.go new file mode 100644 index 00000000000..c50abc1f96c --- /dev/null +++ b/usersync/bidderchooser_test.go @@ -0,0 +1,270 @@ +package usersync + +import ( + "testing" + + "github.com/prebid/prebid-server/config" + "github.com/stretchr/testify/assert" +) + +// These tests use a deterministic test version of the shuffler, where the order +// is reversed rather than randomized. + +func TestChoose(t *testing.T) { + shuffler := reverseShuffler{} + available := []string{"a", "b"} + + testCases := []struct { + description string + givenRequested []string + givenCooperative config.UserSyncCooperative + expected []string + }{ + { + description: "No Coop - Nil", + givenRequested: nil, + givenCooperative: config.UserSyncCooperative{Enabled: false}, + expected: []string{"b", "a"}, + }, + { + description: "No Coop - Empty", + givenRequested: []string{}, + givenCooperative: config.UserSyncCooperative{Enabled: false}, + expected: []string{}, + }, + { + description: "No Coop - One", + givenRequested: []string{"c"}, + givenCooperative: config.UserSyncCooperative{Enabled: false}, + expected: []string{"c"}, + }, + { + description: "No Coop - Many", + givenRequested: []string{"c", "d"}, + givenCooperative: config.UserSyncCooperative{Enabled: false}, + expected: []string{"d", "c"}, + }, + { + description: "Coop - Integration", + givenRequested: []string{"c", "d"}, + givenCooperative: config.UserSyncCooperative{Enabled: true, PriorityGroups: [][]string{{"1", "2"}, {"3", "4"}}}, + expected: []string{"d", "c", "2", "1", "4", "3", "b", "a"}, + }, + } + + for _, test := range testCases { + chooser := randomBidderChooser{shuffler: shuffler} + chosen := chooser.choose(test.givenRequested, available, test.givenCooperative) + + assert.Equal(t, test.expected, chosen, test.description) + } +} + +func TestChooseCooperative(t *testing.T) { + shuffler := reverseShuffler{} + available := []string{"a", "b"} + + testCases := []struct { + description string + givenRequested []string + givenPriorityGroups [][]string + expected []string + }{ + { + description: "Nil", + givenRequested: nil, + givenPriorityGroups: nil, + expected: []string{"b", "a"}, + }, + { + description: "Empty", + givenRequested: []string{}, + givenPriorityGroups: [][]string{}, + expected: []string{"b", "a"}, + }, + { + description: "Requested", + givenRequested: []string{"c", "d"}, + givenPriorityGroups: nil, + expected: []string{"d", "c", "b", "a"}, + }, + { + description: "Priority Groups - One", + givenRequested: nil, + givenPriorityGroups: [][]string{{"c", "d"}}, + expected: []string{"d", "c", "b", "a"}, + }, + { + description: "Priority Groups - Many", + givenRequested: nil, + givenPriorityGroups: [][]string{{"c", "d"}, {"e", "f", "g"}}, + expected: []string{"d", "c", "g", "f", "e", "b", "a"}, + }, + { + description: "Requested + Priority Groups", + givenRequested: []string{"c", "d"}, + givenPriorityGroups: [][]string{{"e", "f"}, {"g", "h", "i"}}, + expected: []string{"d", "c", "f", "e", "i", "h", "g", "b", "a"}, + }, + } + + for _, test := range testCases { + chooser := randomBidderChooser{shuffler: shuffler} + chosen := chooser.chooseCooperative(test.givenRequested, available, test.givenPriorityGroups) + + assert.Equal(t, test.expected, chosen, test.description) + } +} + +func TestShuffledCopy(t *testing.T) { + shuffler := reverseShuffler{} + + testCases := []struct { + description string + given []string + expected []string + }{ + { + description: "Nil", + given: nil, + expected: nil, + }, + { + description: "Empty", + given: []string{}, + expected: []string{}, + }, + { + description: "One", + given: []string{"a"}, + expected: []string{"a"}, + }, + { + description: "Many", + given: []string{"a", "b"}, + expected: []string{"b", "a"}, + }, + } + + for _, test := range testCases { + givenCopy := copySlice(test.given) + + chooser := randomBidderChooser{shuffler: shuffler} + shuffled := chooser.shuffledCopy(test.given) + + assert.Equal(t, givenCopy, test.given, test.description+":input unchanged") + assert.Equal(t, test.expected, shuffled, test.description+":expected") + } +} + +func TestShuffledAppend(t *testing.T) { + shuffler := reverseShuffler{} + + testCases := []struct { + description string + givenA []string + givenB []string + expected []string + }{ + { + description: "Empty - Append Nil", + givenA: []string{}, + givenB: nil, + expected: []string{}, + }, + { + description: "Empty - Append Empty", + givenA: []string{}, + givenB: []string{}, + expected: []string{}, + }, + { + description: "Empty - Append One", + givenA: []string{}, + givenB: []string{"1"}, + expected: []string{"1"}, + }, + { + description: "Empty - Append Many", + givenA: []string{}, + givenB: []string{"1", "2"}, + expected: []string{"2", "1"}, + }, + { + description: "One - Append Nil", + givenA: []string{"a"}, + givenB: nil, + expected: []string{"a"}, + }, + { + description: "One - Append Empty", + givenA: []string{"a"}, + givenB: []string{}, + expected: []string{"a"}, + }, + { + description: "One - Append One", + givenA: []string{"a"}, + givenB: []string{"1"}, + expected: []string{"a", "1"}, + }, + { + description: "One - Append Many", + givenA: []string{"a"}, + givenB: []string{"1", "2"}, + expected: []string{"a", "2", "1"}, + }, + { + description: "Many - Append Nil", + givenA: []string{"a", "b"}, + givenB: nil, + expected: []string{"a", "b"}, + }, + { + description: "Many - Append Empty", + givenA: []string{"a", "b"}, + givenB: []string{}, + expected: []string{"a", "b"}, + }, + { + description: "Many - Append One", + givenA: []string{"a", "b"}, + givenB: []string{"1"}, + expected: []string{"a", "b", "1"}, + }, + { + description: "Many - Append Many", + givenA: []string{"a", "b"}, + givenB: []string{"1", "2"}, + expected: []string{"a", "b", "2", "1"}, + }, + } + + for _, test := range testCases { + givenBCopy := copySlice(test.givenB) + + chooser := randomBidderChooser{shuffler: shuffler} + shuffled := chooser.shuffledAppend(test.givenA, test.givenB) + + assert.Equal(t, givenBCopy, test.givenB, test.description+":append input unchanged") + assert.Equal(t, test.expected, shuffled, test.description+":expected") + } +} + +// copySlice clones a slice with proper nil handling. +func copySlice(a []string) []string { + var aCopy []string + if a != nil { + aCopy = make([]string, len(a)) + copy(aCopy, a) + } + return aCopy +} + +type reverseShuffler struct{} + +func (reverseShuffler) shuffle(a []string) { + for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 { + a[i], a[j] = a[j], a[i] + } +} diff --git a/usersync/cookie.go b/usersync/cookie.go index c4e329b65e0..e96cdf7f8a8 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -31,11 +31,11 @@ var bidderToFamilyNames = map[openrtb_ext.BidderName]string{ openrtb_ext.BidderAppnexus: "adnxs", } -// PBSCookie is the cookie used in Prebid Server. +// Cookie is the cookie used in Prebid Server. // // To get an instance of this from a request, use ParsePBSCookieFromRequest. // To write an instance onto a response, use SetCookieOnResponse. -type PBSCookie struct { +type Cookie struct { uids map[string]uidWithExpiry optOut bool birthday *time.Time @@ -50,22 +50,22 @@ type uidWithExpiry struct { Expires time.Time `json:"expires"` } -// ParsePBSCookieFromRequest parses the UserSyncMap from an HTTP Request. -func ParsePBSCookieFromRequest(r *http.Request, cookie *config.HostCookie) *PBSCookie { +// ParseCookieFromRequest parses the UserSyncMap from an HTTP Request. +func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie { if cookie.OptOutCookie.Name != "" { optOutCookie, err1 := r.Cookie(cookie.OptOutCookie.Name) if err1 == nil && optOutCookie.Value == cookie.OptOutCookie.Value { - pc := NewPBSCookie() + pc := NewCookie() pc.SetPreference(false) return pc } } - var parsed *PBSCookie + var parsed *Cookie uidCookie, err2 := r.Cookie(UID_COOKIE_NAME) if err2 == nil { - parsed = ParsePBSCookie(uidCookie) + parsed = ParseCookie(uidCookie) } else { - parsed = NewPBSCookie() + parsed = NewCookie() } // Fixes #582 if uid, _, _ := parsed.GetUID(cookie.Family); uid == "" && cookie.CookieName != "" { @@ -76,9 +76,9 @@ func ParsePBSCookieFromRequest(r *http.Request, cookie *config.HostCookie) *PBSC return parsed } -// ParsePBSCookie parses the UserSync cookie from a raw HTTP cookie. -func ParsePBSCookie(uidCookie *http.Cookie) *PBSCookie { - pc := NewPBSCookie() +// ParseCookie parses the UserSync cookie from a raw HTTP cookie. +func ParseCookie(uidCookie *http.Cookie) *Cookie { + pc := NewCookie() j, err := base64.URLEncoding.DecodeString(uidCookie.Value) if err != nil { @@ -92,17 +92,17 @@ func ParsePBSCookie(uidCookie *http.Cookie) *PBSCookie { return pc } -// NewPBSCookie returns an empty PBSCookie -func NewPBSCookie() *PBSCookie { - return &PBSCookie{ +// NewCookie returns an empty PBSCookie +func NewCookie() *Cookie { + return &Cookie{ uids: make(map[string]uidWithExpiry), birthday: timestamp(), } } // NewPBSCookie returns an empty PBSCookie with optOut enabled -func NewPBSCookieWithOptOut() *PBSCookie { - return &PBSCookie{ +func NewPBSCookieWithOptOut() *Cookie { + return &Cookie{ uids: make(map[string]uidWithExpiry), optOut: true, birthday: timestamp(), @@ -110,12 +110,12 @@ func NewPBSCookieWithOptOut() *PBSCookie { } // AllowSyncs is true if the user lets bidders sync cookies, and false otherwise. -func (cookie *PBSCookie) AllowSyncs() bool { +func (cookie *Cookie) AllowSyncs() bool { return cookie != nil && !cookie.optOut } // SetPreference is used to change whether or not we're allowed to sync cookies for this user. -func (cookie *PBSCookie) SetPreference(allow bool) { +func (cookie *Cookie) SetPreference(allow bool) { if allow { cookie.optOut = false } else { @@ -125,7 +125,7 @@ func (cookie *PBSCookie) SetPreference(allow bool) { } // Gets an HTTP cookie containing all the data from this UserSyncMap. This is a snapshot--not a live view. -func (cookie *PBSCookie) ToHTTPCookie(ttl time.Duration) *http.Cookie { +func (cookie *Cookie) ToHTTPCookie(ttl time.Duration) *http.Cookie { j, _ := json.Marshal(cookie) b64 := base64.URLEncoding.EncodeToString(j) @@ -143,7 +143,7 @@ func (cookie *PBSCookie) ToHTTPCookie(ttl time.Duration) *http.Cookie { // The third returned value is true if that value is "active", and false if it's expired. // // If no value was stored, then the "isActive" return value will be false. -func (cookie *PBSCookie) GetUID(familyName string) (string, bool, bool) { +func (cookie *Cookie) GetUID(familyName string) (string, bool, bool) { if cookie != nil { if uid, ok := cookie.uids[familyName]; ok { return uid.UID, true, time.Now().Before(uid.Expires) @@ -153,7 +153,7 @@ func (cookie *PBSCookie) GetUID(familyName string) (string, bool, bool) { } // GetUIDs returns this user's ID for all the bidders -func (cookie *PBSCookie) GetUIDs() map[string]string { +func (cookie *Cookie) GetUIDs() map[string]string { uids := make(map[string]string) if cookie != nil { // Extract just the uid for each bidder @@ -165,7 +165,7 @@ func (cookie *PBSCookie) GetUIDs() map[string]string { } // GetId wraps GetUID, letting callers fetch the ID given an OpenRTB BidderName. -func (cookie *PBSCookie) GetId(bidderName openrtb_ext.BidderName) (id string, exists bool) { +func (cookie *Cookie) GetId(bidderName openrtb_ext.BidderName) (id string, exists bool) { if familyName, ok := bidderToFamilyNames[bidderName]; ok { id, exists, _ = cookie.GetUID(familyName) } else { @@ -175,7 +175,7 @@ func (cookie *PBSCookie) GetId(bidderName openrtb_ext.BidderName) (id string, ex } // SetCookieOnResponse is a shortcut for "ToHTTPCookie(); cookie.setDomain(domain); setCookie(w, cookie)" -func (cookie *PBSCookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie bool, cfg *config.HostCookie, ttl time.Duration) { +func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie bool, cfg *config.HostCookie, ttl time.Duration) { httpCookie := cookie.ToHTTPCookie(ttl) var domain string = cfg.Domain @@ -225,18 +225,18 @@ func (cookie *PBSCookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCooki } // Unsync removes the user's ID for the given family from this cookie. -func (cookie *PBSCookie) Unsync(familyName string) { +func (cookie *Cookie) Unsync(familyName string) { delete(cookie.uids, familyName) } // HasLiveSync returns true if we have an active UID for the given family, and false otherwise. -func (cookie *PBSCookie) HasLiveSync(familyName string) bool { +func (cookie *Cookie) HasLiveSync(familyName string) bool { _, _, isLive := cookie.GetUID(familyName) return isLive } // LiveSyncCount returns the number of families which have active UIDs for this user. -func (cookie *PBSCookie) LiveSyncCount() int { +func (cookie *Cookie) LiveSyncCount() int { now := time.Now() numSyncs := 0 if cookie != nil { @@ -250,7 +250,7 @@ func (cookie *PBSCookie) LiveSyncCount() int { } // TrySync tries to set the UID for some family name. It returns an error if the set didn't happen. -func (cookie *PBSCookie) TrySync(familyName string, uid string) error { +func (cookie *Cookie) TrySync(familyName string, uid string) error { if !cookie.AllowSyncs() { return errors.New("The user has opted out of prebid server PBSCookie syncs.") } @@ -280,7 +280,7 @@ type pbsCookieJson struct { Birthday *time.Time `json:"bday,omitempty"` } -func (cookie *PBSCookie) MarshalJSON() ([]byte, error) { +func (cookie *Cookie) MarshalJSON() ([]byte, error) { return json.Marshal(pbsCookieJson{ UIDs: cookie.uids, OptOut: cookie.optOut, @@ -296,7 +296,7 @@ func (cookie *PBSCookie) MarshalJSON() ([]byte, error) { // This Unmarshal method interprets both data formats, and does some conversions on legacy data to make it current. // If you're seeing this message after March 2018, it's safe to assume that all the legacy cookies have been // updated and remove the legacy logic. -func (cookie *PBSCookie) UnmarshalJSON(b []byte) error { +func (cookie *Cookie) UnmarshalJSON(b []byte) error { var cookieContract pbsCookieJson err := json.Unmarshal(b, &cookieContract) if err == nil { diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index ef2e9911e46..d4e89ffdf2d 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -15,7 +15,7 @@ import ( ) func TestOptOutCookie(t *testing.T) { - cookie := &PBSCookie{ + cookie := &Cookie{ uids: make(map[string]uidWithExpiry), optOut: true, birthday: timestamp(), @@ -24,7 +24,7 @@ func TestOptOutCookie(t *testing.T) { } func TestEmptyOptOutCookie(t *testing.T) { - cookie := &PBSCookie{ + cookie := &Cookie{ uids: make(map[string]uidWithExpiry), optOut: true, birthday: timestamp(), @@ -33,7 +33,7 @@ func TestEmptyOptOutCookie(t *testing.T) { } func TestEmptyCookie(t *testing.T) { - cookie := &PBSCookie{ + cookie := &Cookie{ uids: make(map[string]uidWithExpiry, 0), optOut: false, birthday: timestamp(), @@ -66,14 +66,14 @@ func TestBidderNameGets(t *testing.T) { } func TestRejectAudienceNetworkCookie(t *testing.T) { - raw := &PBSCookie{ + raw := &Cookie{ uids: map[string]uidWithExpiry{ "audienceNetwork": newTempId("0", 10), }, optOut: false, birthday: timestamp(), } - parsed := ParsePBSCookie(raw.ToHTTPCookie(90 * 24 * time.Hour)) + parsed := ParseCookie(raw.ToHTTPCookie(90 * 24 * time.Hour)) if parsed.HasLiveSync("audienceNetwork") { t.Errorf("Cookie serializing and deserializing should delete audienceNetwork values of 0") } @@ -98,7 +98,7 @@ func TestOptOutReset(t *testing.T) { } func TestOptIn(t *testing.T) { - cookie := &PBSCookie{ + cookie := &Cookie{ uids: make(map[string]uidWithExpiry), optOut: true, birthday: timestamp(), @@ -116,7 +116,7 @@ func TestParseCorruptedCookie(t *testing.T) { Name: "uids", Value: "bad base64 encoding", } - parsed := ParsePBSCookie(&raw) + parsed := ParseCookie(&raw) ensureEmptyMap(t, parsed) } @@ -126,7 +126,7 @@ func TestParseCorruptedCookieJSON(t *testing.T) { Name: "uids", Value: cookieData, } - parsed := ParsePBSCookie(&raw) + parsed := ParseCookie(&raw) ensureEmptyMap(t, parsed) } @@ -137,7 +137,7 @@ func TestParseNilSyncMap(t *testing.T) { Name: UID_COOKIE_NAME, Value: cookieData, } - parsed := ParsePBSCookie(&raw) + parsed := ParseCookie(&raw) ensureEmptyMap(t, parsed) ensureConsistency(t, parsed) } @@ -150,7 +150,7 @@ func TestParseOtherCookie(t *testing.T) { Name: otherCookieName, Value: id, }) - parsed := ParsePBSCookieFromRequest(req, &config.HostCookie{ + parsed := ParseCookieFromRequest(req, &config.HostCookie{ Family: "adnxs", CookieName: otherCookieName, }) @@ -179,7 +179,7 @@ func TestCookieReadWrite(t *testing.T) { func TestPopulatedLegacyCookieRead(t *testing.T) { legacyJson := `{"uids":{"adnxs":"123","audienceNetwork":"456"},"bday":"2017-08-03T21:04:52.629198911Z"}` - var cookie PBSCookie + var cookie Cookie json.Unmarshal([]byte(legacyJson), &cookie) if cookie.LiveSyncCount() != 0 { @@ -195,7 +195,7 @@ func TestPopulatedLegacyCookieRead(t *testing.T) { func TestEmptyLegacyCookieRead(t *testing.T) { legacyJson := `{"bday":"2017-08-29T18:54:18.393925772Z"}` - var cookie PBSCookie + var cookie Cookie json.Unmarshal([]byte(legacyJson), &cookie) if cookie.LiveSyncCount() != 0 { @@ -204,7 +204,7 @@ func TestEmptyLegacyCookieRead(t *testing.T) { } func TestNilCookie(t *testing.T) { - var nilCookie *PBSCookie + var nilCookie *Cookie if nilCookie.HasLiveSync("anything") { t.Error("nil cookies should respond with false when asked if they have a sync") @@ -250,14 +250,14 @@ func TestGetUIDs(t *testing.T) { } func TestGetUIDsWithEmptyCookie(t *testing.T) { - cookie := &PBSCookie{} + cookie := &Cookie{} uids := cookie.GetUIDs() assert.Len(t, uids, 0, "GetUIDs shouldn't return any user syncs for an empty cookie") } func TestGetUIDsWithNilCookie(t *testing.T) { - var cookie *PBSCookie + var cookie *Cookie uids := cookie.GetUIDs() assert.Len(t, uids, 0, "GetUIDs shouldn't return any user syncs for a nil cookie") @@ -294,7 +294,7 @@ func TestTrimCookiesClosestExpirationDates(t *testing.T) { } } -func ensureEmptyMap(t *testing.T, cookie *PBSCookie) { +func ensureEmptyMap(t *testing.T, cookie *Cookie) { if !cookie.AllowSyncs() { t.Error("Empty cookies should allow user syncs.") } @@ -303,7 +303,7 @@ func ensureEmptyMap(t *testing.T, cookie *PBSCookie) { } } -func ensureConsistency(t *testing.T, cookie *PBSCookie) { +func ensureConsistency(t *testing.T, cookie *Cookie) { if cookie.AllowSyncs() { err := cookie.TrySync("pulsepoint", "1") if err != nil { @@ -340,7 +340,7 @@ func ensureConsistency(t *testing.T, cookie *PBSCookie) { } } - copiedCookie := ParsePBSCookie(cookie.ToHTTPCookie(90 * 24 * time.Hour)) + copiedCookie := ParseCookie(cookie.ToHTTPCookie(90 * 24 * time.Hour)) if copiedCookie.AllowSyncs() != cookie.AllowSyncs() { t.Error("The PBSCookie interface shouldn't let modifications happen if the user has opted out") } @@ -372,8 +372,8 @@ func newTempId(uid string, offset int) uidWithExpiry { } } -func newSampleCookie() *PBSCookie { - return &PBSCookie{ +func newSampleCookie() *Cookie { + return &Cookie{ uids: map[string]uidWithExpiry{ "adnxs": newTempId("123", 10), "rubicon": newTempId("456", 10), @@ -383,8 +383,8 @@ func newSampleCookie() *PBSCookie { } } -func newTestCookie() (*PBSCookie, int) { - var mediumSizeCookie *PBSCookie = &PBSCookie{ +func newTestCookie() (*Cookie, int) { + var mediumSizeCookie *Cookie = &Cookie{ uids: map[string]uidWithExpiry{ "key1": newTempId("12345678901234567890123456789012345678901234567890", 7), "key2": newTempId("abcdefghijklmnopqrstuvwxyz", 6), @@ -400,7 +400,7 @@ func newTestCookie() (*PBSCookie, int) { return mediumSizeCookie, len(mediumSizeCookie.uids) } -func writeThenRead(cookie *PBSCookie, maxCookieSize int) *PBSCookie { +func writeThenRead(cookie *Cookie, maxCookieSize int) *Cookie { w := httptest.NewRecorder() hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: maxCookieSize} cookie.SetCookieOnResponse(w, false, hostCookie, 90*24*time.Hour) @@ -409,7 +409,7 @@ func writeThenRead(cookie *PBSCookie, maxCookieSize int) *PBSCookie { header := http.Header{} header.Add("Cookie", writtenCookie) request := http.Request{Header: header} - return ParsePBSCookieFromRequest(&request, hostCookie) + return ParseCookieFromRequest(&request, hostCookie) } func TestSetCookieOnResponseForSameSiteNone(t *testing.T) { diff --git a/usersync/shuffler.go b/usersync/shuffler.go new file mode 100644 index 00000000000..b75fa3e105b --- /dev/null +++ b/usersync/shuffler.go @@ -0,0 +1,13 @@ +package usersync + +import "math/rand" + +type shuffler interface { + shuffle(v []string) +} + +type randomShuffler struct{} + +func (randomShuffler) shuffle(v []string) { + rand.Shuffle(len(v), func(i, j int) { v[i], v[j] = v[j], v[i] }) +} diff --git a/usersync/shuffler_test.go b/usersync/shuffler_test.go new file mode 100644 index 00000000000..ab5ec8efd01 --- /dev/null +++ b/usersync/shuffler_test.go @@ -0,0 +1,42 @@ +package usersync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestShuffler(t *testing.T) { + testCases := []struct { + description string + given []string + }{ + { + description: "Nil", + given: nil, + }, + { + description: "Empty", + given: []string{}, + }, + { + description: "One", + given: []string{"a"}, + }, + { + // at least 3 elements are required to test the swap logic. + description: "Many", + given: []string{"a", "b", "c"}, + }, + } + + for _, test := range testCases { + givenCopy := make([]string, len(test.given)) + copy(givenCopy, test.given) + + randomShuffler{}.shuffle(test.given) + + // ignores order of elements. we're testing the swap logic, not the rand shuffle algorithm. + assert.ElementsMatch(t, givenCopy, test.given, test.description) + } +} From 8dffdd106b4629f4903437be2048cb32119fef78 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 22 Apr 2021 11:34:57 -0400 Subject: [PATCH 02/50] Added User Sync Chooser + Test Cleanup --- endpoints/cookie_sync.go | 17 +- endpoints/cookie_sync_test.go | 24 -- usersync/bidderchooser.go | 14 +- usersync/bidderchooser_test.go | 110 ++++----- usersync/chooser.go | 123 +++++++++ usersync/chooser_test.go | 371 ++++++++++++++++++++++++++++ usersync/cookie.go | 33 +-- usersync/cookie_test.go | 36 +-- usersync/syncer.go | 26 ++ usersync/usersyncers/syncer_test.go | 138 ----------- 10 files changed, 613 insertions(+), 279 deletions(-) create mode 100644 usersync/chooser.go create mode 100644 usersync/chooser_test.go create mode 100644 usersync/syncer.go delete mode 100755 usersync/usersyncers/syncer_test.go diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index cf04176ddd9..aebc821cb52 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -24,6 +24,11 @@ import ( "github.com/prebid/prebid-server/usersync" ) +// todo +// - update to use chooser +// - handle metrics and analytics from chooser result +// - respond from chooser result + func NewCookieSyncEndpoint( syncers map[openrtb_ext.BidderName]usersync.Usersyncer, cfg *config.Configuration, @@ -110,16 +115,8 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h parsedReq.Bidders = append(parsedReq.Bidders, string(bidder)) } } - setSiteCookie := siteCookieCheck(r.UserAgent()) - needSyncupForSameSite := false - if setSiteCookie { - _, err1 := r.Cookie(usersync.SameSiteCookieName) - if err1 == http.ErrNoCookie { - needSyncupForSameSite = true - } - } - parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie, needSyncupForSameSite) + parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie, false) adapterSyncs := make(map[openrtb_ext.BidderName]bool) // assume all bidders will be privacy blocked @@ -148,7 +145,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h adapterSyncs[openrtb_ext.BidderName(b)] = false } for b, g := range adapterSyncs { - deps.metrics.RecordAdapterCookieSync(b, g) + deps.metrics.RecordAdapterCookieSync(b, g) // how to record now that 1:many mapping for keys? multiple bidders benefit from the same sync. } parsedReq.filterToLimit() diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 201317da381..84e9545e1ff 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -131,15 +131,6 @@ func TestCookieSyncHasCookies(t *testing.T) { assert.Equal(t, "ok", parseStatus(t, rr.Body.Bytes())) } -// Make sure that an empty bidders array returns no syncs -func TestCookieSyncEmptyBidders(t *testing.T) { - rr := doPost(`{"bidders": []}`, nil, true, syncersForTest()) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.Empty(t, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} - func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) { rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true}, config.CCPA{}) assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") @@ -148,21 +139,6 @@ func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) { assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) } -func TestCookieSyncWithLimit(t *testing.T) { - rr := doPost(`{"limit":2}`, nil, true, syncersForTest()) - assert.Equal(t, http.StatusOK, rr.Code) - assert.Len(t, parseSyncs(t, rr.Body.Bytes()), 2, "usersyncs") - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} - -func TestCookieSyncWithLargeLimit(t *testing.T) { - syncers := syncersForTest() - rr := doPost(`{"limit":1000}`, nil, true, syncers) - assert.Equal(t, http.StatusOK, rr.Code) - assert.Len(t, parseSyncs(t, rr.Body.Bytes()), len(syncers), "usersyncs") - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} - func doPost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer) *httptest.ResponseRecorder { return doConfigurablePost(body, existingSyncs, gdprHostConsent, gdprBidders, config.GDPR{}, config.CCPA{}) } diff --git a/usersync/bidderchooser.go b/usersync/bidderchooser.go index 1bed3256703..3dac0625ec8 100644 --- a/usersync/bidderchooser.go +++ b/usersync/bidderchooser.go @@ -3,14 +3,15 @@ package usersync import "github.com/prebid/prebid-server/config" type bidderChooser interface { + // choose returns an ordered collection of potentially non-unique bidders. choose(requested, available []string, cooperative config.UserSyncCooperative) []string } -type randomBidderChooser struct { +type standardBidderChooser struct { shuffler shuffler } -func (c randomBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { +func (c standardBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { if requested == nil { return c.shuffledCopy(available) } @@ -22,7 +23,7 @@ func (c randomBidderChooser) choose(requested, available []string, cooperative c return c.shuffledCopy(requested) } -func (c randomBidderChooser) chooseCooperative(requested, available []string, priorityGroups [][]string) []string { +func (c standardBidderChooser) chooseCooperative(requested, available []string, priorityGroups [][]string) []string { biddersCapacity := int(float64(len(available)) * 1.5) bidders := make([]string, 0, biddersCapacity) @@ -40,17 +41,14 @@ func (c randomBidderChooser) chooseCooperative(requested, available []string, pr return bidders } -func (c randomBidderChooser) shuffledCopy(a []string) []string { - if a == nil { - return nil - } +func (c standardBidderChooser) shuffledCopy(a []string) []string { aCopy := make([]string, len(a)) copy(aCopy, a) c.shuffler.shuffle(aCopy) return aCopy } -func (c randomBidderChooser) shuffledAppend(a, b []string) []string { +func (c standardBidderChooser) shuffledAppend(a, b []string) []string { startIndex := len(a) a = append(a, b...) c.shuffler.shuffle(a[startIndex:]) diff --git a/usersync/bidderchooser_test.go b/usersync/bidderchooser_test.go index c50abc1f96c..cb636586960 100644 --- a/usersync/bidderchooser_test.go +++ b/usersync/bidderchooser_test.go @@ -7,12 +7,13 @@ import ( "github.com/stretchr/testify/assert" ) -// These tests use a deterministic test version of the shuffler, where the order -// is reversed rather than randomized. +// These tests use a deterministic version of the shuffler, where the order +// is reversed rather than randomized. Most tests specify at least 2 elements +// to implicitly verify the shuffler is invoked (or not invoked). -func TestChoose(t *testing.T) { +func TestBidderChooserChoose(t *testing.T) { shuffler := reverseShuffler{} - available := []string{"a", "b"} + available := []string{"a1", "a2"} testCases := []struct { description string @@ -24,7 +25,7 @@ func TestChoose(t *testing.T) { description: "No Coop - Nil", givenRequested: nil, givenCooperative: config.UserSyncCooperative{Enabled: false}, - expected: []string{"b", "a"}, + expected: []string{"a2", "a1"}, }, { description: "No Coop - Empty", @@ -34,35 +35,35 @@ func TestChoose(t *testing.T) { }, { description: "No Coop - One", - givenRequested: []string{"c"}, + givenRequested: []string{"r"}, givenCooperative: config.UserSyncCooperative{Enabled: false}, - expected: []string{"c"}, + expected: []string{"r"}, }, { description: "No Coop - Many", - givenRequested: []string{"c", "d"}, + givenRequested: []string{"r1", "r2"}, givenCooperative: config.UserSyncCooperative{Enabled: false}, - expected: []string{"d", "c"}, + expected: []string{"r2", "r1"}, }, { - description: "Coop - Integration", - givenRequested: []string{"c", "d"}, - givenCooperative: config.UserSyncCooperative{Enabled: true, PriorityGroups: [][]string{{"1", "2"}, {"3", "4"}}}, - expected: []string{"d", "c", "2", "1", "4", "3", "b", "a"}, + description: "Coop - Integration Test", + givenRequested: []string{"r1", "r2"}, + givenCooperative: config.UserSyncCooperative{Enabled: true, PriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}}, + expected: []string{"r2", "r1", "pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, }, } for _, test := range testCases { - chooser := randomBidderChooser{shuffler: shuffler} + chooser := standardBidderChooser{shuffler: shuffler} chosen := chooser.choose(test.givenRequested, available, test.givenCooperative) assert.Equal(t, test.expected, chosen, test.description) } } -func TestChooseCooperative(t *testing.T) { +func TestBidderChooserCooperative(t *testing.T) { shuffler := reverseShuffler{} - available := []string{"a", "b"} + available := []string{"a1", "a2"} testCases := []struct { description string @@ -74,49 +75,49 @@ func TestChooseCooperative(t *testing.T) { description: "Nil", givenRequested: nil, givenPriorityGroups: nil, - expected: []string{"b", "a"}, + expected: []string{"a2", "a1"}, }, { description: "Empty", givenRequested: []string{}, givenPriorityGroups: [][]string{}, - expected: []string{"b", "a"}, + expected: []string{"a2", "a1"}, }, { description: "Requested", - givenRequested: []string{"c", "d"}, + givenRequested: []string{"r1", "r2"}, givenPriorityGroups: nil, - expected: []string{"d", "c", "b", "a"}, + expected: []string{"r2", "r1", "a2", "a1"}, }, { description: "Priority Groups - One", givenRequested: nil, - givenPriorityGroups: [][]string{{"c", "d"}}, - expected: []string{"d", "c", "b", "a"}, + givenPriorityGroups: [][]string{{"pr1A", "pr1B"}}, + expected: []string{"pr1B", "pr1A", "a2", "a1"}, }, { description: "Priority Groups - Many", givenRequested: nil, - givenPriorityGroups: [][]string{{"c", "d"}, {"e", "f", "g"}}, - expected: []string{"d", "c", "g", "f", "e", "b", "a"}, + givenPriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}, + expected: []string{"pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, }, { description: "Requested + Priority Groups", - givenRequested: []string{"c", "d"}, - givenPriorityGroups: [][]string{{"e", "f"}, {"g", "h", "i"}}, - expected: []string{"d", "c", "f", "e", "i", "h", "g", "b", "a"}, + givenRequested: []string{"r1", "r2"}, + givenPriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}, + expected: []string{"r2", "r1", "pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, }, } for _, test := range testCases { - chooser := randomBidderChooser{shuffler: shuffler} + chooser := standardBidderChooser{shuffler: shuffler} chosen := chooser.chooseCooperative(test.givenRequested, available, test.givenPriorityGroups) assert.Equal(t, test.expected, chosen, test.description) } } -func TestShuffledCopy(t *testing.T) { +func TestBidderChooserShuffledCopy(t *testing.T) { shuffler := reverseShuffler{} testCases := []struct { @@ -124,11 +125,6 @@ func TestShuffledCopy(t *testing.T) { given []string expected []string }{ - { - description: "Nil", - given: nil, - expected: nil, - }, { description: "Empty", given: []string{}, @@ -149,7 +145,7 @@ func TestShuffledCopy(t *testing.T) { for _, test := range testCases { givenCopy := copySlice(test.given) - chooser := randomBidderChooser{shuffler: shuffler} + chooser := standardBidderChooser{shuffler: shuffler} shuffled := chooser.shuffledCopy(test.given) assert.Equal(t, givenCopy, test.given, test.description+":input unchanged") @@ -157,7 +153,7 @@ func TestShuffledCopy(t *testing.T) { } } -func TestShuffledAppend(t *testing.T) { +func TestBidderChooserShuffledAppend(t *testing.T) { shuffler := reverseShuffler{} testCases := []struct { @@ -181,14 +177,14 @@ func TestShuffledAppend(t *testing.T) { { description: "Empty - Append One", givenA: []string{}, - givenB: []string{"1"}, - expected: []string{"1"}, + givenB: []string{"b"}, + expected: []string{"b"}, }, { description: "Empty - Append Many", givenA: []string{}, - givenB: []string{"1", "2"}, - expected: []string{"2", "1"}, + givenB: []string{"b1", "b2"}, + expected: []string{"b2", "b1"}, }, { description: "One - Append Nil", @@ -204,46 +200,46 @@ func TestShuffledAppend(t *testing.T) { }, { description: "One - Append One", - givenA: []string{"a"}, - givenB: []string{"1"}, - expected: []string{"a", "1"}, + givenA: []string{"a1"}, + givenB: []string{"b1"}, + expected: []string{"a1", "b1"}, }, { description: "One - Append Many", - givenA: []string{"a"}, - givenB: []string{"1", "2"}, - expected: []string{"a", "2", "1"}, + givenA: []string{"a1"}, + givenB: []string{"b1", "b2"}, + expected: []string{"a1", "b2", "b1"}, }, { description: "Many - Append Nil", - givenA: []string{"a", "b"}, + givenA: []string{"a1", "a2"}, givenB: nil, - expected: []string{"a", "b"}, + expected: []string{"a1", "a2"}, }, { description: "Many - Append Empty", - givenA: []string{"a", "b"}, + givenA: []string{"a1", "a2"}, givenB: []string{}, - expected: []string{"a", "b"}, + expected: []string{"a1", "a2"}, }, { description: "Many - Append One", - givenA: []string{"a", "b"}, - givenB: []string{"1"}, - expected: []string{"a", "b", "1"}, + givenA: []string{"a1", "a2"}, + givenB: []string{"b"}, + expected: []string{"a1", "a2", "b"}, }, { description: "Many - Append Many", - givenA: []string{"a", "b"}, - givenB: []string{"1", "2"}, - expected: []string{"a", "b", "2", "1"}, + givenA: []string{"a1", "a2"}, + givenB: []string{"b1", "b2"}, + expected: []string{"a1", "a2", "b2", "b1"}, }, } for _, test := range testCases { givenBCopy := copySlice(test.givenB) - chooser := randomBidderChooser{shuffler: shuffler} + chooser := standardBidderChooser{shuffler: shuffler} shuffled := chooser.shuffledAppend(test.givenA, test.givenB) assert.Equal(t, givenBCopy, test.givenB, test.description+":append input unchanged") diff --git a/usersync/chooser.go b/usersync/chooser.go new file mode 100644 index 00000000000..e9532661142 --- /dev/null +++ b/usersync/chooser.go @@ -0,0 +1,123 @@ +package usersync + +import "github.com/prebid/prebid-server/config" + +type Chooser interface { + Choose(request Request, cookie Cookie) Result +} + +func NewChooser(bidderSyncerLookup map[string]Syncer) Chooser { + bidders := make([]string, 0, len(bidderSyncerLookup)) + for k := range bidderSyncerLookup { + bidders = append(bidders, k) + } + + return standardChooser{ + bidderSyncerLookup: bidderSyncerLookup, + biddersAvailable: bidders, + bidderChooser: standardBidderChooser{shuffler: randomShuffler{}}, + } +} + +type Request struct { + Bidders []string + Kind Kind + Privacy Privacy + Cooperative config.UserSyncCooperative + Limit int +} + +type Result struct { + Status Status + BiddersEvaluated []BidderEvaluation + SyncersChosen []Syncer +} + +type BidderEvaluation struct { + Bidder string + Status Status +} + +type Status int + +const ( + StatusOK Status = iota + StatusBlockedByUserOptOut + StatusBlockedByGDPR + StatusBlockedByCCPA + StatusAlreadySynced + StatusUnknownBidder + StatusIncompatibleKind + StatusDuplicate +) + +type Privacy interface { + GDPRAllowsHostCookie() bool + GDPRAllowsBidderSync(bidder string) bool + CCPAAllowsBidderSync(bidder string) bool +} + +type standardChooser struct { + bidderSyncerLookup map[string]Syncer + biddersAvailable []string + bidderChooser bidderChooser +} + +func (c standardChooser) Choose(request Request, cookie Cookie) Result { + if !cookie.AllowSyncs() { + return Result{Status: StatusBlockedByUserOptOut} + } + + if !request.Privacy.GDPRAllowsHostCookie() { + return Result{Status: StatusBlockedByGDPR} + } + + syncersSeen := make(map[string]struct{}) + limitDisabled := request.Limit <= 0 + + biddersEvaluated := make([]BidderEvaluation, 0) + syncersChosen := make([]Syncer, 0) + + bidders := c.bidderChooser.choose(request.Bidders, c.biddersAvailable, request.Cooperative) + for i := 0; i < len(bidders) && (limitDisabled || len(syncersChosen) < request.Limit); i++ { + syncer, evaluation := c.evaluate(bidders[i], syncersSeen, request.Kind, request.Privacy, cookie) + + biddersEvaluated = append(biddersEvaluated, evaluation) + if evaluation.Status == StatusOK { + syncersChosen = append(syncersChosen, syncer) + } + } + + return Result{Status: StatusOK, BiddersEvaluated: biddersEvaluated, SyncersChosen: syncersChosen} +} + +func (c standardChooser) evaluate(bidder string, syncersSeen map[string]struct{}, kind Kind, privacy Privacy, cookie Cookie) (Syncer, BidderEvaluation) { + syncer, exists := c.bidderSyncerLookup[bidder] + if !exists { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusUnknownBidder} + } + + _, seen := syncersSeen[syncer.Key()] + if seen { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusDuplicate} + } + syncersSeen[syncer.Key()] = struct{}{} + + if !syncer.SupportsKind(kind) { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusIncompatibleKind} + } + + if cookie.HasLiveSync(syncer.Key()) { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusAlreadySynced} + } + + if !privacy.GDPRAllowsBidderSync(bidder) { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusBlockedByGDPR} + } + + if !privacy.CCPAAllowsBidderSync(bidder) { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusBlockedByCCPA} + } + + return syncer, BidderEvaluation{Bidder: bidder, Status: StatusOK} +} diff --git a/usersync/chooser_test.go b/usersync/chooser_test.go new file mode 100644 index 00000000000..078cd07dbbb --- /dev/null +++ b/usersync/chooser_test.go @@ -0,0 +1,371 @@ +package usersync + +import ( + "testing" + "time" + + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/privacy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestNewChooser(t *testing.T) { + testCases := []struct { + description string + bidderSyncerLookup map[string]Syncer + expectedBiddersAvailable []string + }{ + { + description: "Nil", + bidderSyncerLookup: nil, + expectedBiddersAvailable: []string{}, + }, + { + description: "Empty", + bidderSyncerLookup: map[string]Syncer{}, + expectedBiddersAvailable: []string{}, + }, + { + description: "One", + bidderSyncerLookup: map[string]Syncer{"a": fakeSyncer{}}, + expectedBiddersAvailable: []string{"a"}, + }, + { + description: "Many", + bidderSyncerLookup: map[string]Syncer{"a": fakeSyncer{}, "b": fakeSyncer{}}, + expectedBiddersAvailable: []string{"a", "b"}, + }, + } + + for _, test := range testCases { + chooser, _ := NewChooser(test.bidderSyncerLookup).(standardChooser) + assert.ElementsMatch(t, test.expectedBiddersAvailable, chooser.biddersAvailable, test.description) + } +} + +func TestChooserChoose(t *testing.T) { + fakeSyncerA := fakeSyncer{key: "keyA", supportsKind: true} + fakeSyncerB := fakeSyncer{key: "keyB", supportsKind: true} + fakeSyncerC := fakeSyncer{key: "keyC", supportsKind: false} + bidderSyncerLookup := map[string]Syncer{"a": fakeSyncerA, "b": fakeSyncerB, "c": fakeSyncerC} + + cooperativeConfig := config.UserSyncCooperative{Enabled: true} + + testCases := []struct { + description string + givenRequest Request + givenChosenBidders []string + givenCookie Cookie + expected Result + }{ + { + description: "Cookie Opt Out", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"a"}, + givenCookie: Cookie{optOut: true}, + expected: Result{ + Status: StatusBlockedByUserOptOut, + BiddersEvaluated: nil, + SyncersChosen: nil, + }, + }, + { + description: "GDPR Host Cookie Not Allowed", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: false, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"a"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusBlockedByGDPR, + BiddersEvaluated: nil, + SyncersChosen: nil, + }, + }, + { + description: "No Bidders", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{}, + SyncersChosen: []Syncer{}, + }, + }, + { + description: "One Bidder - Sync", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"a"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}}, + SyncersChosen: []Syncer{fakeSyncerA}, + }, + }, + { + description: "One Bidder - No Sync", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"c"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusIncompatibleKind}}, + SyncersChosen: []Syncer{}, + }, + }, + { + description: "Many Bidders - All Sync - Limit Disabled With 0", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"a", "b"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "b", Status: StatusOK}}, + SyncersChosen: []Syncer{fakeSyncerA, fakeSyncerB}, + }, + }, + { + description: "Many Bidders - All Sync - Limit Disabled With Negative Value", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: -1, + }, + givenChosenBidders: []string{"a", "b"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "b", Status: StatusOK}}, + SyncersChosen: []Syncer{fakeSyncerA, fakeSyncerB}, + }, + }, + { + description: "Many Bidders - Limited Sync", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 1, + }, + givenChosenBidders: []string{"a", "b"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}}, + SyncersChosen: []Syncer{fakeSyncerA}, + }, + }, + { + description: "Many Bidders - Limited Sync - Disqualified Syncers Don't Count Towards Limit", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 1, + }, + givenChosenBidders: []string{"c", "a", "b"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusIncompatibleKind}, {Bidder: "a", Status: StatusOK}}, + SyncersChosen: []Syncer{fakeSyncerA}, + }, + }, + { + description: "Many Bidders - Some Sync, Some Don't", + givenRequest: Request{ + Privacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + Limit: 0, + }, + givenChosenBidders: []string{"a", "c"}, + givenCookie: Cookie{}, + expected: Result{ + Status: StatusOK, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "c", Status: StatusIncompatibleKind}}, + SyncersChosen: []Syncer{fakeSyncerA}, + }, + }, + } + + bidders := []string{"anyRequested"} + biddersAvailable := []string{"anyAvailable"} + for _, test := range testCases { + // set request values which don't need to be specified for each test + test.givenRequest.Bidders = bidders + test.givenRequest.Kind = KindRedirect + test.givenRequest.Cooperative = cooperativeConfig + + mockBidderChooser := &mockBidderChooser{} + mockBidderChooser. + On("choose", test.givenRequest.Bidders, biddersAvailable, cooperativeConfig). + Return(test.givenChosenBidders) + + chooser := standardChooser{ + bidderSyncerLookup: bidderSyncerLookup, + biddersAvailable: biddersAvailable, + bidderChooser: mockBidderChooser, + } + + result := chooser.Choose(test.givenRequest, test.givenCookie) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestChooserEvaluate(t *testing.T) { + fakeSyncerA := fakeSyncer{key: "keyA", supportsKind: true} + fakeSyncerB := fakeSyncer{key: "keyB", supportsKind: false} + bidderSyncerLookup := map[string]Syncer{"a": fakeSyncerA, "b": fakeSyncerB} + + cookieNeedsSync := Cookie{} + cookieAlreadyHasSync := Cookie{uids: map[string]uidWithExpiry{"keyA": {Expires: time.Now().Add(time.Duration(24) * time.Hour)}}} + + testCases := []struct { + description string + givenBidder string + givenSyncersSeen map[string]struct{} + givenPrivacy Privacy + givenCookie Cookie + expectedSyncer Syncer + expectedBidder string + expectedStatus Status + }{ + { + description: "Valid", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieNeedsSync, + expectedSyncer: fakeSyncerA, + expectedBidder: "a", + expectedStatus: StatusOK, + }, + { + description: "Unknown Bidder", + givenBidder: "unknown", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieNeedsSync, + expectedSyncer: nil, + expectedBidder: "unknown", + expectedStatus: StatusUnknownBidder, + }, + { + description: "Duplicate Syncer", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{"keyA": {}}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieNeedsSync, + expectedSyncer: nil, + expectedBidder: "a", + expectedStatus: StatusDuplicate, + }, + { + description: "Incompatible Kind", + givenBidder: "b", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieNeedsSync, + expectedSyncer: nil, + expectedBidder: "b", + expectedStatus: StatusIncompatibleKind, + }, + { + description: "Already Synced", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieAlreadyHasSync, + expectedSyncer: nil, + expectedBidder: "a", + expectedStatus: StatusAlreadySynced, + }, + { + description: "Blocked By GDPR", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: false, ccpaAllowsBidderSync: true}, + givenCookie: cookieNeedsSync, + expectedSyncer: nil, + expectedBidder: "a", + expectedStatus: StatusBlockedByGDPR, + }, + { + description: "Blocked By CCPA", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: false}, + givenCookie: cookieNeedsSync, + expectedSyncer: nil, + expectedBidder: "a", + expectedStatus: StatusBlockedByCCPA, + }, + } + + for _, test := range testCases { + chooser, _ := NewChooser(bidderSyncerLookup).(standardChooser) + sync, evaluation := chooser.evaluate(test.givenBidder, test.givenSyncersSeen, KindBidderPreference, test.givenPrivacy, test.givenCookie) + + assert.Equal(t, test.expectedSyncer, sync, test.description+":syncer") + + expectedEvaluation := BidderEvaluation{Bidder: test.expectedBidder, Status: test.expectedStatus} + assert.Equal(t, expectedEvaluation, evaluation, test.description+":evaluation") + } +} + +type mockBidderChooser struct { + mock.Mock +} + +func (m *mockBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { + args := m.Called(requested, available, cooperative) + return args.Get(0).([]string) +} + +type fakeSyncer struct { + key string + supportsKind bool +} + +func (s fakeSyncer) Key() string { + return s.key +} + +func (s fakeSyncer) SupportsKind(kind Kind) bool { + return s.supportsKind +} + +func (fakeSyncer) GetSync(kind Kind, privacyPolicies privacy.Policies) Sync { + return Sync{} +} + +type fakePrivacy struct { + gdprAllowsHostCookie bool + gdprAllowsBidderSync bool + ccpaAllowsBidderSync bool +} + +func (p fakePrivacy) GDPRAllowsHostCookie() bool { + return p.gdprAllowsHostCookie +} + +func (p fakePrivacy) GDPRAllowsBidderSync(bidder string) bool { + return p.gdprAllowsBidderSync +} + +func (p fakePrivacy) CCPAAllowsBidderSync(bidder string) bool { + return p.ccpaAllowsBidderSync +} diff --git a/usersync/cookie.go b/usersync/cookie.go index e96cdf7f8a8..cf661465806 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -12,14 +12,10 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) -const ( - // DEFAULT_TTL is the default amount of time which a cookie is considered valid. - DEFAULT_TTL = 14 * 24 * time.Hour - UID_COOKIE_NAME = "uids" - SameSiteCookieName = "SSCookie" - SameSiteCookieValue = "1" - SameSiteAttribute = "; SameSite=None" -) +// defaultTTL is the default amount of time a cookie is considered valid. +const defaultTTL = 14 * 24 * time.Hour + +const uidCookieName = "uids" // customBidderTTLs stores rules about how long a particular UID sync is valid for each bidder. // If a bidder does a cookie sync *without* listing a rule here, then the DEFAULT_TTL will be used. @@ -61,7 +57,7 @@ func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie } } var parsed *Cookie - uidCookie, err2 := r.Cookie(UID_COOKIE_NAME) + uidCookie, err2 := r.Cookie(uidCookieName) if err2 == nil { parsed = ParseCookie(uidCookie) } else { @@ -130,7 +126,7 @@ func (cookie *Cookie) ToHTTPCookie(ttl time.Duration) *http.Cookie { b64 := base64.URLEncoding.EncodeToString(j) return &http.Cookie{ - Name: UID_COOKIE_NAME, + Name: uidCookieName, Value: b64, Expires: time.Now().Add(ttl), Path: "/", @@ -203,21 +199,10 @@ func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie b } var uidsCookieStr string - var sameSiteCookie *http.Cookie if setSiteCookie { httpCookie.Secure = true - uidsCookieStr = httpCookie.String() - uidsCookieStr += SameSiteAttribute - sameSiteCookie = &http.Cookie{ - Name: SameSiteCookieName, - Value: SameSiteCookieValue, - Expires: time.Now().Add(ttl), - Path: "/", - Secure: true, - } - sameSiteCookieStr := sameSiteCookie.String() - sameSiteCookieStr += SameSiteAttribute - w.Header().Add("Set-Cookie", sameSiteCookieStr) + httpCookie.SameSite = http.SameSiteNoneMode + } else { uidsCookieStr = httpCookie.String() } @@ -339,7 +324,7 @@ func (cookie *Cookie) UnmarshalJSON(b []byte) error { // getExpiry gets an expiry date for the cookie, assuming it was generated right now. func getExpiry(familyName string) time.Time { - ttl := DEFAULT_TTL + ttl := defaultTTL if customTTL, ok := customBidderTTLs[familyName]; ok { ttl = customTTL } diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index d4e89ffdf2d..5b467471377 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -134,7 +134,7 @@ func TestParseNilSyncMap(t *testing.T) { cookieJSON := "{\"bday\":123,\"optout\":true}" cookieData := base64.URLEncoding.EncodeToString([]byte(cookieJSON)) raw := http.Cookie{ - Name: UID_COOKIE_NAME, + Name: uidCookieName, Value: cookieData, } parsed := ParseCookie(&raw) @@ -412,23 +412,23 @@ func writeThenRead(cookie *Cookie, maxCookieSize int) *Cookie { return ParseCookieFromRequest(&request, hostCookie) } -func TestSetCookieOnResponseForSameSiteNone(t *testing.T) { - cookie := newSampleCookie() - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "http://www.prebid.com", nil) - ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" - req.Header.Set("User-Agent", ua) - hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: 0} - cookie.SetCookieOnResponse(w, true, hostCookie, 90*24*time.Hour) - writtenCookie := w.HeaderMap.Get("Set-Cookie") - t.Log("Set-Cookie is: ", writtenCookie) - if !strings.Contains(writtenCookie, "SSCookie=1") { - t.Error("Set-Cookie should contain SSCookie=1") - } - if !strings.Contains(writtenCookie, "; Secure;") { - t.Error("Set-Cookie should contain Secure") - } -} +// func TestSetCookieOnResponseForSameSiteNone(t *testing.T) { +// cookie := newSampleCookie() +// w := httptest.NewRecorder() +// req := httptest.NewRequest("GET", "http://www.prebid.com", nil) +// ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" +// req.Header.Set("User-Agent", ua) +// hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: 0} +// cookie.SetCookieOnResponse(w, true, hostCookie, 90*24*time.Hour) +// writtenCookie := w.HeaderMap.Get("Set-Cookie") +// t.Log("Set-Cookie is: ", writtenCookie) +// if !strings.Contains(writtenCookie, "SSCookie=1") { +// t.Error("Set-Cookie should contain SSCookie=1") +// } +// if !strings.Contains(writtenCookie, "; Secure;") { +// t.Error("Set-Cookie should contain Secure") +// } +// } func TestSetCookieOnResponseForOlderChromeVersion(t *testing.T) { cookie := newSampleCookie() diff --git a/usersync/syncer.go b/usersync/syncer.go new file mode 100644 index 00000000000..51a6861c172 --- /dev/null +++ b/usersync/syncer.go @@ -0,0 +1,26 @@ +package usersync + +import "github.com/prebid/prebid-server/privacy" + +type Syncer interface { + Key() string + SupportsKind(kind Kind) bool + GetSync(kind Kind, privacyPolicies privacy.Policies) Sync +} + +type Sync struct { + URL string + Kind Kind + SupportCORS bool +} + +type Kind int + +const ( + KindBidderPreference Kind = iota + KindIFrame + KindRedirect +) + +// todo: syncer from config +// - builds up the url template per bidder diff --git a/usersync/usersyncers/syncer_test.go b/usersync/usersyncers/syncer_test.go deleted file mode 100755 index 5b2e80463ac..00000000000 --- a/usersync/usersyncers/syncer_test.go +++ /dev/null @@ -1,138 +0,0 @@ -package usersyncers - -import ( - "strings" - "testing" - - "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" -) - -func TestNewSyncerMap(t *testing.T) { - syncConfig := config.Adapter{ - UserSyncURL: "some-sync-url", - } - cfg := &config.Configuration{ - Adapters: map[string]config.Adapter{ - string(openrtb_ext.Bidder33Across): syncConfig, - string(openrtb_ext.BidderAcuityAds): syncConfig, - string(openrtb_ext.BidderAdform): syncConfig, - string(openrtb_ext.BidderAdkernel): syncConfig, - string(openrtb_ext.BidderAdkernelAdn): syncConfig, - string(openrtb_ext.BidderAdman): syncConfig, - string(openrtb_ext.BidderAdmixer): syncConfig, - string(openrtb_ext.BidderAdOcean): syncConfig, - string(openrtb_ext.BidderAdpone): syncConfig, - string(openrtb_ext.BidderAdtarget): syncConfig, - string(openrtb_ext.BidderAdtelligent): syncConfig, - string(openrtb_ext.BidderAdvangelists): syncConfig, - string(openrtb_ext.BidderAdyoulike): syncConfig, - string(openrtb_ext.BidderAJA): syncConfig, - string(openrtb_ext.BidderAMX): syncConfig, - string(openrtb_ext.BidderAppnexus): syncConfig, - string(openrtb_ext.BidderAudienceNetwork): syncConfig, - string(openrtb_ext.BidderAvocet): syncConfig, - string(openrtb_ext.BidderBeachfront): syncConfig, - string(openrtb_ext.BidderBeintoo): syncConfig, - string(openrtb_ext.BidderBetween): syncConfig, - string(openrtb_ext.BidderBrightroll): syncConfig, - string(openrtb_ext.BidderColossus): syncConfig, - string(openrtb_ext.BidderConnectAd): syncConfig, - string(openrtb_ext.BidderConsumable): syncConfig, - string(openrtb_ext.BidderConversant): syncConfig, - string(openrtb_ext.BidderCpmstar): syncConfig, - string(openrtb_ext.BidderDatablocks): syncConfig, - string(openrtb_ext.BidderDmx): syncConfig, - string(openrtb_ext.BidderDeepintent): syncConfig, - string(openrtb_ext.BidderEmxDigital): syncConfig, - string(openrtb_ext.BidderEngageBDR): syncConfig, - string(openrtb_ext.BidderEPlanning): syncConfig, - string(openrtb_ext.BidderGamma): syncConfig, - string(openrtb_ext.BidderGamoshi): syncConfig, - string(openrtb_ext.BidderGrid): syncConfig, - string(openrtb_ext.BidderGumGum): syncConfig, - string(openrtb_ext.BidderImprovedigital): syncConfig, - string(openrtb_ext.BidderInvibes): syncConfig, - string(openrtb_ext.BidderIx): syncConfig, - string(openrtb_ext.BidderJixie): syncConfig, - string(openrtb_ext.BidderKrushmedia): syncConfig, - string(openrtb_ext.BidderLifestreet): syncConfig, - string(openrtb_ext.BidderLockerDome): syncConfig, - string(openrtb_ext.BidderLogicad): syncConfig, - string(openrtb_ext.BidderLunaMedia): syncConfig, - string(openrtb_ext.BidderMarsmedia): syncConfig, - string(openrtb_ext.BidderMediafuse): syncConfig, - string(openrtb_ext.BidderMgid): syncConfig, - string(openrtb_ext.BidderNanoInteractive): syncConfig, - string(openrtb_ext.BidderNinthDecimal): syncConfig, - string(openrtb_ext.BidderNoBid): syncConfig, - string(openrtb_ext.BidderOneTag): syncConfig, - string(openrtb_ext.BidderOpenx): syncConfig, - string(openrtb_ext.BidderPubmatic): syncConfig, - string(openrtb_ext.BidderPulsepoint): syncConfig, - string(openrtb_ext.BidderRhythmone): syncConfig, - string(openrtb_ext.BidderRTBHouse): syncConfig, - string(openrtb_ext.BidderRubicon): syncConfig, - string(openrtb_ext.BidderSharethrough): syncConfig, - string(openrtb_ext.BidderSmartAdserver): syncConfig, - string(openrtb_ext.BidderSmartRTB): syncConfig, - string(openrtb_ext.BidderSmartyAds): syncConfig, - string(openrtb_ext.BidderSomoaudience): syncConfig, - string(openrtb_ext.BidderSonobi): syncConfig, - string(openrtb_ext.BidderSovrn): syncConfig, - string(openrtb_ext.BidderSynacormedia): syncConfig, - string(openrtb_ext.BidderTappx): syncConfig, - string(openrtb_ext.BidderTelaria): syncConfig, - string(openrtb_ext.BidderTriplelift): syncConfig, - string(openrtb_ext.BidderTripleliftNative): syncConfig, - string(openrtb_ext.BidderTrustX): syncConfig, - string(openrtb_ext.BidderUcfunnel): syncConfig, - string(openrtb_ext.BidderUnruly): syncConfig, - string(openrtb_ext.BidderValueImpression): syncConfig, - string(openrtb_ext.BidderVerizonMedia): syncConfig, - string(openrtb_ext.BidderVisx): syncConfig, - string(openrtb_ext.BidderVrtcal): syncConfig, - string(openrtb_ext.BidderYieldlab): syncConfig, - string(openrtb_ext.BidderYieldmo): syncConfig, - string(openrtb_ext.BidderYieldone): syncConfig, - string(openrtb_ext.BidderZeroClickFraud): syncConfig, - }, - } - - adaptersWithoutSyncers := map[openrtb_ext.BidderName]bool{ - openrtb_ext.BidderAdgeneration: true, - openrtb_ext.BidderAdhese: true, - openrtb_ext.BidderAdoppler: true, - openrtb_ext.BidderAdot: true, - openrtb_ext.BidderAdprime: true, - openrtb_ext.BidderApplogy: true, - openrtb_ext.BidderEpom: true, - openrtb_ext.BidderDecenterAds: true, - openrtb_ext.BidderInMobi: true, - openrtb_ext.BidderKidoz: true, - openrtb_ext.BidderKubient: true, - openrtb_ext.BidderMobfoxpb: true, - openrtb_ext.BidderMobileFuse: true, - openrtb_ext.BidderOrbidder: true, - openrtb_ext.BidderPangle: true, - openrtb_ext.BidderPubnative: true, - openrtb_ext.BidderRevcontent: true, - openrtb_ext.BidderSilverMob: true, - openrtb_ext.BidderSmaato: true, - openrtb_ext.BidderUnicorn: true, - openrtb_ext.BidderYeahmobi: true, - } - - for bidder, config := range cfg.Adapters { - delete(cfg.Adapters, bidder) - cfg.Adapters[strings.ToLower(string(bidder))] = config - } - - syncers := NewSyncerMap(cfg) - for _, bidderName := range openrtb_ext.CoreBidderNames() { - _, adapterWithoutSyncer := adaptersWithoutSyncers[bidderName] - if _, ok := syncers[bidderName]; !ok && !adapterWithoutSyncer { - t.Errorf("No syncer exists for adapter: %s", bidderName) - } - } -} From 9da6cd3ad8e6cf5b4e1b40ccd4db3a1e09ea421f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 22 Apr 2021 11:36:32 -0400 Subject: [PATCH 03/50] Remoe Unused customBidderTTLs --- usersync/cookie.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/usersync/cookie.go b/usersync/cookie.go index cf661465806..3c74ffe0b05 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -17,10 +17,6 @@ const defaultTTL = 14 * 24 * time.Hour const uidCookieName = "uids" -// customBidderTTLs stores rules about how long a particular UID sync is valid for each bidder. -// If a bidder does a cookie sync *without* listing a rule here, then the DEFAULT_TTL will be used. -var customBidderTTLs = map[string]time.Duration{} - // bidderToFamilyNames maps the BidderName to Adapter.Name() for the early adapters. // If a mapping isn't listed here, then we assume that the two are the same. var bidderToFamilyNames = map[openrtb_ext.BidderName]string{ @@ -325,9 +321,6 @@ func (cookie *Cookie) UnmarshalJSON(b []byte) error { // getExpiry gets an expiry date for the cookie, assuming it was generated right now. func getExpiry(familyName string) time.Time { ttl := defaultTTL - if customTTL, ok := customBidderTTLs[familyName]; ok { - ttl = customTTL - } return time.Now().Add(ttl) } From 5e3160bc087e9d99709882b899f0f3a8d7d53521 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 22 Apr 2021 12:44:13 -0400 Subject: [PATCH 04/50] Cookie Refactor --- endpoints/auction.go | 6 +-- endpoints/cookie_sync.go | 10 ++-- endpoints/openrtb2/amp_auction.go | 6 +-- endpoints/openrtb2/auction.go | 6 +-- endpoints/openrtb2/video_auction.go | 6 +-- endpoints/setuid_test.go | 11 +++-- exchange/exchange.go | 2 +- exchange/exchange_test.go | 15 +++--- exchange/utils.go | 2 +- pbs/usersync.go | 2 +- usersync/cookie.go | 74 +++++++++++------------------ usersync/cookie_test.go | 38 +++++++-------- 12 files changed, 84 insertions(+), 94 deletions(-) diff --git a/endpoints/auction.go b/endpoints/auction.go index 92e9769d59e..b17cf14bc87 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -372,11 +372,11 @@ func setLabelSource(labels *metrics.Labels, req *pbs.PBSRequest, status *string) labels.Source = metrics.DemandApp } else { labels.Source = metrics.DemandWeb - if req.Cookie.LiveSyncCount() == 0 { + if req.Cookie.HasAnyLiveSyncs() { + labels.CookieFlag = metrics.CookieFlagYes + } else { labels.CookieFlag = metrics.CookieFlagNo *status = "no_cookie" - } else { - labels.CookieFlag = metrics.CookieFlagYes } } } diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index aebc821cb52..a57f3ab80e8 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -150,7 +150,7 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h parsedReq.filterToLimit() csResp := cookieSyncResponse{ - Status: cookieSyncStatus(userSyncCookie.LiveSyncCount()), + Status: cookieSyncStatus(userSyncCookie.HasAnyLiveSyncs()), BidderStatus: make([]*usersync.CookieSyncBidders, 0, len(parsedReq.Bidders)), } for i := 0; i < len(parsedReq.Bidders); i++ { @@ -215,11 +215,11 @@ func parseBidders(request []byte) ([]byte, error) { return nil, nil } -func cookieSyncStatus(syncCount int) string { - if syncCount == 0 { - return "no_cookie" +func cookieSyncStatus(hasSyncs bool) string { + if hasSyncs { + return "ok" } - return "ok" + return "no_cookie" } type cookieSyncRequest struct { diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index ce1eaa1c33d..beda60e7e07 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -153,10 +153,10 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h defer cancel() usersyncs := usersync.ParseCookieFromRequest(r, &(deps.cfg.HostCookie)) - if usersyncs.LiveSyncCount() == 0 { - labels.CookieFlag = metrics.CookieFlagNo - } else { + if usersyncs.HasAnyLiveSyncs() { labels.CookieFlag = metrics.CookieFlagYes + } else { + labels.CookieFlag = metrics.CookieFlagNo } labels.PubID = getAccountID(req.Site.Publisher) // Look up account now that we have resolved the pubID value diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 52be86c4596..e913d427219 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -157,10 +157,10 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http labels.PubID = getAccountID(req.App.Publisher) } else { //req.Site != nil labels.Source = metrics.DemandWeb - if usersyncs.LiveSyncCount() == 0 { - labels.CookieFlag = metrics.CookieFlagNo - } else { + if usersyncs.HasAnyLiveSyncs() { labels.CookieFlag = metrics.CookieFlagYes + } else { + labels.CookieFlag = metrics.CookieFlagNo } labels.PubID = getAccountID(req.Site.Publisher) } diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index b4b21d7d1b9..ea42a74d5cd 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -259,10 +259,10 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re labels.PubID = getAccountID(bidReq.App.Publisher) } else { // both bidReq.App == nil and bidReq.Site != nil are true labels.Source = metrics.DemandWeb - if usersyncs.LiveSyncCount() == 0 { - labels.CookieFlag = metrics.CookieFlagNo - } else { + if usersyncs.HasAnyLiveSyncs() { labels.CookieFlag = metrics.CookieFlagYes + } else { + labels.CookieFlag = metrics.CookieFlagNo } labels.PubID = getAccountID(bidReq.Site.Publisher) } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 54e3c915dba..6851b582ce6 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -201,6 +201,9 @@ func TestSetUIDEndpoint(t *testing.T) { } func TestSetUIDEndpointMetrics(t *testing.T) { + cookieWithOptOut := usersync.NewCookie() + cookieWithOptOut.SetOptOut(true) + testCases := []struct { uri string cookies []*usersync.Cookie @@ -233,7 +236,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { }, { uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.Cookie{usersync.NewPBSCookieWithOptOut()}, + cookies: []*usersync.Cookie{cookieWithOptOut}, validFamilyNames: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionOptOut, @@ -283,7 +286,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { func TestOptedOut(t *testing.T) { request := httptest.NewRequest("GET", "/setuid?bidder=pubmatic&uid=123", nil) cookie := usersync.NewCookie() - cookie.SetPreference(false) + cookie.SetOptOut(true) addCookie(request, cookie) validFamilyNames := []string{"pubmatic"} metrics := &metricsConf.DummyMetricsEngine{} @@ -366,7 +369,9 @@ func TestGetFamilyName(t *testing.T) { func assertHasSyncs(t *testing.T, testCase string, resp *httptest.ResponseRecorder, syncs map[string]string) { t.Helper() cookie := parseCookieString(t, resp) - assert.Equal(t, len(syncs), cookie.LiveSyncCount(), "Test Case: %s. /setuid response doesn't contain expected number of syncs", testCase) + + assert.Equal(t, len(syncs), len(cookie.GetUIDs()), "Test Case: %s. /setuid response doesn't contain expected number of syncs", testCase) + for bidder, uid := range syncs { assert.True(t, cookie.HasLiveSync(bidder), "Test Case: %s. /setuid response cookie doesn't contain uid for bidder: %s", testCase, bidder) actualUID, _, _ := cookie.GetUID(bidder) diff --git a/exchange/exchange.go b/exchange/exchange.go index d8ac60ba46a..bf30850e4b6 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -46,7 +46,7 @@ type Exchange interface { type IdFetcher interface { // GetId returns the ID for the bidder. The boolean will be true if the ID exists, and false otherwise. GetId(bidder openrtb_ext.BidderName) (string, bool) - LiveSyncCount() int + HasAnyLiveSyncs() bool } type exchange struct { diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 3eaa625f74f..0f0ae871a4e 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/prebid/prebid-server/errortypes" "io/ioutil" "net/http" "net/http/httptest" @@ -16,6 +15,8 @@ import ( "testing" "time" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/currency" @@ -2843,8 +2844,8 @@ func (f mockIdFetcher) GetId(bidder openrtb_ext.BidderName) (id string, ok bool) return } -func (f mockIdFetcher) LiveSyncCount() int { - return len(f) +func (f mockIdFetcher) HasAnyLiveSyncs() bool { + return len(f) > 0 } type validatingBidder struct { @@ -3017,8 +3018,8 @@ func (e *emptyUsersync) GetId(bidder openrtb_ext.BidderName) (string, bool) { return "", false } -func (e *emptyUsersync) LiveSyncCount() int { - return 0 +func (e *emptyUsersync) HasAnyLiveSyncs() bool { + return false } type mockUsersync struct { @@ -3030,8 +3031,8 @@ func (e *mockUsersync) GetId(bidder openrtb_ext.BidderName) (id string, exists b return } -func (e *mockUsersync) LiveSyncCount() int { - return len(e.syncs) +func (e *mockUsersync) HasAnyLiveSyncs() bool { + return len(e.syncs) > 0 } type panicingAdapter struct{} diff --git a/exchange/utils.go b/exchange/utils.go index 4ca02149453..3d3f08a0ae8 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -429,7 +429,7 @@ func isSpecialField(bidder string) bool { // In this function, "givenBidder" may or may not be an alias. "coreBidder" must *not* be an alias. // It returns true if a Cookie User Sync existed, and false otherwise. func prepareUser(req *openrtb.BidRequest, givenBidder string, coreBidder openrtb_ext.BidderName, explicitBuyerUIDs map[string]string, usersyncs IdFetcher) bool { - cookieId, hadCookie := usersyncs.GetId(coreBidder) + cookieId, hadCookie := usersyncs.GetId(coreBidder) // TODO: this will need to have a lookup from bidder name to cookie key if id, ok := explicitBuyerUIDs[givenBidder]; ok { req.User = copyWithBuyerUID(req.User, id) diff --git a/pbs/usersync.go b/pbs/usersync.go index 2af0708bd3d..85b55f42aeb 100644 --- a/pbs/usersync.go +++ b/pbs/usersync.go @@ -93,7 +93,7 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr } pc := usersync.ParseCookieFromRequest(r, deps.HostCookieConfig) - pc.SetPreference(optout == "") + pc.SetOptOut(optout != "") pc.SetCookieOnResponse(w, false, deps.HostCookieConfig, deps.HostCookieConfig.TTLDuration()) diff --git a/usersync/cookie.go b/usersync/cookie.go index 3c74ffe0b05..b66278651b7 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -12,11 +12,12 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) -// defaultTTL is the default amount of time a cookie is considered valid. -const defaultTTL = 14 * 24 * time.Hour - const uidCookieName = "uids" +// uidTTL is the default amount of time a uid stored within a cookie is considered valid. This is +// separate from the cookie ttl. +const uidTTL = 14 * 24 * time.Hour + // bidderToFamilyNames maps the BidderName to Adapter.Name() for the early adapters. // If a mapping isn't listed here, then we assume that the two are the same. var bidderToFamilyNames = map[openrtb_ext.BidderName]string{ @@ -48,7 +49,7 @@ func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie optOutCookie, err1 := r.Cookie(cookie.OptOutCookie.Name) if err1 == nil && optOutCookie.Value == cookie.OptOutCookie.Value { pc := NewCookie() - pc.SetPreference(false) + pc.SetOptOut(true) return pc } } @@ -69,19 +70,20 @@ func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie } // ParseCookie parses the UserSync cookie from a raw HTTP cookie. -func ParseCookie(uidCookie *http.Cookie) *Cookie { - pc := NewCookie() - - j, err := base64.URLEncoding.DecodeString(uidCookie.Value) +func ParseCookie(httpCookie *http.Cookie) *Cookie { + jsonValue, err := base64.URLEncoding.DecodeString(httpCookie.Value) if err != nil { // corrupted cookie; we should reset - return pc + return NewCookie() } - err = json.Unmarshal(j, pc) - // The error on Unmarshal here isn't terribly important. - // If the cookie has been corrupted, we should reset to an empty one anyway. - return pc + var cookie Cookie + if err = json.Unmarshal(jsonValue, &cookie); err != nil { + // corrupted cookie; we should reset + return NewCookie() + } + + return &cookie } // NewCookie returns an empty PBSCookie @@ -92,26 +94,16 @@ func NewCookie() *Cookie { } } -// NewPBSCookie returns an empty PBSCookie with optOut enabled -func NewPBSCookieWithOptOut() *Cookie { - return &Cookie{ - uids: make(map[string]uidWithExpiry), - optOut: true, - birthday: timestamp(), - } -} - // AllowSyncs is true if the user lets bidders sync cookies, and false otherwise. func (cookie *Cookie) AllowSyncs() bool { return cookie != nil && !cookie.optOut } -// SetPreference is used to change whether or not we're allowed to sync cookies for this user. -func (cookie *Cookie) SetPreference(allow bool) { - if allow { - cookie.optOut = false - } else { - cookie.optOut = true +// SetOptOut is used to change whether or not we're allowed to sync cookies for this user. +func (cookie *Cookie) SetOptOut(optOut bool) { + cookie.optOut = optOut + + if optOut { cookie.uids = make(map[string]uidWithExpiry) } } @@ -198,7 +190,6 @@ func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie b if setSiteCookie { httpCookie.Secure = true httpCookie.SameSite = http.SameSiteNoneMode - } else { uidsCookieStr = httpCookie.String() } @@ -216,18 +207,17 @@ func (cookie *Cookie) HasLiveSync(familyName string) bool { return isLive } -// LiveSyncCount returns the number of families which have active UIDs for this user. -func (cookie *Cookie) LiveSyncCount() int { +// HasAnyLiveSyncs returns true if this cookie has at least one active sync. +func (cookie *Cookie) HasAnyLiveSyncs() bool { now := time.Now() - numSyncs := 0 if cookie != nil { for _, value := range cookie.uids { if now.Before(value.Expires) { - numSyncs++ + return true } } } - return numSyncs + return false } // TrySync tries to set the UID for some family name. It returns an error if the set didn't happen. @@ -244,17 +234,17 @@ func (cookie *Cookie) TrySync(familyName string, uid string) error { cookie.uids[familyName] = uidWithExpiry{ UID: uid, - Expires: getExpiry(familyName), + Expires: time.Now().Add(uidTTL), } return nil } -// pbsCookieJson defines the JSON contract for the cookie data's storage format. +// cookieJson defines the JSON contract for the cookie data's storage format. // // This exists so that PBSCookie (which is public) can have private fields, and the rest of // PBS doesn't have to worry about the cookie data storage format. -type pbsCookieJson struct { +type cookieJson struct { LegacyUIDs map[string]string `json:"uids,omitempty"` UIDs map[string]uidWithExpiry `json:"tempUIDs,omitempty"` OptOut bool `json:"optout,omitempty"` @@ -262,7 +252,7 @@ type pbsCookieJson struct { } func (cookie *Cookie) MarshalJSON() ([]byte, error) { - return json.Marshal(pbsCookieJson{ + return json.Marshal(cookieJson{ UIDs: cookie.uids, OptOut: cookie.optOut, Birthday: cookie.birthday, @@ -278,7 +268,7 @@ func (cookie *Cookie) MarshalJSON() ([]byte, error) { // If you're seeing this message after March 2018, it's safe to assume that all the legacy cookies have been // updated and remove the legacy logic. func (cookie *Cookie) UnmarshalJSON(b []byte) error { - var cookieContract pbsCookieJson + var cookieContract cookieJson err := json.Unmarshal(b, &cookieContract) if err == nil { cookie.optOut = cookieContract.OptOut @@ -318,12 +308,6 @@ func (cookie *Cookie) UnmarshalJSON(b []byte) error { return err } -// getExpiry gets an expiry date for the cookie, assuming it was generated right now. -func getExpiry(familyName string) time.Time { - ttl := defaultTTL - return time.Now().Add(ttl) -} - func timestamp() *time.Time { birthday := time.Now() return &birthday diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index 5b467471377..df46640449a 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -90,9 +90,9 @@ func TestRejectAudienceNetworkCookie(t *testing.T) { func TestOptOutReset(t *testing.T) { cookie := newSampleCookie() - cookie.SetPreference(false) + cookie.SetOptOut(true) if cookie.AllowSyncs() { - t.Error("After SetPreference(false), a cookie should not allow more user syncs.") + t.Error("After SetOptOut(true), a cookie should not allow more user syncs.") } ensureConsistency(t, cookie) } @@ -104,9 +104,9 @@ func TestOptIn(t *testing.T) { birthday: timestamp(), } - cookie.SetPreference(true) + cookie.SetOptOut(false) if !cookie.AllowSyncs() { - t.Error("After SetPreference(true), a cookie should allow more user syncs.") + t.Error("After SetOptOut(false), a cookie should allow more user syncs.") } ensureConsistency(t, cookie) } @@ -168,13 +168,14 @@ func TestCookieReadWrite(t *testing.T) { if !exists || !isLive || uid != "123" { t.Errorf("Received cookie should have the adnxs ID=123. Got %s", uid) } + uid, exists, isLive = received.GetUID("rubicon") if !exists || !isLive || uid != "456" { t.Errorf("Received cookie should have the rubicon ID=456. Got %s", uid) } - if received.LiveSyncCount() != 2 { - t.Errorf("Expected 2 user syncs. Got %d", received.LiveSyncCount()) - } + + assert.True(t, received.HasAnyLiveSyncs(), "Has Live Syncs") + assert.Len(t, received.uids, 2, "Sync Count") } func TestPopulatedLegacyCookieRead(t *testing.T) { @@ -182,8 +183,8 @@ func TestPopulatedLegacyCookieRead(t *testing.T) { var cookie Cookie json.Unmarshal([]byte(legacyJson), &cookie) - if cookie.LiveSyncCount() != 0 { - t.Errorf("Expected 0 user syncs. Got %d", cookie.LiveSyncCount()) + if cookie.HasAnyLiveSyncs() { + t.Error("Expected 0 user syncs. Found at least 1.") } if cookie.HasLiveSync("adnxs") { t.Errorf("Received cookie should act like it has no ID for adnxs.") @@ -198,8 +199,8 @@ func TestEmptyLegacyCookieRead(t *testing.T) { var cookie Cookie json.Unmarshal([]byte(legacyJson), &cookie) - if cookie.LiveSyncCount() != 0 { - t.Errorf("Expected 0 user syncs. Got %d", cookie.LiveSyncCount()) + if cookie.HasAnyLiveSyncs() { + t.Error("Expected 0 user syncs. Found at least 1.") } } @@ -210,7 +211,7 @@ func TestNilCookie(t *testing.T) { t.Error("nil cookies should respond with false when asked if they have a sync") } - if nilCookie.LiveSyncCount() != 0 { + if nilCookie.HasAnyLiveSyncs() { t.Error("nil cookies shouldn't have any syncs.") } @@ -298,8 +299,8 @@ func ensureEmptyMap(t *testing.T, cookie *Cookie) { if !cookie.AllowSyncs() { t.Error("Empty cookies should allow user syncs.") } - if cookie.LiveSyncCount() != 0 { - t.Errorf("Empty cookies shouldn't have any user syncs. Found %d.", cookie.LiveSyncCount()) + if cookie.HasAnyLiveSyncs() { + t.Error("Empty cookies shouldn't have any user syncs. Found at least 1.") } } @@ -330,8 +331,8 @@ func ensureConsistency(t *testing.T, cookie *Cookie) { t.Error("PBSCookie.GetUID() should return empty strings if it doesn't have a sync") } } else { - if cookie.LiveSyncCount() != 0 { - t.Errorf("If the user opted out, the PBSCookie should have no user syncs. Got %d", cookie.LiveSyncCount()) + if cookie.HasAnyLiveSyncs() { + t.Error("If the user opted out, the PBSCookie should have no user syncs.") } err := cookie.TrySync("adnxs", "123") @@ -344,9 +345,8 @@ func ensureConsistency(t *testing.T, cookie *Cookie) { if copiedCookie.AllowSyncs() != cookie.AllowSyncs() { t.Error("The PBSCookie interface shouldn't let modifications happen if the user has opted out") } - if cookie.LiveSyncCount() != copiedCookie.LiveSyncCount() { - t.Errorf("Incorrect sync count. Expected %d, got %d", copiedCookie.LiveSyncCount(), cookie.LiveSyncCount()) - } + + assert.Equal(t, len(cookie.uids), len(copiedCookie.uids), "Incorrect sync count on reparsed cookie.") for family, uid := range copiedCookie.uids { if !cookie.HasLiveSync(family) { From 0a0de85d0f8809c61d557013becd01948ef801f6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 28 May 2021 01:22:57 -0400 Subject: [PATCH 05/50] Large Development Checkpoint --- adapters/33across/usersync.go | 12 - adapters/33across/usersync_test.go | 34 - adapters/acuityads/acuityads.go | 4 +- adapters/acuityads/usersync.go | 12 - adapters/acuityads/usersync_test.go | 33 - adapters/adform/usersync.go | 12 - adapters/adform/usersync_test.go | 31 - adapters/adhese/adhese.go | 4 +- adapters/adkernel/adkernel.go | 4 +- adapters/adkernel/usersync.go | 12 - adapters/adkernel/usersync_test.go | 34 - adapters/adkernelAdn/adkernelAdn.go | 4 +- adapters/adkernelAdn/usersync.go | 12 - adapters/adkernelAdn/usersync_test.go | 34 - adapters/adman/usersync.go | 13 - adapters/adman/usersync_test.go | 34 - adapters/admixer/usersync.go | 12 - adapters/admixer/usersync_test.go | 35 - adapters/adocean/adocean.go | 4 +- adapters/adocean/usersync.go | 12 - adapters/adocean/usersync_test.go | 33 - adapters/adoppler/adoppler.go | 2 +- adapters/adpone/usersync.go | 18 - adapters/adpone/usersync_test.go | 23 - adapters/adtarget/usersync.go | 12 - adapters/adtarget/usersync_test.go | 37 - adapters/adtelligent/usersync.go | 12 - adapters/adtelligent/usersync_test.go | 29 - adapters/advangelists/advangelists.go | 4 +- adapters/advangelists/usersync.go | 12 - adapters/advangelists/usersync_test.go | 30 - adapters/adyoulike/usersync.go | 12 - adapters/adyoulike/usersync_test.go | 35 - adapters/aja/usersync.go | 12 - adapters/aja/usersync_test.go | 35 - adapters/amx/usersync.go | 13 - adapters/amx/usersync_test.go | 22 - adapters/appnexus/usersync.go | 12 - adapters/appnexus/usersync_test.go | 24 - adapters/audienceNetwork/usersync.go | 12 - adapters/audienceNetwork/usersync_test.go | 24 - adapters/avocet/usersync.go | 12 - adapters/avocet/usersync_test.go | 34 - adapters/beachfront/usersync.go | 15 - adapters/beachfront/usersync_test.go | 34 - adapters/beintoo/usersync.go | 12 - adapters/beintoo/usersync_test.go | 34 - adapters/between/between.go | 4 +- adapters/between/usersync.go | 13 - adapters/between/usersync_test.go | 24 - adapters/brightroll/usersync.go | 12 - adapters/brightroll/usersync_test.go | 24 - adapters/colossus/usersync.go | 13 - adapters/colossus/usersync_test.go | 34 - adapters/connectad/usersync.go | 12 - adapters/connectad/usersync_test.go | 34 - adapters/consumable/usersync.go | 15 - adapters/consumable/usersync_test.go | 34 - adapters/conversant/usersync.go | 12 - adapters/conversant/usersync_test.go | 29 - adapters/cpmstar/usersync.go | 13 - adapters/cpmstar/usersync_test.go | 24 - adapters/datablocks/datablocks.go | 4 +- adapters/datablocks/usersync.go | 12 - adapters/datablocks/usersync_test.go | 34 - adapters/deepintent/usersync.go | 13 - adapters/deepintent/usersync_test.go | 34 - adapters/dmx/usersync.go | 12 - adapters/dmx/usersync_test.go | 20 - adapters/emx_digital/usersync.go | 12 - adapters/emx_digital/usersync_test.go | 34 - adapters/engagebdr/usersync.go | 12 - adapters/engagebdr/usersync_test.go | 34 - adapters/eplanning/usersync.go | 12 - adapters/eplanning/usersync_test.go | 24 - adapters/gamma/usersync.go | 12 - adapters/gamma/usersync_test.go | 29 - adapters/gamoshi/usersync.go | 12 - adapters/gamoshi/usersync_test.go | 29 - adapters/grid/usersync.go | 12 - adapters/grid/usersync_test.go | 29 - adapters/gumgum/usersync.go | 12 - adapters/gumgum/usersync_test.go | 34 - adapters/improvedigital/usersync.go | 12 - adapters/improvedigital/usersync_test.go | 34 - adapters/invibes/invibes.go | 4 +- adapters/invibes/usersync.go | 16 - adapters/invibes/usersync_test.go | 31 - adapters/ix/usersync.go | 12 - adapters/ix/usersync_test.go | 24 - adapters/jixie/usersync.go | 12 - adapters/jixie/usersync_test.go | 24 - adapters/krushmedia/krushmedia.go | 4 +- adapters/krushmedia/usersync.go | 12 - adapters/krushmedia/usersync_test.go | 33 - adapters/lifestreet/usersync.go | 12 - adapters/lifestreet/usersync_test.go | 29 - adapters/lockerdome/usersync.go | 12 - adapters/lockerdome/usersync_test.go | 24 - adapters/logicad/usersync.go | 12 - adapters/logicad/usersync_test.go | 30 - adapters/lunamedia/lunamedia.go | 4 +- adapters/lunamedia/usersync.go | 12 - adapters/lunamedia/usersync_test.go | 30 - adapters/marsmedia/usersync.go | 12 - adapters/marsmedia/usersync_test.go | 35 - adapters/mediafuse/usersync.go | 12 - adapters/mediafuse/usersync_test.go | 29 - adapters/mgid/usersync.go | 12 - adapters/mgid/usersync_test.go | 29 - adapters/mobilefuse/mobilefuse.go | 4 +- adapters/nanointeractive/usersync.go | 12 - adapters/nanointeractive/usersync_test.go | 35 - adapters/ninthdecimal/ninthdecimal.go | 4 +- adapters/ninthdecimal/usersync.go | 12 - adapters/ninthdecimal/usersync_test.go | 30 - adapters/nobid/usersync.go | 12 - adapters/nobid/usersync_test.go | 34 - adapters/onetag/onetag.go | 4 +- adapters/onetag/usersync.go | 12 - adapters/onetag/usersync_test.go | 23 - adapters/openx/usersync.go | 12 - adapters/openx/usersync_test.go | 24 - adapters/pubmatic/usersync.go | 12 - adapters/pubmatic/usersync_test.go | 34 - adapters/pulsepoint/usersync.go | 12 - adapters/pulsepoint/usersync_test.go | 24 - adapters/rhythmone/usersync.go | 12 - adapters/rhythmone/usersync_test.go | 34 - adapters/rtbhouse/usersync.go | 18 - adapters/rtbhouse/usersync_test.go | 29 - adapters/rubicon/usersync.go | 12 - adapters/rubicon/usersync_test.go | 30 - adapters/sharethrough/usersync.go | 12 - adapters/sharethrough/usersync_test.go | 30 - adapters/silvermob/silvermob.go | 4 +- adapters/smartadserver/usersync.go | 12 - adapters/smartadserver/usersync_test.go | 34 - adapters/smartrtb/smartrtb.go | 4 +- adapters/smartrtb/usersync.go | 12 - adapters/smartrtb/usersync_test.go | 19 - adapters/smartyads/smartyads.go | 4 +- adapters/smartyads/usersync.go | 12 - adapters/smartyads/usersync_test.go | 33 - adapters/somoaudience/usersync.go | 12 - adapters/somoaudience/usersync_test.go | 24 - adapters/sonobi/usersync.go | 12 - adapters/sonobi/usersync_test.go | 29 - adapters/sovrn/usersync.go | 12 - adapters/sovrn/usersync_test.go | 29 - adapters/synacormedia/synacormedia.go | 4 +- adapters/synacormedia/usersync.go | 12 - adapters/synacormedia/usersync_test.go | 24 - adapters/syncer.go | 51 - adapters/syncer_test.go | 36 - adapters/tappx/tappx.go | 4 +- adapters/tappx/usersync.go | 12 - adapters/tappx/usersync_test.go | 34 - adapters/telaria/usersync.go | 12 - adapters/telaria/usersync_test.go | 32 - adapters/triplelift/usersync.go | 12 - adapters/triplelift/usersync_test.go | 24 - adapters/triplelift_native/usersync.go | 12 - adapters/triplelift_native/usersync_test.go | 24 - adapters/trustx/usersync.go | 12 - adapters/trustx/usersync_test.go | 29 - adapters/ucfunnel/usersync.go | 12 - adapters/ucfunnel/usersync_test.go | 29 - adapters/unruly/usersync.go | 12 - adapters/unruly/usersync_test.go | 34 - adapters/valueimpression/usersync.go | 12 - adapters/valueimpression/usersync_test.go | 34 - adapters/verizonmedia/usersync.go | 12 - adapters/verizonmedia/usersync_test.go | 22 - adapters/visx/usersync.go | 12 - adapters/visx/usersync_test.go | 34 - adapters/vrtcal/usersync.go | 12 - adapters/vrtcal/usersync_test.go | 30 - adapters/yeahmobi/yeahmobi.go | 4 +- adapters/yieldlab/usersync.go | 12 - adapters/yieldlab/usersync_test.go | 25 - adapters/yieldmo/usersync.go | 12 - adapters/yieldmo/usersync_test.go | 29 - adapters/yieldone/usersync.go | 12 - adapters/yieldone/usersync_test.go | 29 - adapters/zeroclickfraud/usersync.go | 12 - adapters/zeroclickfraud/usersync_test.go | 34 - adapters/zeroclickfraud/zeroclickfraud.go | 4 +- analytics/core.go | 15 +- analytics/filesystem/file_module_test.go | 6 +- analytics/pubstack/helpers/json_test.go | 8 +- config/adapter.go | 77 +- config/bidderinfo.go | 90 +- config/config.go | 109 +- config/config_test.go | 5 - config/usersync.go | 12 +- endpoints/auction.go | 18 +- endpoints/auction_test.go | 4 +- endpoints/cookie_sync.go | 477 ++++--- endpoints/cookie_sync_test.go | 1388 +++++++++++++++++-- endpoints/httprouterhandler.go | 11 + endpoints/setuid.go | 8 +- endpoints/setuid_test.go | 22 +- exchange/exchange_test.go | 5 +- exchange/legacy_test.go | 5 - gdpr/gdpr.go | 20 - gdpr/gdpr_test.go | 53 - gdpr/impl.go | 31 +- gdpr/impl_test.go | 58 - gdpr/signal.go | 46 + gdpr/signal_test.go | 116 ++ macros/macros.go | 2 +- macros/macros_test.go | 32 +- metrics/config/metrics.go | 20 +- metrics/config/metrics_test.go | 6 +- metrics/config/todo | 6 + metrics/go_metrics.go | 15 +- metrics/metrics.go | 42 +- metrics/metrics_mock.go | 10 +- metrics/prometheus/preload.go | 18 +- metrics/prometheus/prometheus.go | 82 +- metrics/prometheus/prometheus_test.go | 60 +- metrics/prometheus/type_conversion.go | 18 + pbs/pbsrequest.go | 26 +- router/router.go | 19 +- usersync/bidderchooser.go | 10 +- usersync/bidderchooser_test.go | 15 +- usersync/bidderfilter.go | 53 + usersync/bidderfilter_test.go | 104 ++ usersync/chooser.go | 79 +- usersync/chooser_test.go | 89 +- usersync/cookie.go | 18 +- usersync/cookie_test.go | 175 ++- usersync/shuffler.go | 2 + usersync/syncer.go | 200 ++- usersync/synctype.go | 49 + usersync/synctype_test.go | 81 ++ usersync/usersync.go | 29 - usersync/usersyncers/syncer.go | 195 --- 239 files changed, 2873 insertions(+), 4798 deletions(-) delete mode 100644 adapters/33across/usersync.go delete mode 100644 adapters/33across/usersync_test.go delete mode 100644 adapters/acuityads/usersync.go delete mode 100644 adapters/acuityads/usersync_test.go delete mode 100644 adapters/adform/usersync.go delete mode 100644 adapters/adform/usersync_test.go delete mode 100644 adapters/adkernel/usersync.go delete mode 100644 adapters/adkernel/usersync_test.go delete mode 100644 adapters/adkernelAdn/usersync.go delete mode 100644 adapters/adkernelAdn/usersync_test.go delete mode 100644 adapters/adman/usersync.go delete mode 100644 adapters/adman/usersync_test.go delete mode 100644 adapters/admixer/usersync.go delete mode 100644 adapters/admixer/usersync_test.go delete mode 100644 adapters/adocean/usersync.go delete mode 100644 adapters/adocean/usersync_test.go delete mode 100644 adapters/adpone/usersync.go delete mode 100644 adapters/adpone/usersync_test.go delete mode 100644 adapters/adtarget/usersync.go delete mode 100644 adapters/adtarget/usersync_test.go delete mode 100644 adapters/adtelligent/usersync.go delete mode 100644 adapters/adtelligent/usersync_test.go delete mode 100644 adapters/advangelists/usersync.go delete mode 100644 adapters/advangelists/usersync_test.go delete mode 100644 adapters/adyoulike/usersync.go delete mode 100644 adapters/adyoulike/usersync_test.go delete mode 100644 adapters/aja/usersync.go delete mode 100644 adapters/aja/usersync_test.go delete mode 100644 adapters/amx/usersync.go delete mode 100644 adapters/amx/usersync_test.go delete mode 100644 adapters/appnexus/usersync.go delete mode 100644 adapters/appnexus/usersync_test.go delete mode 100644 adapters/audienceNetwork/usersync.go delete mode 100644 adapters/audienceNetwork/usersync_test.go delete mode 100644 adapters/avocet/usersync.go delete mode 100644 adapters/avocet/usersync_test.go delete mode 100644 adapters/beachfront/usersync.go delete mode 100644 adapters/beachfront/usersync_test.go delete mode 100644 adapters/beintoo/usersync.go delete mode 100644 adapters/beintoo/usersync_test.go delete mode 100644 adapters/between/usersync.go delete mode 100644 adapters/between/usersync_test.go delete mode 100644 adapters/brightroll/usersync.go delete mode 100644 adapters/brightroll/usersync_test.go delete mode 100644 adapters/colossus/usersync.go delete mode 100644 adapters/colossus/usersync_test.go delete mode 100644 adapters/connectad/usersync.go delete mode 100644 adapters/connectad/usersync_test.go delete mode 100644 adapters/consumable/usersync.go delete mode 100644 adapters/consumable/usersync_test.go delete mode 100644 adapters/conversant/usersync.go delete mode 100644 adapters/conversant/usersync_test.go delete mode 100644 adapters/cpmstar/usersync.go delete mode 100644 adapters/cpmstar/usersync_test.go delete mode 100644 adapters/datablocks/usersync.go delete mode 100644 adapters/datablocks/usersync_test.go delete mode 100644 adapters/deepintent/usersync.go delete mode 100644 adapters/deepintent/usersync_test.go delete mode 100644 adapters/dmx/usersync.go delete mode 100644 adapters/dmx/usersync_test.go delete mode 100644 adapters/emx_digital/usersync.go delete mode 100644 adapters/emx_digital/usersync_test.go delete mode 100644 adapters/engagebdr/usersync.go delete mode 100644 adapters/engagebdr/usersync_test.go delete mode 100644 adapters/eplanning/usersync.go delete mode 100644 adapters/eplanning/usersync_test.go delete mode 100644 adapters/gamma/usersync.go delete mode 100644 adapters/gamma/usersync_test.go delete mode 100644 adapters/gamoshi/usersync.go delete mode 100644 adapters/gamoshi/usersync_test.go delete mode 100644 adapters/grid/usersync.go delete mode 100644 adapters/grid/usersync_test.go delete mode 100644 adapters/gumgum/usersync.go delete mode 100644 adapters/gumgum/usersync_test.go delete mode 100644 adapters/improvedigital/usersync.go delete mode 100644 adapters/improvedigital/usersync_test.go delete mode 100644 adapters/invibes/usersync.go delete mode 100644 adapters/invibes/usersync_test.go delete mode 100644 adapters/ix/usersync.go delete mode 100644 adapters/ix/usersync_test.go delete mode 100644 adapters/jixie/usersync.go delete mode 100644 adapters/jixie/usersync_test.go delete mode 100644 adapters/krushmedia/usersync.go delete mode 100644 adapters/krushmedia/usersync_test.go delete mode 100644 adapters/lifestreet/usersync.go delete mode 100644 adapters/lifestreet/usersync_test.go delete mode 100644 adapters/lockerdome/usersync.go delete mode 100644 adapters/lockerdome/usersync_test.go delete mode 100644 adapters/logicad/usersync.go delete mode 100644 adapters/logicad/usersync_test.go delete mode 100644 adapters/lunamedia/usersync.go delete mode 100644 adapters/lunamedia/usersync_test.go delete mode 100644 adapters/marsmedia/usersync.go delete mode 100644 adapters/marsmedia/usersync_test.go delete mode 100644 adapters/mediafuse/usersync.go delete mode 100644 adapters/mediafuse/usersync_test.go delete mode 100644 adapters/mgid/usersync.go delete mode 100644 adapters/mgid/usersync_test.go delete mode 100644 adapters/nanointeractive/usersync.go delete mode 100644 adapters/nanointeractive/usersync_test.go delete mode 100755 adapters/ninthdecimal/usersync.go delete mode 100755 adapters/ninthdecimal/usersync_test.go delete mode 100644 adapters/nobid/usersync.go delete mode 100644 adapters/nobid/usersync_test.go delete mode 100644 adapters/onetag/usersync.go delete mode 100644 adapters/onetag/usersync_test.go delete mode 100644 adapters/openx/usersync.go delete mode 100644 adapters/openx/usersync_test.go delete mode 100644 adapters/pubmatic/usersync.go delete mode 100644 adapters/pubmatic/usersync_test.go delete mode 100644 adapters/pulsepoint/usersync.go delete mode 100644 adapters/pulsepoint/usersync_test.go delete mode 100644 adapters/rhythmone/usersync.go delete mode 100644 adapters/rhythmone/usersync_test.go delete mode 100644 adapters/rtbhouse/usersync.go delete mode 100644 adapters/rtbhouse/usersync_test.go delete mode 100644 adapters/rubicon/usersync.go delete mode 100644 adapters/rubicon/usersync_test.go delete mode 100644 adapters/sharethrough/usersync.go delete mode 100644 adapters/sharethrough/usersync_test.go delete mode 100644 adapters/smartadserver/usersync.go delete mode 100644 adapters/smartadserver/usersync_test.go delete mode 100644 adapters/smartrtb/usersync.go delete mode 100644 adapters/smartrtb/usersync_test.go delete mode 100644 adapters/smartyads/usersync.go delete mode 100644 adapters/smartyads/usersync_test.go delete mode 100644 adapters/somoaudience/usersync.go delete mode 100644 adapters/somoaudience/usersync_test.go delete mode 100644 adapters/sonobi/usersync.go delete mode 100644 adapters/sonobi/usersync_test.go delete mode 100644 adapters/sovrn/usersync.go delete mode 100644 adapters/sovrn/usersync_test.go delete mode 100644 adapters/synacormedia/usersync.go delete mode 100644 adapters/synacormedia/usersync_test.go delete mode 100644 adapters/syncer.go delete mode 100644 adapters/syncer_test.go delete mode 100644 adapters/tappx/usersync.go delete mode 100644 adapters/tappx/usersync_test.go delete mode 100644 adapters/telaria/usersync.go delete mode 100644 adapters/telaria/usersync_test.go delete mode 100644 adapters/triplelift/usersync.go delete mode 100644 adapters/triplelift/usersync_test.go delete mode 100644 adapters/triplelift_native/usersync.go delete mode 100644 adapters/triplelift_native/usersync_test.go delete mode 100644 adapters/trustx/usersync.go delete mode 100644 adapters/trustx/usersync_test.go delete mode 100644 adapters/ucfunnel/usersync.go delete mode 100644 adapters/ucfunnel/usersync_test.go delete mode 100644 adapters/unruly/usersync.go delete mode 100644 adapters/unruly/usersync_test.go delete mode 100644 adapters/valueimpression/usersync.go delete mode 100644 adapters/valueimpression/usersync_test.go delete mode 100644 adapters/verizonmedia/usersync.go delete mode 100644 adapters/verizonmedia/usersync_test.go delete mode 100644 adapters/visx/usersync.go delete mode 100644 adapters/visx/usersync_test.go delete mode 100644 adapters/vrtcal/usersync.go delete mode 100644 adapters/vrtcal/usersync_test.go delete mode 100644 adapters/yieldlab/usersync.go delete mode 100644 adapters/yieldlab/usersync_test.go delete mode 100644 adapters/yieldmo/usersync.go delete mode 100644 adapters/yieldmo/usersync_test.go delete mode 100644 adapters/yieldone/usersync.go delete mode 100644 adapters/yieldone/usersync_test.go delete mode 100644 adapters/zeroclickfraud/usersync.go delete mode 100644 adapters/zeroclickfraud/usersync_test.go create mode 100644 endpoints/httprouterhandler.go create mode 100644 gdpr/signal.go create mode 100644 gdpr/signal_test.go create mode 100644 metrics/config/todo create mode 100644 usersync/bidderfilter.go create mode 100644 usersync/bidderfilter_test.go create mode 100644 usersync/synctype.go create mode 100644 usersync/synctype_test.go delete mode 100644 usersync/usersync.go delete mode 100644 usersync/usersyncers/syncer.go diff --git a/adapters/33across/usersync.go b/adapters/33across/usersync.go deleted file mode 100644 index df26f3b6325..00000000000 --- a/adapters/33across/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package ttx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func New33AcrossSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("33across", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/33across/usersync_test.go b/adapters/33across/usersync_test.go deleted file mode 100644 index e99b5965746..00000000000 --- a/adapters/33across/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package ttx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func Test33AcrossSyncer(t *testing.T) { - syncURL := "https://ic.tynt.com/r/d?m=xch&rt=html&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&ru=%2Fsetuid%3Fbidder%3D33across%26uid%3D33XUSERID33X&id=zzz000000000002zzz" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := New33AcrossSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ic.tynt.com/r/d?m=xch&rt=html&gdpr=A&gdpr_consent=B&us_privacy=C&ru=%2Fsetuid%3Fbidder%3D33across%26uid%3D33XUSERID33X&id=zzz000000000002zzz", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/acuityads/acuityads.go b/adapters/acuityads/acuityads.go index 9c6f73c27f0..be958bc33e8 100644 --- a/adapters/acuityads/acuityads.go +++ b/adapters/acuityads/acuityads.go @@ -15,7 +15,7 @@ import ( ) type AcuityAdsAdapter struct { - endpoint template.Template + endpoint *template.Template } // Builder builds a new instance of the AcuityAds adapter for the given bidder with the given config. @@ -26,7 +26,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &AcuityAdsAdapter{ - endpoint: *template, + endpoint: template, } return bidder, nil } diff --git a/adapters/acuityads/usersync.go b/adapters/acuityads/usersync.go deleted file mode 100644 index e2fc1f41961..00000000000 --- a/adapters/acuityads/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package acuityads - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAcuityAdsSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("acuityads", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/acuityads/usersync_test.go b/adapters/acuityads/usersync_test.go deleted file mode 100644 index b3ad10bdbb8..00000000000 --- a/adapters/acuityads/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package acuityads - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAcuityAdsSyncer(t *testing.T) { - syncURL := "https://cs.admanmedia.com/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dacuityads%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewAcuityAdsSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "ANDFJDS", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://cs.admanmedia.com/sync/prebid?gdpr=0&gdpr_consent=ANDFJDS&us_privacy=1-YY&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dacuityads%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adform/usersync.go b/adapters/adform/usersync.go deleted file mode 100644 index 6a237f794a6..00000000000 --- a/adapters/adform/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adform - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdformSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adform", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adform/usersync_test.go b/adapters/adform/usersync_test.go deleted file mode 100644 index f133be86583..00000000000 --- a/adapters/adform/usersync_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package adform - -import ( - "testing" - "text/template" - - "github.com/stretchr/testify/assert" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" -) - -func TestAdformSyncer(t *testing.T) { - syncURL := "//cm.adform.net?return_url=localhost%2Fsetuid%3Fbidder%3Dadform%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdformSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//cm.adform.net?return_url=localhost%2Fsetuid%3Fbidder%3Dadform%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adhese/adhese.go b/adapters/adhese/adhese.go index e13c03a101d..7defa9bc97d 100644 --- a/adapters/adhese/adhese.go +++ b/adapters/adhese/adhese.go @@ -20,7 +20,7 @@ import ( ) type AdheseAdapter struct { - endpointTemplate template.Template + endpointTemplate *template.Template } func extractSlotParameter(parameters openrtb_ext.ExtImpAdhese) string { @@ -276,7 +276,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &AdheseAdapter{ - endpointTemplate: *template, + endpointTemplate: template, } return bidder, nil } diff --git a/adapters/adkernel/adkernel.go b/adapters/adkernel/adkernel.go index 6bedeec612f..104d6e09dd6 100644 --- a/adapters/adkernel/adkernel.go +++ b/adapters/adkernel/adkernel.go @@ -16,7 +16,7 @@ import ( ) type adkernelAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } //MakeRequests prepares request information for prebid-server core @@ -262,7 +262,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &adkernelAdapter{ - EndpointTemplate: *urlTemplate, + EndpointTemplate: urlTemplate, } return bidder, nil } diff --git a/adapters/adkernel/usersync.go b/adapters/adkernel/usersync.go deleted file mode 100644 index 2de82a00f6e..00000000000 --- a/adapters/adkernel/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adkernel - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdkernelSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adkernel", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adkernel/usersync_test.go b/adapters/adkernel/usersync_test.go deleted file mode 100644 index 2ff81668d41..00000000000 --- a/adapters/adkernel/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package adkernel - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdkernelAdnSyncer(t *testing.T) { - syncURL := "https://sync.adkernel.com/user-sync?t=image&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dadkernel%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdkernelSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.adkernel.com/user-sync?t=image&gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dadkernel%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adkernelAdn/adkernelAdn.go b/adapters/adkernelAdn/adkernelAdn.go index acd27c9e894..0141626f29d 100644 --- a/adapters/adkernelAdn/adkernelAdn.go +++ b/adapters/adkernelAdn/adkernelAdn.go @@ -18,7 +18,7 @@ import ( const defaultDomain string = "tag.adkernel.com" type adkernelAdnAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } //MakeRequests prepares request information for prebid-server core @@ -286,7 +286,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &adkernelAdnAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/adkernelAdn/usersync.go b/adapters/adkernelAdn/usersync.go deleted file mode 100644 index 5a890e1565b..00000000000 --- a/adapters/adkernelAdn/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adkernelAdn - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdkernelAdnSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adkernelAdn", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adkernelAdn/usersync_test.go b/adapters/adkernelAdn/usersync_test.go deleted file mode 100644 index 17cf97ce779..00000000000 --- a/adapters/adkernelAdn/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package adkernelAdn - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdkernelAdnSyncer(t *testing.T) { - syncURL := "https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3DadkernelAdn%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdkernelAdnSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://tag.adkernel.com/syncr?gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3DadkernelAdn%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adman/usersync.go b/adapters/adman/usersync.go deleted file mode 100644 index 2cb62fe7824..00000000000 --- a/adapters/adman/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package adman - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -// NewAdmanSyncer returns adman syncer -func NewAdmanSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adman", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adman/usersync_test.go b/adapters/adman/usersync_test.go deleted file mode 100644 index d0dc90c8c5d..00000000000 --- a/adapters/adman/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package adman - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdmanSyncer(t *testing.T) { - syncURL := "https://sync.admanmedia.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dadman%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdmanSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "ANDFJDS", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.admanmedia.com/pbs.gif?gdpr=0&gdpr_consent=ANDFJDS&us_privacy=1-YY&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dadman%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/admixer/usersync.go b/adapters/admixer/usersync.go deleted file mode 100644 index 89e162dff32..00000000000 --- a/adapters/admixer/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package admixer - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdmixerSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("admixer", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/admixer/usersync_test.go b/adapters/admixer/usersync_test.go deleted file mode 100644 index 6acc48453fb..00000000000 --- a/adapters/admixer/usersync_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package admixer - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdmixerSyncer(t *testing.T) { - syncURL := "http://anyHost/anyPath" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdmixerSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "http://anyHost/anyPath", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adocean/adocean.go b/adapters/adocean/adocean.go index 43aa067b3a9..fa1f09b6577 100644 --- a/adapters/adocean/adocean.go +++ b/adapters/adocean/adocean.go @@ -69,14 +69,14 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters whiteSpace := regexp.MustCompile(`\s+`) bidder := &AdOceanAdapter{ - endpointTemplate: *endpointTemplate, + endpointTemplate: endpointTemplate, measurementCode: whiteSpace.ReplaceAllString(measurementCode, " "), } return bidder, nil } type AdOceanAdapter struct { - endpointTemplate template.Template + endpointTemplate *template.Template measurementCode string } diff --git a/adapters/adocean/usersync.go b/adapters/adocean/usersync.go deleted file mode 100644 index b189f822b46..00000000000 --- a/adapters/adocean/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adocean - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdOceanSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adocean", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adocean/usersync_test.go b/adapters/adocean/usersync_test.go deleted file mode 100644 index 5257017adfa..00000000000 --- a/adapters/adocean/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package adocean - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdOceanSyncer(t *testing.T) { - syncURL := "https://sync-host.com/redataredir/?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&url=localhost%2Fsetuid%3Fbidder%3Dadocean%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3DUUID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdOceanSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "consent-string", - }, - }) - - assert.NoError(t, err) - assert.Equal( - t, - "https://sync-host.com/redataredir/?gdpr=1&gdpr_consent=consent-string&url=localhost%2Fsetuid%3Fbidder%3Dadocean%26gdpr%3D1%26gdpr_consent%3Dconsent-string%26uid%3DUUID", - syncInfo.URL, - ) - assert.Equal(t, "redirect", syncInfo.Type) -} diff --git a/adapters/adoppler/adoppler.go b/adapters/adoppler/adoppler.go index adbb177d887..47c7f6b2f2e 100644 --- a/adapters/adoppler/adoppler.go +++ b/adapters/adoppler/adoppler.go @@ -200,7 +200,7 @@ func (ads *AdopplerAdapter) bidUri(ext *openrtb_ext.ExtImpAdoppler) (string, err params.AccountID = url.PathEscape(ext.Client) } - return macros.ResolveMacros(*ads.endpoint, params) + return macros.ResolveMacros(ads.endpoint, params) } func unmarshalExt(ext json.RawMessage) (*openrtb_ext.ExtImpAdoppler, error) { diff --git a/adapters/adpone/usersync.go b/adapters/adpone/usersync.go deleted file mode 100644 index 38dec19bb72..00000000000 --- a/adapters/adpone/usersync.go +++ /dev/null @@ -1,18 +0,0 @@ -package adpone - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -const adponeFamilyName = "adpone" - -func NewadponeSyncer(urlTemplate *template.Template) usersync.Usersyncer { - return adapters.NewSyncer( - adponeFamilyName, - urlTemplate, - adapters.SyncTypeRedirect, - ) -} diff --git a/adapters/adpone/usersync_test.go b/adapters/adpone/usersync_test.go deleted file mode 100644 index e744de5eb91..00000000000 --- a/adapters/adpone/usersync_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package adpone - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestAdponeSyncer(t *testing.T) { - syncURL := "https://usersync.adpone.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dadtelligent%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewadponeSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://usersync.adpone.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dadtelligent%26gdpr%3D%26gdpr_consent%3D%26uid%3D%7Buid%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) -} diff --git a/adapters/adtarget/usersync.go b/adapters/adtarget/usersync.go deleted file mode 100644 index d720f110d89..00000000000 --- a/adapters/adtarget/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adtarget - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdtargetSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adtarget", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adtarget/usersync_test.go b/adapters/adtarget/usersync_test.go deleted file mode 100644 index ea6146ceec8..00000000000 --- a/adapters/adtarget/usersync_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package adtarget - -import ( - "fmt" - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy/ccpa" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdtargetSyncer(t *testing.T) { - syncURL := "//sync.console.adtarget.com.tr/csync?t=p&ep=0&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=localhost%2Fsetuid%3Fbidder%3Dadtarget%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D" - fmt.Println("adtarget sync") - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdtargetSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "123", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//sync.console.adtarget.com.tr/csync?t=p&ep=0&gdpr=0&gdpr_consent=123&us_privacy=1-YY&redir=localhost%2Fsetuid%3Fbidder%3Dadtarget%26gdpr%3D0%26gdpr_consent%3D123%26uid%3D%7Buid%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adtelligent/usersync.go b/adapters/adtelligent/usersync.go deleted file mode 100644 index 613c8e294f0..00000000000 --- a/adapters/adtelligent/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adtelligent - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdtelligentSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adtelligent", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/adtelligent/usersync_test.go b/adapters/adtelligent/usersync_test.go deleted file mode 100644 index 2430f377bd4..00000000000 --- a/adapters/adtelligent/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package adtelligent - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdtelligentSyncer(t *testing.T) { - syncURL := "//sync.adtelligent.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dadtelligent%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdtelligentSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//sync.adtelligent.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dadtelligent%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%7Buid%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/advangelists/advangelists.go b/adapters/advangelists/advangelists.go index 95ddec0f452..06367274746 100644 --- a/adapters/advangelists/advangelists.go +++ b/adapters/advangelists/advangelists.go @@ -15,7 +15,7 @@ import ( ) type AdvangelistsAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } //MakeRequests prepares request information for prebid-server core @@ -247,7 +247,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &AdvangelistsAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/advangelists/usersync.go b/adapters/advangelists/usersync.go deleted file mode 100644 index 83930774773..00000000000 --- a/adapters/advangelists/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package advangelists - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdvangelistsSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("advangelists", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/advangelists/usersync_test.go b/adapters/advangelists/usersync_test.go deleted file mode 100644 index 04ee7968d87..00000000000 --- a/adapters/advangelists/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package advangelists - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAdvangelistsSyncer(t *testing.T) { - syncURL := "https://nep.advangelists.com/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=advangelists&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&uid=$UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdvangelistsSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://nep.advangelists.com/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=advangelists&gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&uid=$UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/adyoulike/usersync.go b/adapters/adyoulike/usersync.go deleted file mode 100644 index ffea6f69a27..00000000000 --- a/adapters/adyoulike/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package adyoulike - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAdyoulikeSyncer(urlTemplate *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adyoulike", urlTemplate, adapters.SyncTypeRedirect) -} diff --git a/adapters/adyoulike/usersync_test.go b/adapters/adyoulike/usersync_test.go deleted file mode 100644 index 72def4cf9b0..00000000000 --- a/adapters/adyoulike/usersync_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package adyoulike - -import ( - "testing" - "text/template" - - "github.com/stretchr/testify/assert" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" -) - -func TestAdyoulikeSyncer(t *testing.T) { - syncURL := "//visitor.omnitagjs.com/visitor/bsync?uid=19340f4f097d16f41f34fc0274981ca4&name=PrebidServer&gdpr_consent_string={{.GDPRConsent}}&gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAdyoulikeSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//visitor.omnitagjs.com/visitor/bsync?uid=19340f4f097d16f41f34fc0274981ca4&name=PrebidServer&gdpr_consent_string=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&gdpr=1&us_privacy=1-YY", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/aja/usersync.go b/adapters/aja/usersync.go deleted file mode 100644 index 6a9fad74e32..00000000000 --- a/adapters/aja/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package aja - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAJASyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("aja", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/aja/usersync_test.go b/adapters/aja/usersync_test.go deleted file mode 100644 index dabd5e190b9..00000000000 --- a/adapters/aja/usersync_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package aja - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy/ccpa" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAJASyncer(t *testing.T) { - syncURL := "https://ad.as.amanad.adtdp.com/v1/sync/ssp?ssp=4&gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&redir=localhost/setuid?bidder=aja&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&uid=%s" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAJASyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ad.as.amanad.adtdp.com/v1/sync/ssp?ssp=4&gdpr=1&us_privacy=C&redir=localhost/setuid?bidder=aja&gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&uid=%s", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/amx/usersync.go b/adapters/amx/usersync.go deleted file mode 100644 index 17ad04d5cfb..00000000000 --- a/adapters/amx/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package amx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -// NewAMXSyncer produces an AMX RTB usersyncer -func NewAMXSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("amx", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/amx/usersync_test.go b/adapters/amx/usersync_test.go deleted file mode 100644 index b6b6e6babe8..00000000000 --- a/adapters/amx/usersync_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package amx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestAMXSyncer(t *testing.T) { - syncURL := "http://pbs.amxrtb.com/cchain/0?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&cb=localhost%2Fsetuid%3Fbidder%3Damx%26uid%3D" - syncURLTemplate := template.Must(template.New("sync-template").Parse(syncURL)) - - syncer := NewAMXSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "http://pbs.amxrtb.com/cchain/0?gdpr=&gdpr_consent=&cb=localhost%2Fsetuid%3Fbidder%3Damx%26uid%3D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/appnexus/usersync.go b/adapters/appnexus/usersync.go deleted file mode 100644 index d29f0e3cb6b..00000000000 --- a/adapters/appnexus/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package appnexus - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAppnexusSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("adnxs", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/appnexus/usersync_test.go b/adapters/appnexus/usersync_test.go deleted file mode 100644 index d01e5704e28..00000000000 --- a/adapters/appnexus/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package appnexus - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestAppNexusSyncer(t *testing.T) { - syncURL := "//ib.adnxs.com/getuid?https%3A%2F%2Fprebid.adnxs.com%2Fpbs%2Fv1%2Fsetuid%3Fbidder%3Dadnxs%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAppnexusSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//ib.adnxs.com/getuid?https%3A%2F%2Fprebid.adnxs.com%2Fpbs%2Fv1%2Fsetuid%3Fbidder%3Dadnxs%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/audienceNetwork/usersync.go b/adapters/audienceNetwork/usersync.go deleted file mode 100644 index 4dd0b68ccff..00000000000 --- a/adapters/audienceNetwork/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package audienceNetwork - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewFacebookSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("audienceNetwork", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/audienceNetwork/usersync_test.go b/adapters/audienceNetwork/usersync_test.go deleted file mode 100644 index 591ea74f7ba..00000000000 --- a/adapters/audienceNetwork/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package audienceNetwork - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestFacebookSyncer(t *testing.T) { - syncURL := "https://www.facebook.com/audiencenetwork/idsync/?partner=partnerId&callback=localhost%2Fsetuid%3Fbidder%3DaudienceNetwork%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewFacebookSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://www.facebook.com/audiencenetwork/idsync/?partner=partnerId&callback=localhost%2Fsetuid%3Fbidder%3DaudienceNetwork%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/avocet/usersync.go b/adapters/avocet/usersync.go deleted file mode 100644 index 0cfa055ae86..00000000000 --- a/adapters/avocet/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package avocet - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewAvocetSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("avocet", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/avocet/usersync_test.go b/adapters/avocet/usersync_test.go deleted file mode 100644 index bd4cd4145a2..00000000000 --- a/adapters/avocet/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package avocet - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestAvocetSyncer(t *testing.T) { - syncURL := "https://ads.avct.cloud/getuid?&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=%2Fsetuid%3Fbidder%3Davocet%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7BUUID%7D%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewAvocetSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "ConsentString", - }, - CCPA: ccpa.Policy{ - Consent: "PrivacyString", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ads.avct.cloud/getuid?&gdpr=1&gdpr_consent=ConsentString&us_privacy=PrivacyString&url=%2Fsetuid%3Fbidder%3Davocet%26gdpr%3D1%26gdpr_consent%3DConsentString%26uid%3D%7B%7BUUID%7D%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/beachfront/usersync.go b/adapters/beachfront/usersync.go deleted file mode 100644 index 72466b59478..00000000000 --- a/adapters/beachfront/usersync.go +++ /dev/null @@ -1,15 +0,0 @@ -package beachfront - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewBeachfrontSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer( - "beachfront", - temp, - adapters.SyncTypeIframe) -} diff --git a/adapters/beachfront/usersync_test.go b/adapters/beachfront/usersync_test.go deleted file mode 100644 index edb52bd4f53..00000000000 --- a/adapters/beachfront/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package beachfront - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestBeachfrontSyncer(t *testing.T) { - syncURL := "https://sync.bfmio.com/sync_s2s?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&url=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dbeachfront%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bio_cid%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewBeachfrontSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.bfmio.com/sync_s2s?gdpr=A&us_privacy=C&url=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dbeachfront%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%5Bio_cid%5D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/beintoo/usersync.go b/adapters/beintoo/usersync.go deleted file mode 100644 index fb60c6ab0a7..00000000000 --- a/adapters/beintoo/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package beintoo - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewBeintooSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("Beintoo", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/beintoo/usersync_test.go b/adapters/beintoo/usersync_test.go deleted file mode 100644 index 65d92e6d58f..00000000000 --- a/adapters/beintoo/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package beintoo - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestBeintooSyncer(t *testing.T) { - syncURL := "https://ib.beintoo.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=localhost%2Fsetuid%3Fbidder%3Dbeintoo%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewBeintooSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ib.beintoo.com/um?ssp=pbs&gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&redirect=localhost%2Fsetuid%3Fbidder%3Dbeintoo%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/between/between.go b/adapters/between/between.go index f8106bdd113..08d257f7f6c 100644 --- a/adapters/between/between.go +++ b/adapters/between/between.go @@ -17,7 +17,7 @@ import ( ) type BetweenAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // BetweenSSP requires bidfloor > 0. @@ -220,7 +220,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := BetweenAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return &bidder, nil } diff --git a/adapters/between/usersync.go b/adapters/between/usersync.go deleted file mode 100644 index 142282730a3..00000000000 --- a/adapters/between/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package between - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -// NewBetweenSyncer returns "between" syncer -func NewBetweenSyncer(template *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("between", template, adapters.SyncTypeRedirect) -} diff --git a/adapters/between/usersync_test.go b/adapters/between/usersync_test.go deleted file mode 100644 index c54f473b209..00000000000 --- a/adapters/between/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package between - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestNewBetweenSyncerSyncer(t *testing.T) { - syncURL := "https://ads.betweendigital.com/match?bidder_id=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&callback_url=localhost:8080%2Fsetuid%3Fbidder%3Dbetween%26gdpr%3D0%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUSER_ID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewBetweenSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://ads.betweendigital.com/match?bidder_id=pbs&gdpr=&gdpr_consent=&us_privacy=&callback_url=localhost:8080%2Fsetuid%3Fbidder%3Dbetween%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24%7BUSER_ID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/brightroll/usersync.go b/adapters/brightroll/usersync.go deleted file mode 100644 index 099303fdd01..00000000000 --- a/adapters/brightroll/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package brightroll - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewBrightrollSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("brightroll", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/brightroll/usersync_test.go b/adapters/brightroll/usersync_test.go deleted file mode 100644 index fac17b0a9ae..00000000000 --- a/adapters/brightroll/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package brightroll - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestBrightrollSyncer(t *testing.T) { - syncURL := "http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=localhost%2Fsetuid%3Fbidder%3Dbrightroll%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewBrightrollSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr=&euconsent=&us_privacy=&url=localhost%2Fsetuid%3Fbidder%3Dbrightroll%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/colossus/usersync.go b/adapters/colossus/usersync.go deleted file mode 100644 index bb25b187352..00000000000 --- a/adapters/colossus/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package colossus - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -// NewColossusSyncer returns colossus syncer -func NewColossusSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("colossus", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/colossus/usersync_test.go b/adapters/colossus/usersync_test.go deleted file mode 100644 index 4cb3a4bbbdf..00000000000 --- a/adapters/colossus/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package colossus - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestColossusSyncer(t *testing.T) { - syncURL := "https://sync.colossusssp.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dcolossus%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewColossusSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "A", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.colossusssp.com/pbs.gif?gdpr=0&gdpr_consent=A&us_privacy=1-YY&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dcolossus%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/connectad/usersync.go b/adapters/connectad/usersync.go deleted file mode 100644 index bae96656bce..00000000000 --- a/adapters/connectad/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package connectad - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewConnectAdSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("connectad", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/connectad/usersync_test.go b/adapters/connectad/usersync_test.go deleted file mode 100644 index 81b0dc47e33..00000000000 --- a/adapters/connectad/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package connectad - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestConnectAdSyncer(t *testing.T) { - syncURL := "https://cdn.connectad.io/connectmyusers.php?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb=localhost%2Fsetuid%3Fbidder%3Dconnectad%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewConnectAdSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "fakeconsent", - }, - CCPA: ccpa.Policy{ - Consent: "fake", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://cdn.connectad.io/connectmyusers.php?gdpr=1&consent=fakeconsent&us_privacy=fake&cb=localhost%2Fsetuid%3Fbidder%3Dconnectad%26gdpr%3D1%26gdpr_consent%3Dfakeconsent%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/consumable/usersync.go b/adapters/consumable/usersync.go deleted file mode 100644 index 0cb25ca9864..00000000000 --- a/adapters/consumable/usersync.go +++ /dev/null @@ -1,15 +0,0 @@ -package consumable - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewConsumableSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer( - "consumable", - temp, - adapters.SyncTypeRedirect) -} diff --git a/adapters/consumable/usersync_test.go b/adapters/consumable/usersync_test.go deleted file mode 100644 index 0af868a0a62..00000000000 --- a/adapters/consumable/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package consumable - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestConsumableSyncer(t *testing.T) { - syncURL := "//e.serverbid.com/udb/9969/match?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dconsumable%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewConsumableSyncer(syncURLTemplate) - u, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//e.serverbid.com/udb/9969/match?gdpr=A&euconsent=B&us_privacy=C&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dconsumable%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D", u.URL) - assert.Equal(t, "redirect", u.Type) - assert.Equal(t, false, u.SupportCORS) -} diff --git a/adapters/conversant/usersync.go b/adapters/conversant/usersync.go deleted file mode 100644 index a38631f282c..00000000000 --- a/adapters/conversant/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package conversant - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewConversantSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("conversant", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/conversant/usersync_test.go b/adapters/conversant/usersync_test.go deleted file mode 100644 index 62ab732c443..00000000000 --- a/adapters/conversant/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package conversant - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestConversantSyncer(t *testing.T) { - syncURL := "usersync?rurl=localhost%2Fsetuid%3Fbidder%3Dconversant%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewConversantSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "usersync?rurl=localhost%2Fsetuid%3Fbidder%3Dconversant%26gdpr%3D0%26gdpr_consent%3D%26uid%3D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/cpmstar/usersync.go b/adapters/cpmstar/usersync.go deleted file mode 100644 index d3086f65b24..00000000000 --- a/adapters/cpmstar/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package cpmstar - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -//NewCpmstarSyncer : -func NewCpmstarSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("cpmstar", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/cpmstar/usersync_test.go b/adapters/cpmstar/usersync_test.go deleted file mode 100644 index 9bae7062b1c..00000000000 --- a/adapters/cpmstar/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package cpmstar - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestCpmstarSyncer(t *testing.T) { - syncURL := "https://server.cpmstar.com/usersync.aspx?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri=http%3A%2F%2Flocalhost:8000%2Fsetuid%3Fbidder%3Dcpmstar%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26us_privacy%3D{{.USPrivacy}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewCpmstarSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://server.cpmstar.com/usersync.aspx?gdpr=&gdpr_consent=&us_privacy=&redirectUri=http%3A%2F%2Flocalhost:8000%2Fsetuid%3Fbidder%3Dcpmstar%26gdpr%3D%26gdpr_consent%3D%26us_privacy%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/datablocks/datablocks.go b/adapters/datablocks/datablocks.go index 56ac8f681d7..96982375859 100644 --- a/adapters/datablocks/datablocks.go +++ b/adapters/datablocks/datablocks.go @@ -16,7 +16,7 @@ import ( ) type DatablocksAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } func (a *DatablocksAdapter) MakeRequests(request *openrtb.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { @@ -186,7 +186,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &DatablocksAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/datablocks/usersync.go b/adapters/datablocks/usersync.go deleted file mode 100644 index 2c69963ae74..00000000000 --- a/adapters/datablocks/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package datablocks - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewDatablocksSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("datablocks", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/datablocks/usersync_test.go b/adapters/datablocks/usersync_test.go deleted file mode 100644 index d6ee580408a..00000000000 --- a/adapters/datablocks/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package datablocks - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestDatablocksSyncer(t *testing.T) { - syncURL := "https://sync.v5prebid.datablocks.net/s2ssync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Ddatablocks%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7Buid%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewDatablocksSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.v5prebid.datablocks.net/s2ssync?gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Ddatablocks%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%24%7Buid%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/deepintent/usersync.go b/adapters/deepintent/usersync.go deleted file mode 100644 index 7a17f5336d5..00000000000 --- a/adapters/deepintent/usersync.go +++ /dev/null @@ -1,13 +0,0 @@ -package deepintent - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -// NewDeepintentSyncer returns deepintent syncer -func NewDeepintentSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("deepintent", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/deepintent/usersync_test.go b/adapters/deepintent/usersync_test.go deleted file mode 100644 index 6709a838100..00000000000 --- a/adapters/deepintent/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package deepintent - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestDeepintentSyncer(t *testing.T) { - syncURL := "https://match.deepintent.com/usersync/136?id=unk&gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&url=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Ddeepintent%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bio_cid%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewDeepintentSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://match.deepintent.com/usersync/136?id=unk&gdpr=A&us_privacy=C&url=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Ddeepintent%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%5Bio_cid%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/dmx/usersync.go b/adapters/dmx/usersync.go deleted file mode 100644 index 07fedf1c124..00000000000 --- a/adapters/dmx/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package dmx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewDmxSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("dmx", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/dmx/usersync_test.go b/adapters/dmx/usersync_test.go deleted file mode 100644 index a316af04cfb..00000000000 --- a/adapters/dmx/usersync_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package dmx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - - "github.com/stretchr/testify/assert" -) - -func TestDmxSyncer(t *testing.T) { - temp := template.Must(template.New("sync-template").Parse("https://dmx.districtm.io/s/v1/img/s/10007")) - syncer := NewDmxSyncer(temp) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - assert.NoError(t, err) - assert.Equal(t, "https://dmx.districtm.io/s/v1/img/s/10007", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/emx_digital/usersync.go b/adapters/emx_digital/usersync.go deleted file mode 100644 index 2d3011e0d69..00000000000 --- a/adapters/emx_digital/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package emx_digital - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewEMXDigitalSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("emx_digital", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/emx_digital/usersync_test.go b/adapters/emx_digital/usersync_test.go deleted file mode 100644 index 7aaf133512f..00000000000 --- a/adapters/emx_digital/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package emx_digital - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestEMXDigitalSyncer(t *testing.T) { - syncURL := "https://cs.emxdgt.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=localhost%2Fsetuid%3Fbidder%3Demx_digital%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewEMXDigitalSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://cs.emxdgt.com/um?ssp=pbs&gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&redirect=localhost%2Fsetuid%3Fbidder%3Demx_digital%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/engagebdr/usersync.go b/adapters/engagebdr/usersync.go deleted file mode 100644 index 205097b07be..00000000000 --- a/adapters/engagebdr/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package engagebdr - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewEngageBDRSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("engagebdr", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/engagebdr/usersync_test.go b/adapters/engagebdr/usersync_test.go deleted file mode 100644 index 5cee3abe35c..00000000000 --- a/adapters/engagebdr/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package engagebdr - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestEngageBDRSyncer(t *testing.T) { - syncURL := "https://match.bnmla.com/usersync/s2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=localhost%2Fsetuid%3Fbidder%3Dengagebdr%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewEngageBDRSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://match.bnmla.com/usersync/s2s?gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&r=localhost%2Fsetuid%3Fbidder%3Dengagebdr%26gdpr%3D1%26gdpr_consent%3DBOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA%26uid%3D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/eplanning/usersync.go b/adapters/eplanning/usersync.go deleted file mode 100644 index 6e8abf5de27..00000000000 --- a/adapters/eplanning/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package eplanning - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewEPlanningSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("eplanning", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/eplanning/usersync_test.go b/adapters/eplanning/usersync_test.go deleted file mode 100644 index 922ab4e20ce..00000000000 --- a/adapters/eplanning/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package eplanning - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestEPlanningSyncer(t *testing.T) { - syncURL := "https://ads.us.e-planning.net/uspd/1/?du=https%3A%2F%2Fads.us.e-planning.net%2Fgetuid%2F1%2F5a1ad71d2d53a0f5%3Flocalhost%2Fsetuid%3Fbidder%3Deplanning%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewEPlanningSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://ads.us.e-planning.net/uspd/1/?du=https%3A%2F%2Fads.us.e-planning.net%2Fgetuid%2F1%2F5a1ad71d2d53a0f5%3Flocalhost%2Fsetuid%3Fbidder%3Deplanning%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/gamma/usersync.go b/adapters/gamma/usersync.go deleted file mode 100644 index 4e4412a1efd..00000000000 --- a/adapters/gamma/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package gamma - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewGammaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("gamma", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/gamma/usersync_test.go b/adapters/gamma/usersync_test.go deleted file mode 100644 index 7636e05a6ef..00000000000 --- a/adapters/gamma/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package gamma - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestGammaSyncer(t *testing.T) { - syncURL := "//hb.gammaplatform.com/sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dgamma%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewGammaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//hb.gammaplatform.com/sync?gdpr=0&gdpr_consent=&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dgamma%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/gamoshi/usersync.go b/adapters/gamoshi/usersync.go deleted file mode 100644 index 9d6cdd0e518..00000000000 --- a/adapters/gamoshi/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package gamoshi - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewGamoshiSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("gamoshi", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/gamoshi/usersync_test.go b/adapters/gamoshi/usersync_test.go deleted file mode 100644 index 0465365f5e3..00000000000 --- a/adapters/gamoshi/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package gamoshi - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/stretchr/testify/assert" -) - -func TestGamoshiSyncer(t *testing.T) { - syncURL := "https://rtb.gamoshi.io/user_sync_prebid?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dgamoshi%26gdpr%3D%7B%7B.GDPR%7D%7D%26gdpr_consent%3D%7B%7B.GDPRConsent%7D%7D%26uid%3D%5Bgusr%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewGamoshiSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - CCPA: ccpa.Policy{ - Consent: "anyValue", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://rtb.gamoshi.io/user_sync_prebid?gdpr=&consent=&us_privacy=anyValue&rurl=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dgamoshi%26gdpr%3D%7B%7B.GDPR%7D%7D%26gdpr_consent%3D%7B%7B.GDPRConsent%7D%7D%26uid%3D%5Bgusr%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/grid/usersync.go b/adapters/grid/usersync.go deleted file mode 100644 index 31788ea3f06..00000000000 --- a/adapters/grid/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package grid - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewGridSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("grid", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/grid/usersync_test.go b/adapters/grid/usersync_test.go deleted file mode 100644 index 2d7f3fc6a4f..00000000000 --- a/adapters/grid/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package grid - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestGridSyncer(t *testing.T) { - syncURL := "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dgrid%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewGridSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dgrid%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/gumgum/usersync.go b/adapters/gumgum/usersync.go deleted file mode 100644 index defef87dad9..00000000000 --- a/adapters/gumgum/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package gumgum - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewGumGumSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("gumgum", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/gumgum/usersync_test.go b/adapters/gumgum/usersync_test.go deleted file mode 100644 index 9b7c7aa4578..00000000000 --- a/adapters/gumgum/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package gumgum - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestGumGumSyncer(t *testing.T) { - syncURL := "https://rtb.gumgum.com/usync/prbds2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=localhost%2Fsetuid%3Fbidder%3Dgumgum%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewGumGumSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://rtb.gumgum.com/usync/prbds2s?gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&r=localhost%2Fsetuid%3Fbidder%3Dgumgum%26gdpr%3D1%26gdpr_consent%3DBOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA%26uid%3D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/improvedigital/usersync.go b/adapters/improvedigital/usersync.go deleted file mode 100644 index a58b97c3864..00000000000 --- a/adapters/improvedigital/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package improvedigital - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewImprovedigitalSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("improvedigital", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/improvedigital/usersync_test.go b/adapters/improvedigital/usersync_test.go deleted file mode 100644 index 9892e0cc703..00000000000 --- a/adapters/improvedigital/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package improvedigital - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestImprovedigitalSyncer(t *testing.T) { - syncURL := "https://ad.360yield.com/server_match?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=%2Fsetuid%3Fbidder%3Dimprovedigital%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BPUB_USER_ID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewImprovedigitalSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ad.360yield.com/server_match?gdpr=A&gdpr_consent=B&us_privacy=C&r=%2Fsetuid%3Fbidder%3Dimprovedigital%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%7BPUB_USER_ID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/invibes/invibes.go b/adapters/invibes/invibes.go index 31124bd108f..9801089680f 100644 --- a/adapters/invibes/invibes.go +++ b/adapters/invibes/invibes.go @@ -70,7 +70,7 @@ func (a *InvibesInternalParams) IsTestRequest() bool { } type InvibesAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // Builder builds a new instance of the Invibes adapter for the given bidder with the given config. @@ -81,7 +81,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := InvibesAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return &bidder, nil } diff --git a/adapters/invibes/usersync.go b/adapters/invibes/usersync.go deleted file mode 100644 index e6922444d44..00000000000 --- a/adapters/invibes/usersync.go +++ /dev/null @@ -1,16 +0,0 @@ -package invibes - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewInvibesSyncer(urlTemplate *template.Template) usersync.Usersyncer { - return adapters.NewSyncer( - "invibes", - urlTemplate, - adapters.SyncTypeIframe, - ) -} diff --git a/adapters/invibes/usersync_test.go b/adapters/invibes/usersync_test.go deleted file mode 100644 index 492886228fd..00000000000 --- a/adapters/invibes/usersync_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package invibes - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestInvibesSyncer(t *testing.T) { - syncURL := "http://localhost:56479/home/getLid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri=test.com%2Fsetuid%3Fbidder%3Dinvibes%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewInvibesSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "abc", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "http://localhost:56479/home/getLid?gdpr=1&gdpr_consent=abc&us_privacy=&redirectUri=test.com%2Fsetuid%3Fbidder%3Dinvibes%26gdpr%3D1%26gdpr_consent%3Dabc%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - -} diff --git a/adapters/ix/usersync.go b/adapters/ix/usersync.go deleted file mode 100644 index 621fa17945a..00000000000 --- a/adapters/ix/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package ix - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewIxSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("ix", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/ix/usersync_test.go b/adapters/ix/usersync_test.go deleted file mode 100644 index 3288b1ae443..00000000000 --- a/adapters/ix/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package ix - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestIxSyncer(t *testing.T) { - syncURL := "//ssum-sec.casalemedia.com/usermatchredir?s=184932&cb=localhost%2Fsetuid%3Fbidder%3Dix%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewIxSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//ssum-sec.casalemedia.com/usermatchredir?s=184932&cb=localhost%2Fsetuid%3Fbidder%3Dix%26gdpr%3D%26gdpr_consent%3D%26uid%3D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/jixie/usersync.go b/adapters/jixie/usersync.go deleted file mode 100644 index 137d78f2859..00000000000 --- a/adapters/jixie/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package jixie - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewJixieSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("jixie", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/jixie/usersync_test.go b/adapters/jixie/usersync_test.go deleted file mode 100644 index 575482435ff..00000000000 --- a/adapters/jixie/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package jixie - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestJixieSyncer(t *testing.T) { - syncURL := "https://id.jixie.io/api/sync?pid=&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Djixie%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25JXUID%25%25" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewJixieSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://id.jixie.io/api/sync?pid=&gdpr=&gdpr_consent=&us_privacy=&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Djixie%26gdpr%3D%26gdpr_consent%3D%26uid%3D%25%25JXUID%25%25", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/krushmedia/krushmedia.go b/adapters/krushmedia/krushmedia.go index d05b6e7979a..f962148a1ad 100644 --- a/adapters/krushmedia/krushmedia.go +++ b/adapters/krushmedia/krushmedia.go @@ -16,7 +16,7 @@ import ( ) type KrushmediaAdapter struct { - endpoint template.Template + endpoint *template.Template } // Builder builds a new instance of the KrushmediaA adapter for the given bidder with the given config. @@ -27,7 +27,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &KrushmediaAdapter{ - endpoint: *template, + endpoint: template, } return bidder, nil } diff --git a/adapters/krushmedia/usersync.go b/adapters/krushmedia/usersync.go deleted file mode 100644 index 860cb2204e9..00000000000 --- a/adapters/krushmedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package krushmedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewKrushmediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("krushmedia", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/krushmedia/usersync_test.go b/adapters/krushmedia/usersync_test.go deleted file mode 100644 index 765b0faa18b..00000000000 --- a/adapters/krushmedia/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package krushmedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestKrushmediaSyncer(t *testing.T) { - syncURL := "https://cs.krushmedia.com/4e4abdd5ecc661643458a730b1aa927d.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dkrushmedia%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewKrushmediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "allGdpr", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://cs.krushmedia.com/4e4abdd5ecc661643458a730b1aa927d.gif?gdpr=0&gdpr_consent=allGdpr&us_privacy=1-YY&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dkrushmedia%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/lifestreet/usersync.go b/adapters/lifestreet/usersync.go deleted file mode 100644 index f5300ebaa90..00000000000 --- a/adapters/lifestreet/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package lifestreet - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewLifestreetSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("lifestreet", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/lifestreet/usersync_test.go b/adapters/lifestreet/usersync_test.go deleted file mode 100644 index e41217fe10f..00000000000 --- a/adapters/lifestreet/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package lifestreet - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestLifestreetSyncer(t *testing.T) { - syncURL := "//ads.lfstmedia.com/idsync/137062?synced=1&ttl=1s&rurl=localhost%2Fsetuid%3Fbidder%3Dlifestreet%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%24visitor_cookie%24%24" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewLifestreetSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//ads.lfstmedia.com/idsync/137062?synced=1&ttl=1s&rurl=localhost%2Fsetuid%3Fbidder%3Dlifestreet%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24%24visitor_cookie%24%24", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/lockerdome/usersync.go b/adapters/lockerdome/usersync.go deleted file mode 100644 index d5cce16804a..00000000000 --- a/adapters/lockerdome/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package lockerdome - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewLockerDomeSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("lockerdome", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/lockerdome/usersync_test.go b/adapters/lockerdome/usersync_test.go deleted file mode 100644 index 3a2bd7f325b..00000000000 --- a/adapters/lockerdome/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package lockerdome - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestLockerDomeSyncer(t *testing.T) { - syncURL := "https://lockerdome.com/usync/prebidserver?pid=&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dlockerdome%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7Buid%7D%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewLockerDomeSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://lockerdome.com/usync/prebidserver?pid=&gdpr=&gdpr_consent=&us_privacy=&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dlockerdome%26gdpr%3D%26gdpr_consent%3D%26uid%3D%7B%7Buid%7D%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/logicad/usersync.go b/adapters/logicad/usersync.go deleted file mode 100644 index e685cc985fc..00000000000 --- a/adapters/logicad/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package logicad - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewLogicadSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("logicad", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/logicad/usersync_test.go b/adapters/logicad/usersync_test.go deleted file mode 100644 index e8b10c665fe..00000000000 --- a/adapters/logicad/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package logicad - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestLogicadSyncer(t *testing.T) { - syncURL := "https://localhost/cookiesender?r=true&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ru=localhost%2Fsetuid%3Fbidder%3Dlogicad%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewLogicadSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "A", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://localhost/cookiesender?r=true&gdpr=1&gdpr_consent=A&ru=localhost%2Fsetuid%3Fbidder%3Dlogicad%26gdpr%3D1%26gdpr_consent%3DA%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/lunamedia/lunamedia.go b/adapters/lunamedia/lunamedia.go index 289d062c8bb..3916249206f 100644 --- a/adapters/lunamedia/lunamedia.go +++ b/adapters/lunamedia/lunamedia.go @@ -15,7 +15,7 @@ import ( ) type LunaMediaAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } //MakeRequests prepares request information for prebid-server core @@ -235,7 +235,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &LunaMediaAdapter{ - EndpointTemplate: *urlTemplate, + EndpointTemplate: urlTemplate, } return bidder, nil } diff --git a/adapters/lunamedia/usersync.go b/adapters/lunamedia/usersync.go deleted file mode 100644 index 39c9a808040..00000000000 --- a/adapters/lunamedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package lunamedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewLunaMediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("lunamedia", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/lunamedia/usersync_test.go b/adapters/lunamedia/usersync_test.go deleted file mode 100644 index 24cd740d600..00000000000 --- a/adapters/lunamedia/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package lunamedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestLunaMediaSyncer(t *testing.T) { - syncURL := "https://api.lunamedia.io/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=lunamedia&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&uid=$UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewLunaMediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "A", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://api.lunamedia.io/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=lunamedia&gdpr=1&gdpr_consent=A&uid=$UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/marsmedia/usersync.go b/adapters/marsmedia/usersync.go deleted file mode 100644 index 4ac76d1f5f2..00000000000 --- a/adapters/marsmedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package marsmedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewMarsmediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("marsmedia", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/marsmedia/usersync_test.go b/adapters/marsmedia/usersync_test.go deleted file mode 100644 index 975af65fcf5..00000000000 --- a/adapters/marsmedia/usersync_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package marsmedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestMarsmediaSyncer(t *testing.T) { - syncURL := "https://dmp.rtbsrv.com/dmp/profiles/cm?p_id=179&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=localhost:8000%2Fsetuid%3Fbidder%3Dmarsmedia%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewMarsmediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://dmp.rtbsrv.com/dmp/profiles/cm?p_id=179&gdpr=A&gdpr_consent=B&us_privacy=C&redirect=localhost:8000%2Fsetuid%3Fbidder%3Dmarsmedia%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%24%7BUUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - -} diff --git a/adapters/mediafuse/usersync.go b/adapters/mediafuse/usersync.go deleted file mode 100644 index b91a6b5052c..00000000000 --- a/adapters/mediafuse/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package mediafuse - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewMediafuseSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("mediafuse", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/mediafuse/usersync_test.go b/adapters/mediafuse/usersync_test.go deleted file mode 100644 index e3dfa06831d..00000000000 --- a/adapters/mediafuse/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package mediafuse - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestMediafuseSyncer(t *testing.T) { - syncURL := "//sync.hbmp.mediafuse.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dmediafuse%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewMediafuseSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//sync.hbmp.mediafuse.com/csync?t=p&ep=0&redir=localhost%2Fsetuid%3Fbidder%3Dmediafuse%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%7Buid%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/mgid/usersync.go b/adapters/mgid/usersync.go deleted file mode 100644 index 94cf12e119d..00000000000 --- a/adapters/mgid/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package mgid - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewMgidSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("mgid", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/mgid/usersync_test.go b/adapters/mgid/usersync_test.go deleted file mode 100644 index 3fa5a151bd8..00000000000 --- a/adapters/mgid/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package mgid - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestMgidSyncer(t *testing.T) { - syncURL := "https://cm.mgid.com/m?cdsp=363893&adu=https%3A//external.com%2Fsetuid%3Fbidder%3Dmgid%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Bmuidn%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewMgidSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://cm.mgid.com/m?cdsp=363893&adu=https%3A//external.com%2Fsetuid%3Fbidder%3Dmgid%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%7Bmuidn%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/mobilefuse/mobilefuse.go b/adapters/mobilefuse/mobilefuse.go index 76c18e27007..d6458be1858 100644 --- a/adapters/mobilefuse/mobilefuse.go +++ b/adapters/mobilefuse/mobilefuse.go @@ -16,7 +16,7 @@ import ( ) type MobileFuseAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // Builder builds a new instance of the MobileFuse adapter for the given bidder with the given config. @@ -27,7 +27,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &MobileFuseAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/nanointeractive/usersync.go b/adapters/nanointeractive/usersync.go deleted file mode 100644 index 6bd9cd1f036..00000000000 --- a/adapters/nanointeractive/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package nanointeractive - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewNanoInteractiveSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("nanointeractive", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/nanointeractive/usersync_test.go b/adapters/nanointeractive/usersync_test.go deleted file mode 100644 index 4d816ab7384..00000000000 --- a/adapters/nanointeractive/usersync_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package nanointeractive - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestNewNanoInteractiveSyncer(t *testing.T) { - syncURL := "https://ad.audiencemanager.de/hbs/cookie_sync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri=http%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dnanointeractive%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - userSync := NewNanoInteractiveSyncer(syncURLTemplate) - syncInfo, err := userSync.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ad.audiencemanager.de/hbs/cookie_sync?gdpr=1&consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&redirectUri=http%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dnanointeractive%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/ninthdecimal/ninthdecimal.go b/adapters/ninthdecimal/ninthdecimal.go index 03095ee6c44..e12329aeb8c 100755 --- a/adapters/ninthdecimal/ninthdecimal.go +++ b/adapters/ninthdecimal/ninthdecimal.go @@ -15,7 +15,7 @@ import ( ) type NinthDecimalAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } //MakeRequests prepares request information for prebid-server core @@ -235,7 +235,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &NinthDecimalAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/ninthdecimal/usersync.go b/adapters/ninthdecimal/usersync.go deleted file mode 100755 index a01fdb636e3..00000000000 --- a/adapters/ninthdecimal/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package ninthdecimal - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewNinthDecimalSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("ninthdecimal", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/ninthdecimal/usersync_test.go b/adapters/ninthdecimal/usersync_test.go deleted file mode 100755 index e722a2b6e69..00000000000 --- a/adapters/ninthdecimal/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package ninthdecimal - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestNinthDecimalSyncer(t *testing.T) { - syncURL := "https://rtb.ninthdecimal.com/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=ninthdecimal&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&uid=$UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewNinthDecimalSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "A", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://rtb.ninthdecimal.com/xp/user-sync?acctid={aid}&&redirect=localhost/setuid?bidder=ninthdecimal&gdpr=1&gdpr_consent=A&uid=$UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/nobid/usersync.go b/adapters/nobid/usersync.go deleted file mode 100644 index 442075648ce..00000000000 --- a/adapters/nobid/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package nobid - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewNoBidSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("nobid", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/nobid/usersync_test.go b/adapters/nobid/usersync_test.go deleted file mode 100644 index bc55d130509..00000000000 --- a/adapters/nobid/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package nobid - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestNoBidSyncer(t *testing.T) { - syncURL := "https://ads.servenobid.com/sync?tek=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=localhost%2Fsetuid%3Fbidder%3Dtest_bidder%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewNoBidSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://ads.servenobid.com/sync?tek=pbs&gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&redirect=localhost%2Fsetuid%3Fbidder%3Dtest_bidder%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/onetag/onetag.go b/adapters/onetag/onetag.go index ebf428439ed..ff5d49e5b51 100644 --- a/adapters/onetag/onetag.go +++ b/adapters/onetag/onetag.go @@ -15,7 +15,7 @@ import ( ) type adapter struct { - endpointTemplate template.Template + endpointTemplate *template.Template } func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters.Bidder, error) { @@ -25,7 +25,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &adapter{ - endpointTemplate: *template, + endpointTemplate: template, } return bidder, nil diff --git a/adapters/onetag/usersync.go b/adapters/onetag/usersync.go deleted file mode 100644 index 9a2b700dd3d..00000000000 --- a/adapters/onetag/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package onetag - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSyncer(template *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("onetag", template, adapters.SyncTypeIframe) -} diff --git a/adapters/onetag/usersync_test.go b/adapters/onetag/usersync_test.go deleted file mode 100644 index 21f4837d5e1..00000000000 --- a/adapters/onetag/usersync_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package onetag - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestOneTagSyncer(t *testing.T) { - syncURL := "https://onetag-sys.com/usync/?redir=" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://onetag-sys.com/usync/?redir=", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) -} diff --git a/adapters/openx/usersync.go b/adapters/openx/usersync.go deleted file mode 100644 index 875b60fbd10..00000000000 --- a/adapters/openx/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package openx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewOpenxSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("openx", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/openx/usersync_test.go b/adapters/openx/usersync_test.go deleted file mode 100644 index 14ec38be118..00000000000 --- a/adapters/openx/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package openx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestOpenxSyncer(t *testing.T) { - syncURL := "https://rtb.openx.net/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r=localhost%2Fsetuid%3Fbidder%3Dopenx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewOpenxSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "https://rtb.openx.net/sync/prebid?gdpr=&gdpr_consent=&r=localhost%2Fsetuid%3Fbidder%3Dopenx%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/pubmatic/usersync.go b/adapters/pubmatic/usersync.go deleted file mode 100644 index 822f13cea3d..00000000000 --- a/adapters/pubmatic/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package pubmatic - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewPubmaticSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("pubmatic", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/pubmatic/usersync_test.go b/adapters/pubmatic/usersync_test.go deleted file mode 100644 index 0f4d2e857d3..00000000000 --- a/adapters/pubmatic/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package pubmatic - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestPubmaticSyncer(t *testing.T) { - syncURL := "//ads.pubmatic.com/AdServer/js/user_sync.html?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&predirect=localhost%2Fsetuid%3Fbidder%3Dpubmatic%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewPubmaticSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//ads.pubmatic.com/AdServer/js/user_sync.html?gdpr=A&gdpr_consent=B&us_privacy=C&predirect=localhost%2Fsetuid%3Fbidder%3Dpubmatic%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/pulsepoint/usersync.go b/adapters/pulsepoint/usersync.go deleted file mode 100644 index 1b6903f9f02..00000000000 --- a/adapters/pulsepoint/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package pulsepoint - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewPulsepointSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("pulsepoint", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/pulsepoint/usersync_test.go b/adapters/pulsepoint/usersync_test.go deleted file mode 100644 index 7cfea57cb91..00000000000 --- a/adapters/pulsepoint/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package pulsepoint - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestPulsepointSyncer(t *testing.T) { - syncURL := "//bh.contextweb.com/rtset?pid=561205&ev=1&rurl=http%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dpulsepoint%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25VGUID%25%25" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewPulsepointSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//bh.contextweb.com/rtset?pid=561205&ev=1&rurl=http%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dpulsepoint%26gdpr%3D%26gdpr_consent%3D%26uid%3D%25%25VGUID%25%25", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/rhythmone/usersync.go b/adapters/rhythmone/usersync.go deleted file mode 100644 index 990fcb065f6..00000000000 --- a/adapters/rhythmone/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package rhythmone - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewRhythmoneSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("rhythmone", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/rhythmone/usersync_test.go b/adapters/rhythmone/usersync_test.go deleted file mode 100644 index 97920fb4980..00000000000 --- a/adapters/rhythmone/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package rhythmone - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestRhythmoneSyncer(t *testing.T) { - syncURL := "https://sync.1rx.io/usersync2/rmphb?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=localhost%2Fsetuid%3Fbidder%3Drhythmone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BRX_UUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewRhythmoneSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.1rx.io/usersync2/rmphb?gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&us_privacy=1NYN&redir=localhost%2Fsetuid%3Fbidder%3Drhythmone%26gdpr%3D1%26gdpr_consent%3DBOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA%26uid%3D%5BRX_UUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/rtbhouse/usersync.go b/adapters/rtbhouse/usersync.go deleted file mode 100644 index baa0b994373..00000000000 --- a/adapters/rtbhouse/usersync.go +++ /dev/null @@ -1,18 +0,0 @@ -package rtbhouse - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -const rtbHouseFamilyName = "rtbhouse" - -func NewRTBHouseSyncer(urlTemplate *template.Template) usersync.Usersyncer { - return adapters.NewSyncer( - rtbHouseFamilyName, - urlTemplate, - adapters.SyncTypeRedirect, - ) -} diff --git a/adapters/rtbhouse/usersync_test.go b/adapters/rtbhouse/usersync_test.go deleted file mode 100644 index a7cb2590f90..00000000000 --- a/adapters/rtbhouse/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package rtbhouse - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestRTBHouseSyncer(t *testing.T) { - syncURL := "https://creativecdn.com/cm-notify?pi=prebidsrvtst&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewRTBHouseSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://creativecdn.com/cm-notify?pi=prebidsrvtst&gdpr=0&gdpr_consent=", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/rubicon/usersync.go b/adapters/rubicon/usersync.go deleted file mode 100644 index a4ab464a73f..00000000000 --- a/adapters/rubicon/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package rubicon - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewRubiconSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("rubicon", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/rubicon/usersync_test.go b/adapters/rubicon/usersync_test.go deleted file mode 100644 index eca4056206e..00000000000 --- a/adapters/rubicon/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package rubicon - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestRubiconSyncer(t *testing.T) { - syncURL := "https://pixel.rubiconproject.com/exchange/sync.php?p=prebid&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewRubiconSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://pixel.rubiconproject.com/exchange/sync.php?p=prebid&gdpr=0&gdpr_consent=", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - assert.Equal(t, "rubicon", syncer.FamilyName()) -} diff --git a/adapters/sharethrough/usersync.go b/adapters/sharethrough/usersync.go deleted file mode 100644 index f76f41ca83e..00000000000 --- a/adapters/sharethrough/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package sharethrough - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSharethroughSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("sharethrough", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/sharethrough/usersync_test.go b/adapters/sharethrough/usersync_test.go deleted file mode 100644 index 00b3c427fb8..00000000000 --- a/adapters/sharethrough/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package sharethrough - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSharethroughSyncer(t *testing.T) { - syncURL := "https://match.sharethrough.com?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSharethroughSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://match.sharethrough.com?gdpr=0&gdpr_consent=", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - assert.Equal(t, "sharethrough", syncer.FamilyName()) -} diff --git a/adapters/silvermob/silvermob.go b/adapters/silvermob/silvermob.go index 2e31e51c0ad..1ffe1958996 100644 --- a/adapters/silvermob/silvermob.go +++ b/adapters/silvermob/silvermob.go @@ -15,7 +15,7 @@ import ( ) type SilverMobAdapter struct { - endpoint template.Template + endpoint *template.Template } // Builder builds a new instance of the SilverMob adapter for the given bidder with the given config. @@ -26,7 +26,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &SilverMobAdapter{ - endpoint: *template, + endpoint: template, } return bidder, nil } diff --git a/adapters/smartadserver/usersync.go b/adapters/smartadserver/usersync.go deleted file mode 100644 index f7199965441..00000000000 --- a/adapters/smartadserver/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package smartadserver - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSmartadserverSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("smartadserver", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/smartadserver/usersync_test.go b/adapters/smartadserver/usersync_test.go deleted file mode 100644 index 319ce5b58a6..00000000000 --- a/adapters/smartadserver/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package smartadserver - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSmartadserverSyncer(t *testing.T) { - syncURL := "//ssbsync.smartadserver.com/getuid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=localhost%2Fsetuid%3Fbidder%3Dsmartadserver%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bsas_uid%5D%22" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSmartadserverSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "COyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA", - }, - CCPA: ccpa.Policy{ - Consent: "1YNN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//ssbsync.smartadserver.com/getuid?gdpr=1&gdpr_consent=COyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA&us_privacy=1YNN&url=localhost%2Fsetuid%3Fbidder%3Dsmartadserver%26gdpr%3D1%26gdpr_consent%3DCOyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA%26uid%3D%5Bsas_uid%5D%22", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/smartrtb/smartrtb.go b/adapters/smartrtb/smartrtb.go index 4950f1cefb3..259e254245d 100644 --- a/adapters/smartrtb/smartrtb.go +++ b/adapters/smartrtb/smartrtb.go @@ -16,7 +16,7 @@ import ( // Base adapter structure. type SmartRTBAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // Bid request extension appended to downstream request. @@ -49,7 +49,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &SmartRTBAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/smartrtb/usersync.go b/adapters/smartrtb/usersync.go deleted file mode 100644 index 74ef0e9960b..00000000000 --- a/adapters/smartrtb/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package smartrtb - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSmartRTBSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("smartrtb", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/smartrtb/usersync_test.go b/adapters/smartrtb/usersync_test.go deleted file mode 100644 index 68a8452a316..00000000000 --- a/adapters/smartrtb/usersync_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package smartrtb - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestSmartRTBSyncer(t *testing.T) { - temp := template.Must(template.New("sync-template").Parse("http://market-east.smrtb.com/sync/all?nid=smartrtb&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&url=localhost%2Fsetuid%3Fbidder%smartrtb%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D")) - syncer := NewSmartRTBSyncer(temp) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - assert.NoError(t, err) - assert.Equal(t, "http://market-east.smrtb.com/sync/all?nid=smartrtb&gdpr=&gdpr_consent=&url=localhost%2Fsetuid%3Fbidder%smartrtb%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/smartyads/smartyads.go b/adapters/smartyads/smartyads.go index ba727b4730c..caabb312cd0 100644 --- a/adapters/smartyads/smartyads.go +++ b/adapters/smartyads/smartyads.go @@ -16,7 +16,7 @@ import ( ) type SmartyAdsAdapter struct { - endpoint template.Template + endpoint *template.Template } // Builder builds a new instance of the SmartyAds adapter for the given bidder with the given config. @@ -27,7 +27,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &SmartyAdsAdapter{ - endpoint: *template, + endpoint: template, } return bidder, nil } diff --git a/adapters/smartyads/usersync.go b/adapters/smartyads/usersync.go deleted file mode 100644 index 9075aa9bcd7..00000000000 --- a/adapters/smartyads/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package smartyads - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSmartyAdsSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("smartyads", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/smartyads/usersync_test.go b/adapters/smartyads/usersync_test.go deleted file mode 100644 index 4f94591c634..00000000000 --- a/adapters/smartyads/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package smartyads - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSmartyAdsSyncer(t *testing.T) { - syncURL := "https://as.ck-ie.com/prebid.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dsmartyads%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewSmartyAdsSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "ANDFJDS", - }, - CCPA: ccpa.Policy{ - Consent: "1-YY", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://as.ck-ie.com/prebid.gif?gdpr=0&gdpr_consent=ANDFJDS&us_privacy=1-YY&redir=http%3A%2F%2Flocalhost%3A8000%2Fsetuid%3Fbidder%3Dsmartyads%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/somoaudience/usersync.go b/adapters/somoaudience/usersync.go deleted file mode 100644 index 5d1ddd71bc6..00000000000 --- a/adapters/somoaudience/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package somoaudience - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSomoaudienceSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("somoaudience", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/somoaudience/usersync_test.go b/adapters/somoaudience/usersync_test.go deleted file mode 100644 index 2367a5674dd..00000000000 --- a/adapters/somoaudience/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package somoaudience - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestSomoaudienceSyncer(t *testing.T) { - syncURL := "//publisher-east.mobileadtrading.com/usersync?ru=localhost%2Fsetuid%3Fbidder%3Dsomoaudience%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSomoaudienceSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//publisher-east.mobileadtrading.com/usersync?ru=localhost%2Fsetuid%3Fbidder%3Dsomoaudience%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24%7BUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/sonobi/usersync.go b/adapters/sonobi/usersync.go deleted file mode 100644 index 6fedd8bfa05..00000000000 --- a/adapters/sonobi/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package sonobi - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSonobiSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("sonobi", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/sonobi/usersync_test.go b/adapters/sonobi/usersync_test.go deleted file mode 100644 index 995c3757ba4..00000000000 --- a/adapters/sonobi/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package sonobi - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSonobiSyncer(t *testing.T) { - syncURL := "//sync.go.sonobi.com/us.gif?loc=external.com%2Fsetuid%3Fbidder%3Dsonobi%26consent_string%3D{{.GDPR}}%26gdpr%3D{{.GDPRConsent}}%26uid%3D%5BUID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSonobiSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//sync.go.sonobi.com/us.gif?loc=external.com%2Fsetuid%3Fbidder%3Dsonobi%26consent_string%3D0%26gdpr%3D%26uid%3D%5BUID%5D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/sovrn/usersync.go b/adapters/sovrn/usersync.go deleted file mode 100644 index 225f2888196..00000000000 --- a/adapters/sovrn/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package sovrn - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSovrnSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("sovrn", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/sovrn/usersync_test.go b/adapters/sovrn/usersync_test.go deleted file mode 100644 index 6c35ecdb05d..00000000000 --- a/adapters/sovrn/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package sovrn - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSovrnSyncer(t *testing.T) { - syncURL := "//ap.lijit.com/pixel?redir=external.com%2Fsetuid%3Fbidder%3Dsovrn%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSovrnSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//ap.lijit.com/pixel?redir=external.com%2Fsetuid%3Fbidder%3Dsovrn%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/synacormedia/synacormedia.go b/adapters/synacormedia/synacormedia.go index 7c9b4fab19f..a4306a7c02f 100644 --- a/adapters/synacormedia/synacormedia.go +++ b/adapters/synacormedia/synacormedia.go @@ -15,7 +15,7 @@ import ( ) type SynacorMediaAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } type SyncEndpointTemplateParams struct { @@ -35,7 +35,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &SynacorMediaAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/synacormedia/usersync.go b/adapters/synacormedia/usersync.go deleted file mode 100644 index c7fa5c42aea..00000000000 --- a/adapters/synacormedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package synacormedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSynacorMediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("synacormedia", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/synacormedia/usersync_test.go b/adapters/synacormedia/usersync_test.go deleted file mode 100644 index ce45353c7e5..00000000000 --- a/adapters/synacormedia/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package synacormedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestSynacormediaSyncer(t *testing.T) { - syncURL := "//sync.technoratimedia.com/services?srv=cs&pid=70&cb=localhost%2Fsetuid%3Fbidder%3Dsynacormedia%26uid%3D%5BUSER_ID%5D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSynacorMediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//sync.technoratimedia.com/services?srv=cs&pid=70&cb=localhost%2Fsetuid%3Fbidder%3Dsynacormedia%26uid%3D%5BUSER_ID%5D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/syncer.go b/adapters/syncer.go deleted file mode 100644 index b9752b50ce1..00000000000 --- a/adapters/syncer.go +++ /dev/null @@ -1,51 +0,0 @@ -package adapters - -import ( - "text/template" - - "github.com/prebid/prebid-server/macros" - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/usersync" -) - -type Syncer struct { - familyName string - syncType SyncType - urlTemplate *template.Template -} - -func NewSyncer(familyName string, urlTemplate *template.Template, syncType SyncType) *Syncer { - return &Syncer{ - familyName: familyName, - urlTemplate: urlTemplate, - syncType: syncType, - } -} - -type SyncType string - -const ( - SyncTypeRedirect SyncType = "redirect" - SyncTypeIframe SyncType = "iframe" -) - -func (s *Syncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { - syncURL, err := macros.ResolveMacros(*s.urlTemplate, macros.UserSyncTemplateParams{ - GDPR: privacyPolicies.GDPR.Signal, - GDPRConsent: privacyPolicies.GDPR.Consent, - USPrivacy: privacyPolicies.CCPA.Consent, - }) - if err != nil { - return nil, err - } - - return &usersync.UsersyncInfo{ - URL: syncURL, - Type: string(s.syncType), - SupportCORS: false, - }, err -} - -func (s *Syncer) FamilyName() string { - return s.familyName -} diff --git a/adapters/syncer_test.go b/adapters/syncer_test.go deleted file mode 100644 index ca33a9a130d..00000000000 --- a/adapters/syncer_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package adapters - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestGetUsersyncInfo(t *testing.T) { - privacyPolicies := privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - } - - syncURL := "{{.GDPR}}{{.GDPRConsent}}{{.USPrivacy}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := Syncer{ - urlTemplate: syncURLTemplate, - } - - syncInfo, err := syncer.GetUsersyncInfo(privacyPolicies) - - assert.NoError(t, err) - assert.Equal(t, "ABC", syncInfo.URL) -} diff --git a/adapters/tappx/tappx.go b/adapters/tappx/tappx.go index de8cb49a471..907ecbc094c 100644 --- a/adapters/tappx/tappx.go +++ b/adapters/tappx/tappx.go @@ -21,7 +21,7 @@ const TAPPX_BIDDER_VERSION = "1.1" const TYPE_CNN = "prebid" type TappxAdapter struct { - endpointTemplate template.Template + endpointTemplate *template.Template } // Builder builds a new instance of the Tappx adapter for the given bidder with the given config. @@ -32,7 +32,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &TappxAdapter{ - endpointTemplate: *template, + endpointTemplate: template, } return bidder, nil } diff --git a/adapters/tappx/usersync.go b/adapters/tappx/usersync.go deleted file mode 100644 index 01477c35363..00000000000 --- a/adapters/tappx/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package tappx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewTappxSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("tappx", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/tappx/usersync_test.go b/adapters/tappx/usersync_test.go deleted file mode 100644 index 992a35de9a8..00000000000 --- a/adapters/tappx/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package tappx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestTappxSyncer(t *testing.T) { - syncURL := "//testing.ssp.tappx.com/cs/usersync.php?gdpr_optin={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&type=iframe&ruid=localhost%2Fsetuid%3Fbidder%3Dtappx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7BTPPXUID%7D%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewTappxSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "A", - }, - CCPA: ccpa.Policy{ - Consent: "1YNN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//testing.ssp.tappx.com/cs/usersync.php?gdpr_optin=1&gdpr_consent=A&us_privacy=1YNN&type=iframe&ruid=localhost%2Fsetuid%3Fbidder%3Dtappx%26gdpr%3D1%26gdpr_consent%3DA%26uid%3D%7B%7BTPPXUID%7D%7D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/telaria/usersync.go b/adapters/telaria/usersync.go deleted file mode 100644 index 76be62026f8..00000000000 --- a/adapters/telaria/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package telaria - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewTelariaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("telaria", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/telaria/usersync_test.go b/adapters/telaria/usersync_test.go deleted file mode 100644 index f019ed5ceaf..00000000000 --- a/adapters/telaria/usersync_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package telaria - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestTelariaSyncer(t *testing.T) { - - syncURL := "https://pbs.publishers.tremorhub.com/pubsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewTelariaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://pbs.publishers.tremorhub.com/pubsync?gdpr=0&gdpr_consent=", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - assert.Equal(t, "telaria", syncer.FamilyName()) - -} diff --git a/adapters/triplelift/usersync.go b/adapters/triplelift/usersync.go deleted file mode 100644 index 4a47615bd29..00000000000 --- a/adapters/triplelift/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package triplelift - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewTripleliftSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("triplelift", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/triplelift/usersync_test.go b/adapters/triplelift/usersync_test.go deleted file mode 100644 index 012e33a4d0a..00000000000 --- a/adapters/triplelift/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package triplelift - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestTripleliftSyncer(t *testing.T) { - syncURL := "//eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=https%3A%2F%2Feb2.3lift.com%2Fsetuid%3Fbidder%3Dtriplelift%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewTripleliftSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//eb2.3lift.com/getuid?gdpr=&cmp_cs=&us_privacy=&redir=https%3A%2F%2Feb2.3lift.com%2Fsetuid%3Fbidder%3Dtriplelift%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/triplelift_native/usersync.go b/adapters/triplelift_native/usersync.go deleted file mode 100644 index 2a07740a761..00000000000 --- a/adapters/triplelift_native/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package triplelift_native - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewTripleliftSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("triplelift", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/triplelift_native/usersync_test.go b/adapters/triplelift_native/usersync_test.go deleted file mode 100644 index 7fd878b7f92..00000000000 --- a/adapters/triplelift_native/usersync_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package triplelift_native - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestTripleliftSyncer(t *testing.T) { - syncURL := "//eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=https%3A%2F%2Feb2.3lift.com%2Fsetuid%3Fbidder%3Dtriplelift%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewTripleliftSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "//eb2.3lift.com/getuid?gdpr=&cmp_cs=&us_privacy=&redir=https%3A%2F%2Feb2.3lift.com%2Fsetuid%3Fbidder%3Dtriplelift%26gdpr%3D%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/trustx/usersync.go b/adapters/trustx/usersync.go deleted file mode 100644 index a738e8e9b79..00000000000 --- a/adapters/trustx/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package trustx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewTrustXSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("trustx", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/trustx/usersync_test.go b/adapters/trustx/usersync_test.go deleted file mode 100644 index ba371bc2061..00000000000 --- a/adapters/trustx/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package trustx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestTrustXSyncer(t *testing.T) { - syncURL := "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dgrid%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewTrustXSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dgrid%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/ucfunnel/usersync.go b/adapters/ucfunnel/usersync.go deleted file mode 100644 index 0ae9948dd77..00000000000 --- a/adapters/ucfunnel/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package ucfunnel - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewUcfunnelSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("ucfunnel", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/ucfunnel/usersync_test.go b/adapters/ucfunnel/usersync_test.go deleted file mode 100644 index 204de978150..00000000000 --- a/adapters/ucfunnel/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package ucfunnel - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestUcfunnelSyncer(t *testing.T) { - syncURL := "//sync.aralego.com/idsync?gdpr={{.GDPR}}&redirect=externalURL.com%2Fsetuid%3Fbidder%3Ducfunnel%26uid%3DSspCookieUserId" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewUcfunnelSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//sync.aralego.com/idsync?gdpr=0&redirect=externalURL.com%2Fsetuid%3Fbidder%3Ducfunnel%26uid%3DSspCookieUserId", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/unruly/usersync.go b/adapters/unruly/usersync.go deleted file mode 100644 index 5dc60c859bb..00000000000 --- a/adapters/unruly/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package unruly - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewUnrulySyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("unruly", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/unruly/usersync_test.go b/adapters/unruly/usersync_test.go deleted file mode 100644 index f0702cebd34..00000000000 --- a/adapters/unruly/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package unruly - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestUnrulySyncer(t *testing.T) { - syncURL := "https://usermatch.targeting.unrulymedia.com/pbsync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl=%2Fsetuid%3Fbidder%3Dunruly%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewUnrulySyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://usermatch.targeting.unrulymedia.com/pbsync?gdpr=A&consent=B&us_privacy=C&rurl=%2Fsetuid%3Fbidder%3Dunruly%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/valueimpression/usersync.go b/adapters/valueimpression/usersync.go deleted file mode 100644 index 08490a5ed3e..00000000000 --- a/adapters/valueimpression/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package valueimpression - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewValueImpressionSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("valueimpression", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/valueimpression/usersync_test.go b/adapters/valueimpression/usersync_test.go deleted file mode 100644 index 7b3a13c5dd6..00000000000 --- a/adapters/valueimpression/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package valueimpression - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestValueImpressionSyncer(t *testing.T) { - syncURL := "https://rtb.valueimpression.com/usersync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redirectUri=http%3A%2F%2Flocalhost:8000%2Fsetuid%3Fbidder%3Dvalueimpression%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewValueImpressionSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://rtb.valueimpression.com/usersync?gdpr=1&gdpr_consent=BOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA&redirectUri=http%3A%2F%2Flocalhost:8000%2Fsetuid%3Fbidder%3Dvalueimpression%26gdpr%3D1%26gdpr_consent%3DBOPVK28OVJoTBABABAENBs-AAAAhuAKAANAAoACwAGgAPAAxAB0AHgAQAAiABOADkA%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/verizonmedia/usersync.go b/adapters/verizonmedia/usersync.go deleted file mode 100644 index 2c7354d1146..00000000000 --- a/adapters/verizonmedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package verizonmedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewVerizonMediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("verizonmedia", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/verizonmedia/usersync_test.go b/adapters/verizonmedia/usersync_test.go deleted file mode 100644 index 9dd1ae70c25..00000000000 --- a/adapters/verizonmedia/usersync_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package verizonmedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/stretchr/testify/assert" -) - -func TestVerizonMediaSyncer(t *testing.T) { - syncURL := "https://pixel.advertising.com/ups/58207/occ?http://localhost/%2Fsetuid%3Fbidder%3Dverizonmedia%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewVerizonMediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{}) - - assert.NoError(t, err) - assert.Equal(t, "redirect", syncInfo.Type) -} diff --git a/adapters/visx/usersync.go b/adapters/visx/usersync.go deleted file mode 100644 index dc143570ab9..00000000000 --- a/adapters/visx/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package visx - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewVisxSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("visx", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/visx/usersync_test.go b/adapters/visx/usersync_test.go deleted file mode 100644 index ec4cf82b04b..00000000000 --- a/adapters/visx/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package visx - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestVisxSyncer(t *testing.T) { - syncURL := "https://t.visx.net/s2s_sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=%2Fsetuid%3Fbidder%3Dvisx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUUID%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewVisxSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "A", - Consent: "B", - }, - CCPA: ccpa.Policy{ - Consent: "C", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://t.visx.net/s2s_sync?gdpr=A&gdpr_consent=B&us_privacy=C&redir=%2Fsetuid%3Fbidder%3Dvisx%26gdpr%3DA%26gdpr_consent%3DB%26uid%3D%24%7BUUID%7D", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/vrtcal/usersync.go b/adapters/vrtcal/usersync.go deleted file mode 100644 index 8a6ddc0ca26..00000000000 --- a/adapters/vrtcal/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package vrtcal - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewVrtcalSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("vrtcal", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/vrtcal/usersync_test.go b/adapters/vrtcal/usersync_test.go deleted file mode 100644 index 7f8ca220a96..00000000000 --- a/adapters/vrtcal/usersync_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package vrtcal - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestVrtcalSyncer(t *testing.T) { - syncURL := "http://usync-prebid.vrtcal.com/s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewVrtcalSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "http://usync-prebid.vrtcal.com/s?gdpr=0&gdpr_consent=", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) - assert.Equal(t, "vrtcal", syncer.FamilyName()) -} diff --git a/adapters/yeahmobi/yeahmobi.go b/adapters/yeahmobi/yeahmobi.go index c62f5bc032f..73b955dc922 100644 --- a/adapters/yeahmobi/yeahmobi.go +++ b/adapters/yeahmobi/yeahmobi.go @@ -17,7 +17,7 @@ import ( ) type YeahmobiAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // Builder builds a new instance of the Yeahmobi adapter for the given bidder with the given config. @@ -28,7 +28,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &YeahmobiAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/adapters/yieldlab/usersync.go b/adapters/yieldlab/usersync.go deleted file mode 100644 index 90507e31161..00000000000 --- a/adapters/yieldlab/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package yieldlab - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewYieldlabSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("yieldlab", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/yieldlab/usersync_test.go b/adapters/yieldlab/usersync_test.go deleted file mode 100644 index eabd46f6dce..00000000000 --- a/adapters/yieldlab/usersync_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package yieldlab - -import ( - "testing" - "text/template" - - "github.com/stretchr/testify/assert" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" -) - -func TestYieldlabSyncer(t *testing.T) { - temp := template.Must(template.New("sync-template").Parse("https://ad.yieldlab.net/mr?t=2&pid=9140838&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dyieldlab%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25YL_UID%25%25")) - syncer := NewYieldlabSyncer(temp) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - assert.NoError(t, err) - assert.Equal(t, "https://ad.yieldlab.net/mr?t=2&pid=9140838&gdpr=0&gdpr_consent=&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dyieldlab%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%25%25YL_UID%25%25", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/yieldmo/usersync.go b/adapters/yieldmo/usersync.go deleted file mode 100644 index d06caa90c0b..00000000000 --- a/adapters/yieldmo/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package yieldmo - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewYieldmoSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("yieldmo", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/yieldmo/usersync_test.go b/adapters/yieldmo/usersync_test.go deleted file mode 100644 index 5d12c63e4aa..00000000000 --- a/adapters/yieldmo/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package yieldmo - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestYieldmoSyncer(t *testing.T) { - syncURL := "//ads.yieldmo.com/pbsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dyieldmo%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewYieldmoSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//ads.yieldmo.com/pbsync?gdpr=0&gdpr_consent=&us_privacy=&redirectUri=http%3A%2F%2Flocalhost%2F%2Fsetuid%3Fbidder%3Dyieldmo%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.False(t, syncInfo.SupportCORS) -} diff --git a/adapters/yieldone/usersync.go b/adapters/yieldone/usersync.go deleted file mode 100644 index 4d5d8283a68..00000000000 --- a/adapters/yieldone/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package yieldone - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewYieldoneSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("yieldone", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/yieldone/usersync_test.go b/adapters/yieldone/usersync_test.go deleted file mode 100644 index c4d0fee92dd..00000000000 --- a/adapters/yieldone/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package yieldone - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestYieldoneSyncer(t *testing.T) { - syncURL := "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dyieldone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewYieldoneSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//not_localhost/synclocalhost%2Fsetuid%3Fbidder%3Dyieldone%26gdpr%3D0%26gdpr_consent%3D%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/zeroclickfraud/usersync.go b/adapters/zeroclickfraud/usersync.go deleted file mode 100644 index 41c589818ca..00000000000 --- a/adapters/zeroclickfraud/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package zeroclickfraud - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewZeroClickFraudSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("zeroclickfraud", temp, adapters.SyncTypeIframe) -} diff --git a/adapters/zeroclickfraud/usersync_test.go b/adapters/zeroclickfraud/usersync_test.go deleted file mode 100644 index 5e8f8fdf111..00000000000 --- a/adapters/zeroclickfraud/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package zeroclickfraud - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestZeroClickFraudSyncer(t *testing.T) { - syncURL := "https://s.0cf.io/sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dzeroclickfraud%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7Buid%7D" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewZeroClickFraudSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - }, - CCPA: ccpa.Policy{ - Consent: "1NYN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://s.0cf.io/sync?gdpr=1&gdpr_consent=BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw&us_privacy=1NYN&r=https%3A%2F%2Flocalhost%3A8888%2Fsetuid%3Fbidder%3Dzeroclickfraud%26gdpr%3D1%26gdpr_consent%3DBONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw%26uid%3D%24%7Buid%7D", syncInfo.URL) - assert.Equal(t, "iframe", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/zeroclickfraud/zeroclickfraud.go b/adapters/zeroclickfraud/zeroclickfraud.go index 17fbba747df..fa240c165a3 100644 --- a/adapters/zeroclickfraud/zeroclickfraud.go +++ b/adapters/zeroclickfraud/zeroclickfraud.go @@ -16,7 +16,7 @@ import ( ) type ZeroClickFraudAdapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } func (a *ZeroClickFraudAdapter) MakeRequests(request *openrtb.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { @@ -185,7 +185,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters } bidder := &ZeroClickFraudAdapter{ - EndpointTemplate: *template, + EndpointTemplate: template, } return bidder, nil } diff --git a/analytics/core.go b/analytics/core.go index 6837304541d..3977e3a3554 100644 --- a/analytics/core.go +++ b/analytics/core.go @@ -6,7 +6,6 @@ import ( "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/usersync" ) /* @@ -71,7 +70,19 @@ type SetUIDObject struct { type CookieSyncObject struct { Status int Errors []error - BidderStatus []*usersync.CookieSyncBidders + BidderStatus []*CookieSyncBidder +} + +type CookieSyncBidder struct { + BidderCode string `json:"bidder"` + NoCookie bool `json:"no_cookie,omitempty"` + UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` +} + +type UsersyncInfo struct { + URL string `json:"url,omitempty"` + Type string `json:"type,omitempty"` + SupportCORS bool `json:"supportCORS,omitempty"` } // NotificationEvent is a loggable object diff --git a/analytics/filesystem/file_module_test.go b/analytics/filesystem/file_module_test.go index cfc923c70f6..2bd1a58b9b4 100644 --- a/analytics/filesystem/file_module_test.go +++ b/analytics/filesystem/file_module_test.go @@ -1,15 +1,15 @@ package filesystem import ( - "github.com/prebid/prebid-server/config" "net/http" "os" "strings" "testing" + "github.com/prebid/prebid-server/config" + "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/analytics" - "github.com/prebid/prebid-server/usersync" ) const TEST_DIR string = "testFiles" @@ -58,7 +58,7 @@ func TestSetUIDObject_ToJson(t *testing.T) { func TestCookieSyncObject_ToJson(t *testing.T) { cso := &analytics.CookieSyncObject{ Status: http.StatusOK, - BidderStatus: []*usersync.CookieSyncBidders{}, + BidderStatus: []*analytics.CookieSyncBidder{}, } if csoJson := jsonifyCookieSync(cso); strings.Contains(csoJson, "Transactional Logs Error") { t.Fatalf("CookieSyncObject failed to convert to json") diff --git a/analytics/pubstack/helpers/json_test.go b/analytics/pubstack/helpers/json_test.go index 4e36e8db2be..2db0941b0a6 100644 --- a/analytics/pubstack/helpers/json_test.go +++ b/analytics/pubstack/helpers/json_test.go @@ -1,11 +1,11 @@ package helpers import ( - "github.com/mxmCherry/openrtb" - "github.com/prebid/prebid-server/analytics" - "github.com/prebid/prebid-server/usersync" "net/http" "testing" + + "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/analytics" ) func TestJsonifyAuctionObject(t *testing.T) { @@ -30,7 +30,7 @@ func TestJsonifyVideoObject(t *testing.T) { func TestJsonifyCookieSync(t *testing.T) { cso := &analytics.CookieSyncObject{ Status: http.StatusOK, - BidderStatus: []*usersync.CookieSyncBidders{}, + BidderStatus: []*analytics.CookieSyncBidder{}, } if _, err := JsonifyCookieSync(cso, "scopeId"); err != nil { t.Fail() diff --git a/config/adapter.go b/config/adapter.go index ff262b186fd..845eafc0d49 100644 --- a/config/adapter.go +++ b/config/adapter.go @@ -9,24 +9,8 @@ import ( ) type Adapter struct { - Endpoint string `mapstructure:"endpoint"` // Required - // UserSyncURL is the URL returned by /cookie_sync for this Bidder. It is _usually_ optional. - // If not defined, sensible defaults will be derived based on the config.external_url. - // Note that some Bidders don't have sensible defaults, because their APIs require an ID that will vary - // from one PBS host to another. - // - // For these bidders, there will be a warning logged on startup that usersyncs will not work if you have not - // defined one in the app config. Check your app logs for more info. - // - // This value will be interpreted as a Golang Template. At runtime, the following Template variables will be replaced. - // - // {{.GDPR}} -- This will be replaced with the "gdpr" property sent to /cookie_sync - // {{.Consent}} -- This will be replaced with the "consent" property sent to /cookie_sync - // {{.USPrivacy}} -- This will be replaced with the "us_privacy" property sent to /cookie_sync - // - // For more info on templates, see: https://golang.org/pkg/text/template/ - UserSyncURL string `mapstructure:"usersync_url"` Disabled bool `mapstructure:"disabled"` + Endpoint string `mapstructure:"endpoint"` ExtraAdapterInfo string `mapstructure:"extra_info"` // needed for Rubicon @@ -47,24 +31,20 @@ type AdapterXAPI struct { func validateAdapters(adapterMap map[string]Adapter, errs []error) []error { for adapterName, adapter := range adapterMap { if !adapter.Disabled { - // Verify that every adapter has a valid endpoint associated with it errs = validateAdapterEndpoint(adapter.Endpoint, adapterName, errs) - - // Verify that valid user_sync URLs are specified in the config - errs = validateAdapterUserSyncURL(adapter.UserSyncURL, adapterName, errs) } } return errs } -const ( - dummyHost string = "dummyhost.com" - dummyPublisherID string = "12" - dummyAccountID string = "some_account" - dummyGDPR string = "0" - dummyGDPRConsent string = "someGDPRConsentString" - dummyCCPA string = "1NYN" -) +var testEndpointTemplateParams = macros.EndpointTemplateParams{ + Host: "anyHost", + PublisherID: "anyPublisherID", + AccountID: "anyAccountID", + ZoneID: "anyZoneID", + SourceId: "anySourceID", + AdUnit: "anyAdUnit", +} // validateAdapterEndpoint makes sure that an adapter has a valid endpoint // associated with it @@ -80,11 +60,7 @@ func validateAdapterEndpoint(endpoint string, adapterName string, errs []error) return append(errs, fmt.Errorf("Invalid endpoint template: %s for adapter: %s. %v", endpoint, adapterName, err)) } // Resolve macros (if any) in the endpoint URL - resolvedEndpoint, err := macros.ResolveMacros(*endpointTemplate, macros.EndpointTemplateParams{ - Host: dummyHost, - PublisherID: dummyPublisherID, - AccountID: dummyAccountID, - }) + resolvedEndpoint, err := macros.ResolveMacros(endpointTemplate, testEndpointTemplateParams) if err != nil { return append(errs, fmt.Errorf("Unable to resolve endpoint: %s for adapter: %s. %v", endpoint, adapterName, err)) } @@ -101,36 +77,3 @@ func validateAdapterEndpoint(endpoint string, adapterName string, errs []error) } return errs } - -// validateAdapterUserSyncURL validates an adapter's user sync URL if it is set -func validateAdapterUserSyncURL(userSyncURL string, adapterName string, errs []error) []error { - if userSyncURL != "" { - // Create user_sync URL template - userSyncTemplate, err := template.New("userSyncTemplate").Parse(userSyncURL) - if err != nil { - return append(errs, fmt.Errorf("Invalid user sync URL template: %s for adapter: %s. %v", userSyncURL, adapterName, err)) - } - // Resolve macros (if any) in the user_sync URL - dummyMacroValues := macros.UserSyncTemplateParams{ - GDPR: dummyGDPR, - GDPRConsent: dummyGDPRConsent, - USPrivacy: dummyCCPA, - } - resolvedUserSyncURL, err := macros.ResolveMacros(*userSyncTemplate, dummyMacroValues) - if err != nil { - return append(errs, fmt.Errorf("Unable to resolve user sync URL: %s for adapter: %s. %v", userSyncURL, adapterName, err)) - } - // Validate the resolved user sync URL - // - // Validating using both IsURL and IsRequestURL because IsURL allows relative paths - // whereas IsRequestURL requires absolute path but fails to check other valid URL - // format constraints. - // - // For example: IsURL will allow "abcd.com" but IsRequestURL won't - // IsRequestURL will allow "http://http://abcd.com" but IsURL won't - if !validator.IsURL(resolvedUserSyncURL) || !validator.IsRequestURL(resolvedUserSyncURL) { - errs = append(errs, fmt.Errorf("The user_sync URL: %s for %s is invalid", resolvedUserSyncURL, adapterName)) - } - } - return errs -} diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 9eff288f64f..3fb3ce86242 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -12,37 +12,113 @@ import ( // BidderInfos contains a mapping of bidder name to bidder info. type BidderInfos map[string]BidderInfo -// BidderInfo is the maintainer information, supported auction types, and feature opts-in for a bidder. +// BidderInfo specifies all configuration for a bidder except for enabled status, endpoint, and extra information. type BidderInfo struct { Enabled bool // copied from adapter config for convenience. to be refactored. Maintainer *MaintainerInfo `yaml:"maintainer"` Capabilities *CapabilitiesInfo `yaml:"capabilities"` ModifyingVastXmlAllowed bool `yaml:"modifyingVastXmlAllowed"` - Debug *DebugInfo `yaml:"debug,omitempty"` - GVLVendorID uint16 `yaml:"gvlVendorID,omitempty"` + Debug *DebugInfo `yaml:"debug"` + GVLVendorID uint16 `yaml:"gvlVendorID"` + Syncer *Syncer `yaml:"userSync"` } -// MaintainerInfo is the support email address for a bidder. +// MaintainerInfo specifies the support email address for a bidder. type MaintainerInfo struct { Email string `yaml:"email"` } -// CapabilitiesInfo is the supported platforms for a bidder. +// CapabilitiesInfo specifies the supported platforms for a bidder. type CapabilitiesInfo struct { App *PlatformInfo `yaml:"app"` Site *PlatformInfo `yaml:"site"` } -// PlatformInfo is the supported media types for a bidder. +// PlatformInfo specifies the supported media types for a bidder. type PlatformInfo struct { MediaTypes []openrtb_ext.BidType `yaml:"mediaTypes"` } -// DebugInfo is the supported debug options for a bidder. +// DebugInfo specifies the supported debug options for a bidder. type DebugInfo struct { Allow bool `yaml:"allow"` } +// Syncer specifies the user sync settings for a bidder. +type Syncer struct { + // Key is used as the record key for the user sync cookie. We recommend using the bidder name + // as the key for consistency, but that is not enforced as a requirement. Each bidder must + // have a unique key. + Key string `yaml:"key"` + + // Default identifies which endpoint is preferred if both are allowed by the publisher. This is + // only required if there is more than one endpoint configured for the bidder. Valid values are + // `iframe` and `redirect`. + Default string `yaml:"default"` + + // IFrame configures an iframe endpoint for user syncing. + IFrame *SyncerEndpoint `yaml:"iframe"` + + // Redirect configures an redirect endpoint for user syncing. This is also known as an image + // endpoint in the Prebid.js project. + Redirect *SyncerEndpoint `yaml:"redirect"` +} + +// SyncerEndpoint specifies the configuration of the URL returned by the /cookie_sync endpoint +// for a specific bidder. Bidders must specify at least one endpoint configuration to be eligible +// for selection during a user sync request. +// +// URL is the only required field, although we highly recommend to use the available macros to +// make the configuration readable and maintainable. User sync urls include a redirect url back to +// Prebid Server which is url escaped and can be very diffcult for humans to read. +// +// In most cases, bidders will specify a URL with a `{{.RedirectURL}}` macro for the call back to +// Prebid Server and a UserMacro which the bidder server will replace with the user's id. Example: +// +// url: "https://sync.bidderserver.com/usersync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" +// userMacro: "$UID" +// +// Prebid Server is configured with a default RedirectURL template matching the /setuid call. This +// may be overridden for all bidders with the `user_sync.redirect_url` host configuration or for a +// specific bidder with the RedirectURL value in this struct. +type SyncerEndpoint struct { + // URL is the endpoint on the bidder server the user will be redirected to when a user sync is + // requested. The following macros are resolved at application startup: + // + // {{.RedirectURL}} - This will be replaced with a redirect url generated using the RedirectURL + // template and url escaped for safe inclusion in any part of the URL. + // + // The following macros are specific to individual requests and are resolved at runtime using the + // Go template engine. For more information on Go templates, see: https://golang.org/pkg/text/template/ + // + // {{.GDPR}} - This will be replaced with the "gdpr" property sent to /cookie_sync. + // {{.Consent}} - This will be replaced with the "consent" property sent to /cookie_sync. + // {{.USPrivacy}} - This will be replaced with the "us_privacy" property sent to /cookie_sync. + URL string + + // RedirectURL is an endpoint on the host server the user will be redirected to when a user sync + // request has been completed by the bidder server. The following macros are resolved at application + // startup: + // + // {{.ExternalURL}} - This will be replaced with the host server's externally reachable http path. + // {{.SyncerKey}} - This will be replaced with the syncer key. + // {{.SyncType}} - This will be replaced with the sync type, either 'b' for iframe syncs or 'i' + // for redirect/image syncs. + // {{.UserMacro}} - This will be replaced with the bidder server's user id macro. + // + // The endpoint on the host server is usually Prebid Server's /setuid endpoint. The default value is: + // `{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&f={{.SyncType}}&uid={{.UserMacro}}` + RedirectURL string + + // ExternalURL is available as a macro to the RedirectURL template. If not specified, the host configuration + // value is used. + ExternalURL string + + // UserMacro is available as a macro to the RedirectURL template. This value is specific to the bidder server + // and has no default. + UserMacro string +} + // LoadBidderInfoFromDisk parses all static/bidder-info/{bidder}.yaml files from the file system. func LoadBidderInfoFromDisk(path string, adapterConfigs map[string]Adapter, bidders []string) (BidderInfos, error) { reader := infoReaderFromDisk{path} diff --git a/config/config.go b/config/config.go index 22ce8faf0b6..a62a1ab469d 100644 --- a/config/config.go +++ b/config/config.go @@ -31,7 +31,7 @@ type Configuration struct { CacheURL Cache `mapstructure:"cache"` ExtCacheURL ExternalCache `mapstructure:"external_cache"` RecaptchaSecret string `mapstructure:"recaptcha_secret"` - HostCookie HostCookie `mapstructure:"host_cookie"` + HostCookie HostCookie `mapstructure:"host_cookie"` // to do - move to usersync Metrics Metrics `mapstructure:"metrics"` DataCache DataCache `mapstructure:"datacache"` StoredRequests StoredRequests `mapstructure:"stored_requests"` @@ -40,6 +40,7 @@ type Configuration struct { VTrack VTrack `mapstructure:"vtrack"` Event Event `mapstructure:"event"` Accounts StoredRequests `mapstructure:"accounts"` + UserSync UserSync `mapstructure:"user_sync"` // Note that StoredVideo refers to stored video requests, and has nothing to do with caching video creatives. StoredVideo StoredRequests `mapstructure:"stored_video_req"` @@ -472,7 +473,6 @@ func New(v *viper.Viper) (*Configuration, error) { if err := v.Unmarshal(&c); err != nil { return nil, fmt.Errorf("viper failed to unmarshal app config: %v", err) } - c.setDerivedDefaults() if err := c.RequestValidation.Parse(); err != nil { return nil, err @@ -558,109 +558,6 @@ func (cfg *Configuration) GetCachedAssetURL(uuid string) string { return fmt.Sprintf("%s/cache?%s", cfg.CacheURL.GetBaseURL(), strings.Replace(cfg.CacheURL.Query, "%PBS_CACHE_UUID%", uuid, 1)) } -// Initialize any default config values which have sensible defaults, but those defaults depend on other config values. -// -// For example, the typical Bidder's usersync URL includes the PBS config.external_url, because it redirects to the `external_url/setuid` endpoint. -// -func (cfg *Configuration) setDerivedDefaults() { - externalURL := cfg.ExternalURL - setDefaultUsersync(cfg.Adapters, openrtb_ext.Bidder33Across, "https://ic.tynt.com/r/d?m=xch&rt=html&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&ru="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3D33across%26uid%3D33XUSERID33X&id=zzz000000000002zzz") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAcuityAds, "https://cs.admanmedia.com/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dacuityads%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdform, "https://cm.adform.net/cookie?redirect_url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadform%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - // openrtb_ext.BidderAdgeneration doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdkernel, "https://sync.adkernel.com/user-sync?t=image&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadkernel%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdkernelAdn, "https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3DadkernelAdn%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdpone, "https://usersync.adpone.com/csync?redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadpone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdtarget, "https://sync.console.adtarget.com.tr/csync?t=p&ep=0&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadtarget%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdtelligent, "https://sync.adtelligent.com/csync?t=p&ep=0&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadtelligent%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdmixer, "https://inv-nets.admixer.net/adxcm.aspx?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=1&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadmixer%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%24visitor_cookie%24%24") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdman, "https://sync.admanmedia.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadman%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BUID%5D") - // openrtb_ext.BidderAdOcean doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdvangelists, "https://nep.advangelists.com/xp/user-sync?acctid={aid}&&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadvangelists%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAdyoulike, "http://visitor.omnitagjs.com/visitor/bsync?uid=19340f4f097d16f41f34fc0274981ca4&name=PrebidServer&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadyoulike%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BBUYER_USERID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAJA, "https://ad.as.amanad.adtdp.com/v1/sync/ssp?ssp=4&gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Daja%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25s") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAMX, "https://prebid.a-mo.net/cchain/0?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&cb="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Damx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAppnexus, "https://ib.adnxs.com/getuid?"+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dadnxs%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderAvocet, "https://ads.avct.cloud/getuid?&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Davocet%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7BUUID%7D%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderBeachfront, "https://sync.bfmio.com/sync_s2s?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dbeachfront%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bio_cid%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderBeintoo, "https://ib.beintoo.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dbeintoo%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderBrightroll, "https://pr-bh.ybp.yahoo.com/sync/appnexusprebidserver/?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dbrightroll%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderColossus, "https://sync.colossusssp.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dcolossus%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderConnectAd, "https://cdn.connectad.io/connectmyusers.php?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dconnectad%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderConsumable, "https://e.serverbid.com/udb/9969/match?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dconsumable%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderConversant, "https://prebid-match.dotomi.com/match/bounce/current?version=1&networkId=72582&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dconversant%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderCpmstar, "https://server.cpmstar.com/usersync.aspx?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dcpmstar%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderDatablocks, "https://sync.v5prebid.datablocks.net/s2ssync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Ddatablocks%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7Buid%7D") - // openrtb_ext.BidderDecenterAds doesn't have a good default. - // openrtb_ext.BidderDMX doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderDeepintent, "https://match.deepintent.com/usersync/136?id=unk&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Ddeepintent%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderEmxDigital, "https://cs.emxdgt.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Demx_digital%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderEngageBDR, "https://match.bnmla.com/usersync/s2s_sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dengagebdr%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderEPlanning, "https://ads.us.e-planning.net/uspd/1/?du="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Deplanning%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - // openrtb_ext.BidderEpom doesn't have a good default. - // openrtb_ext.BidderFacebook doesn't have a good default. - // openrtb_ext.BidderGamma doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderGamoshi, "https://rtb.gamoshi.io/user_sync_prebid?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dgamoshi%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bgusr%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderGrid, "https://x.bidswitch.net/check_uuid/"+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dgrid%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BBSW_UUID%7D?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderGumGum, "https://rtb.gumgum.com/usync/prbds2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dgumgum%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderImprovedigital, "https://ad.360yield.com/server_match?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dimprovedigital%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BPUB_USER_ID%7D") - // openrtb_ext.BidderIx doesn't have a good default. - // openrtb_ext.BidderInvibes doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderJixie, "https://id.jixie.io/api/sync?pid=&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Djixie%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25JXUID%25%25") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderKrushmedia, "https://cs.krushmedia.com/4e4abdd5ecc661643458a730b1aa927d.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dkrushmedia%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderLifestreet, "https://ads.lfstmedia.com/idsync/137062?synced=1&ttl=1s&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dlifestreet%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%24visitor_cookie%24%24") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderLockerDome, "https://lockerdome.com/usync/prebidserver?pid="+cfg.Adapters["lockerdome"].PlatformID+"&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dlockerdome%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7Buid%7D%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderLogicad, "https://cr-p31.ladsp.jp/cookiesender/31?r=true&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ru="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dlogicad%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderLunaMedia, "https://api.lunamedia.io/xp/user-sync?redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dlunamedia%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderMarsmedia, "https://dmp.rtbsrv.com/dmp/profiles/cm?p_id=179&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dmarsmedia%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderMediafuse, "https://sync.hbmp.mediafuse.com/csync?t=p&ep=0&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dmediafuse%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Buid%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderMgid, "https://cm.mgid.com/m?cdsp=363893&adu="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dmgid%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7Bmuidn%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderNanoInteractive, "https://ad.audiencemanager.de/hbs/cookie_sync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dnanointeractive%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderNinthDecimal, "https://rtb.ninthdecimal.com/xp/user-sync?acctid={aid}&&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dninthdecimal%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderNoBid, "https://ads.servenobid.com/getsync?tek=pbs&ver=1&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dnobid%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderOpenx, "https://rtb.openx.net/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dopenx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderOneTag, "https://onetag-sys.com/usync/?redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Donetag%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUSER_TOKEN%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderPubmatic, "https://ads.pubmatic.com/AdServer/js/user_sync.html?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&predirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dpubmatic%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderPulsepoint, "https://bh.contextweb.com/rtset?pid=561205&ev=1&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dpulsepoint%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25VGUID%25%25") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderRhythmone, "https://sync.1rx.io/usersync2/rmphb?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Drhythmone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BRX_UUID%5D") - // openrtb_ext.BidderRTBHouse doesn't have a good default. - // openrtb_ext.BidderRubicon doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSharethrough, "https://match.sharethrough.com/FGMrCMMc/v1?redirectUri="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsharethrough%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSmartAdserver, "https://ssbsync-global.smartadserver.com/api/sync?callerId=5&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsmartadserver%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Bssb_sync_pid%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSmartRTB, "https://market-global.smrtb.com/sync/all?nid=smartrtb&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&rr="+url.QueryEscape(externalURL)+"%252Fsetuid%253Fbidder%253Dsmartrtb%2526gdpr%253D{{.GDPR}}%2526gdpr_consent%253D{{.GDPRConsent}}%2526uid%253D%257BXID%257D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSmartyAds, "https://as.ck-ie.com/prebid.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsmartyads%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSomoaudience, "https://publisher-east.mobileadtrading.com/usersync?ru="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsomoaudience%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSonobi, "https://sync.go.sonobi.com/us.gif?loc="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsonobi%26consent_string%3D{{.GDPR}}%26gdpr%3D{{.GDPRConsent}}%26uid%3D%5BUID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSovrn, "https://ap.lijit.com/pixel?redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsovrn%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderSynacormedia, "https://sync.technoratimedia.com/services?srv=cs&pid=70&cb="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dsynacormedia%26uid%3D%5BUSER_ID%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderTappx, "https://ssp.api.tappx.com/cs/usersync.php?gdpr_optin={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&type=iframe&ruid="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dtappx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7B%7BTPPXUID%7D%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderTelaria, "https://pbs.publishers.tremorhub.com/pubsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dtelaria%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5Btvid%5D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderTriplelift, "https://eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dtriplelift%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderTripleliftNative, "https://eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dtriplelift_native%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderTrustX, "https://x.bidswitch.net/check_uuid/"+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dtrustx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BBSW_UUID%7D?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderUcfunnel, "https://sync.aralego.com/idsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&usprivacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Ducfunnel%26uid%3DSspCookieUserId") - // openrtb_ext.BidderUnicorn doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderUnruly, "https://usermatch.targeting.unrulymedia.com/pbsync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dunruly%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderValueImpression, "https://rtb.valueimpression.com/usersync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dvalueimpression%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderVisx, "https://t.visx.net/s2s_sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dvisx%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUUID%7D") - // openrtb_ext.BidderVrtcal doesn't have a good default. - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderYieldlab, "https://ad.yieldlab.net/mr?t=2&pid=9140838&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dyieldlab%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25YL_UID%25%25") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderYieldmo, "https://ads.yieldmo.com/pbsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dyieldmo%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderYieldone, "https://y.one.impact-ad.jp/hbs_cs?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dyieldone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderZeroClickFraud, "https://s.0cf.io/sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dzeroclickfraud%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7Buid%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderBetween, "https://ads.betweendigital.com/match?bidder_id=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&callback_url="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dbetween%26gdpr%3D0%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24%7BUSER_ID%7D") -} - -func setDefaultUsersync(m map[string]Adapter, bidder openrtb_ext.BidderName, defaultValue string) { - lowercased := strings.ToLower(string(bidder)) - if m[lowercased].UserSyncURL == "" { - // Go doesnt let us edit the properties of a value inside a map directly. - editable := m[lowercased] - editable.UserSyncURL = defaultValue - m[lowercased] = editable - } -} - // Set the default config values for the viper object we are using. func SetupViper(v *viper.Viper, filename string) { if filename != "" { @@ -791,6 +688,8 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("accounts.filesystem.directorypath", "./stored_requests/data/by_id") v.SetDefault("accounts.in_memory_cache.type", "none") + v.SetDefault("user_sync.redirect_url", "{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&f={{.SyncType}}&uid={{.UserMacro}}") + for _, bidder := range openrtb_ext.CoreBidderNames() { setBidderDefaults(v, strings.ToLower(string(bidder))) } diff --git a/config/config_test.go b/config/config_test.go index 7ff5f0fafa1..8fae7f725b4 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -392,21 +392,16 @@ func TestFullConfig(t *testing.T) { cmpStrings(t, "adapters.appnexus.endpoint", cfg.Adapters[string(openrtb_ext.BidderAppnexus)].Endpoint, "http://ib.adnxs.com/some/endpoint") cmpStrings(t, "adapters.appnexus.extra_info", cfg.Adapters[string(openrtb_ext.BidderAppnexus)].ExtraAdapterInfo, "{\"native\":\"http://www.native.org/endpoint\",\"video\":\"http://www.video.org/endpoint\"}") cmpStrings(t, "adapters.audiencenetwork.endpoint", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAudienceNetwork))].Endpoint, "http://facebook.com/pbs") - cmpStrings(t, "adapters.audiencenetwork.usersync_url", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAudienceNetwork))].UserSyncURL, "http://facebook.com/ortb/prebid-s2s") cmpStrings(t, "adapters.audiencenetwork.platform_id", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAudienceNetwork))].PlatformID, "abcdefgh1234") cmpStrings(t, "adapters.audiencenetwork.app_secret", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAudienceNetwork))].AppSecret, "987abc") cmpStrings(t, "adapters.beachfront.endpoint", cfg.Adapters[string(openrtb_ext.BidderBeachfront)].Endpoint, "https://display.bfmio.com/prebid_display") cmpStrings(t, "adapters.beachfront.extra_info", cfg.Adapters[string(openrtb_ext.BidderBeachfront)].ExtraAdapterInfo, "{\"video_endpoint\":\"https://reachms.bfmio.com/bid.json?exchange_id\"}") cmpStrings(t, "adapters.ix.endpoint", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderIx))].Endpoint, "http://ixtest.com/api") cmpStrings(t, "adapters.rubicon.endpoint", cfg.Adapters[string(openrtb_ext.BidderRubicon)].Endpoint, "http://rubitest.com/api") - cmpStrings(t, "adapters.rubicon.usersync_url", cfg.Adapters[string(openrtb_ext.BidderRubicon)].UserSyncURL, "http://pixel.rubiconproject.com/sync.php?p=prebid") cmpStrings(t, "adapters.rubicon.xapi.username", cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Username, "rubiuser") cmpStrings(t, "adapters.rubicon.xapi.password", cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Password, "rubipw23") cmpStrings(t, "adapters.brightroll.endpoint", cfg.Adapters[string(openrtb_ext.BidderBrightroll)].Endpoint, "http://test-bid.ybp.yahoo.com/bid/appnexuspbs") - cmpStrings(t, "adapters.brightroll.usersync_url", cfg.Adapters[string(openrtb_ext.BidderBrightroll)].UserSyncURL, "http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=%s") - cmpStrings(t, "adapters.adkerneladn.usersync_url", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAdkernelAdn))].UserSyncURL, "https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r=") cmpStrings(t, "adapters.rhythmone.endpoint", cfg.Adapters[string(openrtb_ext.BidderRhythmone)].Endpoint, "http://tag.1rx.io/rmp") - cmpStrings(t, "adapters.rhythmone.usersync_url", cfg.Adapters[string(openrtb_ext.BidderRhythmone)].UserSyncURL, "https://sync.1rx.io/usersync2/rmphb?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=http%3A%2F%2Fprebid-server.prebid.org%2F%2Fsetuid%3Fbidder%3Drhythmone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BRX_UUID%5D") cmpBools(t, "account_required", cfg.AccountRequired, true) cmpBools(t, "auto_gen_source_tid", cfg.AutoGenSourceTID, false) cmpBools(t, "account_adapter_details", cfg.Metrics.Disabled.AccountAdapterDetails, true) diff --git a/config/usersync.go b/config/usersync.go index 7940c5eb885..27badbcb815 100644 --- a/config/usersync.go +++ b/config/usersync.go @@ -1,6 +1,14 @@ package config +// UserSync specifies the static global user sync configuration. +type UserSync struct { + Cooperative UserSyncCooperative `mapstructure:"coop_sync"` + ExternalURL string `mapstructure:"external_url"` + RedirectURL string `mapstructure:"redirect_url"` +} + +// UserSyncCooperative specifies the static global default cooperative cookie sync type UserSyncCooperative struct { - Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"` - PriorityGroups [][]string `mapstructure:"priorityGroups" json:"priorityGroups,omitempty"` + EnabledByDefault bool `mapstructure:"default"` + PriorityGroups [][]string `mapstructure:"priority_groups"` } diff --git a/endpoints/auction.go b/endpoints/auction.go index b17cf14bc87..6d47d2b9f51 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -27,6 +27,8 @@ import ( "github.com/prebid/prebid-server/usersync" ) +var allSyncTypes []usersync.SyncType = []usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect} + type bidResult struct { bidder *pbs.PBSBidder bidList pbs.PBSBidSlice @@ -58,14 +60,14 @@ func writeAuctionError(w http.ResponseWriter, s string, err error) { type auction struct { cfg *config.Configuration - syncers map[openrtb_ext.BidderName]usersync.Usersyncer + syncers map[string]usersync.Syncer gdprPerms gdpr.Permissions metricsEngine metrics.MetricsEngine dataCache cache.Cache exchanges map[string]adapters.Adapter } -func Auction(cfg *config.Configuration, syncers map[openrtb_ext.BidderName]usersync.Usersyncer, gdprPerms gdpr.Permissions, metricsEngine metrics.MetricsEngine, dataCache cache.Cache, exchanges map[string]adapters.Adapter) httprouter.Handle { +func Auction(cfg *config.Configuration, syncers map[string]usersync.Syncer, gdprPerms gdpr.Permissions, metricsEngine metrics.MetricsEngine, dataCache cache.Cache, exchanges map[string]adapters.Adapter) httprouter.Handle { a := &auction{ cfg: cfg, syncers: syncers, @@ -478,8 +480,8 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl if syncerCode == "districtm" { syncerCode = "appnexus" } - syncer := a.syncers[openrtb_ext.BidderName(syncerCode)] - uid, _, _ := req.Cookie.GetUID(syncer.FamilyName()) + syncer := a.syncers[syncerCode] + uid, _, _ := req.Cookie.GetUID(syncer.Key()) if uid == "" { bidder.NoCookie = true privacyPolicies := privacy.Policies{ @@ -489,9 +491,13 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl }, } if a.shouldUsersync(*ctx, openrtb_ext.BidderName(syncerCode), privacyPolicies.GDPR) { - syncInfo, err := syncer.GetUsersyncInfo(privacyPolicies) + sync, err := syncer.GetSync(allSyncTypes, privacyPolicies) if err == nil { - bidder.UsersyncInfo = syncInfo + bidder.UsersyncInfo = &pbs.UsersyncInfo{ + URL: sync.URL, + Type: sync.Type.String(), + SupportCORS: sync.SupportsCORS, + } } else { glog.Errorf("Failed to get usersync info for %s: %v", syncerCode, err) } diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go index ed9a526d760..fb2e58dc3aa 100644 --- a/endpoints/auction_test.go +++ b/endpoints/auction_test.go @@ -20,7 +20,7 @@ import ( "github.com/prebid/prebid-server/pbs" "github.com/prebid/prebid-server/prebid_cache_client" gdprPolicy "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/prebid/prebid-server/usersync/usersyncers" + "github.com/prebid/prebid-server/usersync" "github.com/spf13/viper" "github.com/stretchr/testify/assert" @@ -350,7 +350,7 @@ func TestCacheVideoOnly(t *testing.T) { if err != nil { t.Fatal(err.Error()) } - syncers := usersyncers.NewSyncerMap(cfg) + syncers := map[string]usersync.Syncer{} gdprPerms := gdpr.NewPermissions(context.Background(), config.GDPR{ HostVendorID: 0, }, nil, nil) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index a57f3ab80e8..ed55fd6da28 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -6,11 +6,8 @@ import ( "errors" "fmt" "io/ioutil" - "math/rand" "net/http" - "strconv" - "github.com/buger/jsonparser" "github.com/golang/glog" "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/analytics" @@ -24,278 +21,358 @@ import ( "github.com/prebid/prebid-server/usersync" ) -// todo -// - update to use chooser -// - handle metrics and analytics from chooser result -// - respond from chooser result +var ( + errCookieSyncOptOut = errors.New("User has opted out") + errCookieSyncBody = errors.New("Failed to read request body") + errCookieSyncGDPRConsentMissing = errors.New("gdpr_consent is required if gdpr=1") + errCookieSyncGDPRConsentMissingSignalAmbiguous = errors.New("gdpr_consent is required. gdpr is not specified and is assumed to be 1 by the server. set gdpr=0 to exempt this request") + errCookieSyncInvalidBiddersType = errors.New("invalid bidders type. must either be a string '*' or a string array of bidders") +) + +var emptyUserSyncRequest = usersync.Request{} +var emptyPrivacyPolicies = privacy.Policies{} +var emptySyncTypeFilter = usersync.SyncTypeFilter{} +var emptySyncBidderFilter = usersync.BidderFilter{} +var emptySyncersChosen = []usersync.SyncerChoice{} + +var cookieSyncBidderFilterAllowAll = usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude) func NewCookieSyncEndpoint( - syncers map[openrtb_ext.BidderName]usersync.Usersyncer, - cfg *config.Configuration, - syncPermissions gdpr.Permissions, + syncers map[string]usersync.Syncer, + config *config.Configuration, + gdprPermissions gdpr.Permissions, metrics metrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, - bidderMap map[string]openrtb_ext.BidderName) httprouter.Handle { + bidders map[string]openrtb_ext.BidderName) HTTPRouterHandler { - bidderLookup := make(map[string]struct{}) - for k := range bidderMap { - bidderLookup[k] = struct{}{} + bidderHashSet := make(map[string]struct{}, len(bidders)) + for _, bidder := range bidders { + bidderHashSet[string(bidder)] = struct{}{} } - deps := &cookieSyncDeps{ - syncers: syncers, - hostCookie: &cfg.HostCookie, - gDPR: &cfg.GDPR, - syncPermissions: syncPermissions, - metrics: metrics, - pbsAnalytics: pbsAnalytics, - enforceCCPA: cfg.CCPA.Enforce, - bidderLookup: bidderLookup, + return &cookieSyncEndpoint{ + chooser: usersync.NewChooser(syncers), + config: config.UserSync, + hostCookieConfig: &config.HostCookie, + privacyConfig: usersyncPrivacyConfig{ + gdprConfig: config.GDPR, + gdprPermissions: gdprPermissions, + ccpaEnforce: config.CCPA.Enforce, + bidderHashSet: bidderHashSet, + }, + metrics: metrics, + pbsAnalytics: pbsAnalytics, } - return deps.Endpoint } -type cookieSyncDeps struct { - syncers map[openrtb_ext.BidderName]usersync.Usersyncer - hostCookie *config.HostCookie - gDPR *config.GDPR - syncPermissions gdpr.Permissions - metrics metrics.MetricsEngine - pbsAnalytics analytics.PBSAnalyticsModule - enforceCCPA bool - bidderLookup map[string]struct{} +type cookieSyncEndpoint struct { + chooser usersync.Chooser + config config.UserSync + hostCookieConfig *config.HostCookie + privacyConfig usersyncPrivacyConfig + metrics metrics.MetricsEngine + pbsAnalytics analytics.PBSAnalyticsModule } -func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - //CookieSyncObject makes a log of requests and responses to /cookie_sync endpoint - co := analytics.CookieSyncObject{ - Status: http.StatusOK, - Errors: make([]error, 0), - BidderStatus: make([]*usersync.CookieSyncBidders, 0), +func (c *cookieSyncEndpoint) Handle(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + request, privacyPolicies, err := c.parseRequest(r) + if err != nil { + c.metrics.RecordCookieSync(metrics.CookieSyncBadRequest) + c.handleError(w, err, http.StatusBadRequest) + return } - defer deps.pbsAnalytics.LogCookieSyncObject(&co) - - deps.metrics.RecordCookieSync() - userSyncCookie := usersync.ParseCookieFromRequest(r, deps.hostCookie) - if !userSyncCookie.AllowSyncs() { - http.Error(w, "User has opted out", http.StatusUnauthorized) - co.Status = http.StatusUnauthorized - co.Errors = append(co.Errors, fmt.Errorf("user has opted out")) - return + cookie := usersync.ParseCookieFromRequest(r, c.hostCookieConfig) + + result := c.chooser.Choose(request, cookie) + switch result.Status { + case usersync.StatusBlockedByUserOptOut: + c.metrics.RecordCookieSync(metrics.CookieSyncOptOut) + c.handleError(w, errCookieSyncOptOut, http.StatusUnauthorized) + case usersync.StatusBlockedByGDPR: + c.metrics.RecordCookieSync(metrics.CookieSyncGDPRHostCookieBlocked) + c.handleResponse(w, request.SyncTypeFilter, cookie, privacyPolicies, emptySyncersChosen) + case usersync.StatusOK: + c.metrics.RecordCookieSync(metrics.CookieSyncOK) + c.writeBidderMetrics(result.BiddersEvaluated) + c.handleResponse(w, request.SyncTypeFilter, cookie, privacyPolicies, result.SyncersChosen) } +} +func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, privacy.Policies, error) { defer r.Body.Close() - bodyBytes, err := ioutil.ReadAll(r.Body) + body, err := ioutil.ReadAll(r.Body) if err != nil { - co.Status = http.StatusBadRequest - co.Errors = append(co.Errors, errors.New("Failed to read request body")) - http.Error(w, "Failed to read request body", http.StatusBadRequest) - return - } - biddersJSON, err := parseBidders(bodyBytes) - if err != nil { - co.Status = http.StatusBadRequest - co.Errors = append(co.Errors, errors.New("Failed to check request.bidders in request body. Was your JSON well-formed?")) - http.Error(w, "Failed to check request.bidders in request body. Was your JSON well-formed?", http.StatusBadRequest) - return + return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncBody } - parsedReq := &cookieSyncRequest{} - if err := parseRequest(parsedReq, bodyBytes, deps.gDPR.UsersyncIfAmbiguous); err != nil { - co.Status = http.StatusBadRequest - co.Errors = append(co.Errors, err) - http.Error(w, co.Errors[len(co.Errors)-1].Error(), co.Status) - return + request := cookieSyncRequest{} + if err := json.Unmarshal(body, &request); err != nil { + return emptyUserSyncRequest, emptyPrivacyPolicies, fmt.Errorf("JSON parsing failed: %s", err.Error()) } - if len(biddersJSON) == 0 { - parsedReq.Bidders = make([]string, 0, len(deps.syncers)) - for bidder := range deps.syncers { - parsedReq.Bidders = append(parsedReq.Bidders, string(bidder)) - } + gdprSignal, err := gdpr.SignalParse(request.GDPR) + if err != nil { + return emptyUserSyncRequest, emptyPrivacyPolicies, err } - parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie, false) + if request.GDPRConsent == "" { + if gdprSignal == gdpr.SignalYes { + return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncGDPRConsentMissing + } - adapterSyncs := make(map[openrtb_ext.BidderName]bool) - // assume all bidders will be privacy blocked - for _, b := range parsedReq.Bidders { - adapterSyncs[openrtb_ext.BidderName(b)] = true + if gdprSignal == gdpr.SignalAmbiguous && gdpr.SignalNormalize(gdprSignal, c.privacyConfig.gdprConfig) == gdpr.SignalYes { + return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncGDPRConsentMissingSignalAmbiguous + } } - privacyPolicy := privacy.Policies{ + privacyPolicies := privacy.Policies{ GDPR: gdprPrivacy.Policy{ - Signal: gdprToString(parsedReq.GDPR), - Consent: parsedReq.Consent, + Signal: request.GDPR, + Consent: request.GDPRConsent, }, CCPA: ccpa.Policy{ - Consent: parsedReq.USPrivacy, + Consent: request.USPrivacy, }, } - parsedReq.filterForGDPR(deps.syncPermissions) - - if deps.enforceCCPA { - parsedReq.filterForCCPA(deps.bidderLookup) + ccpaParsedPolicy := ccpa.ParsedPolicy{} + if request.USPrivacy != "" { + parsedPolicy, err := privacyPolicies.CCPA.Parse(c.privacyConfig.bidderHashSet) + if err != nil { + privacyPolicies.CCPA.Consent = "" + } + if c.privacyConfig.ccpaEnforce { + ccpaParsedPolicy = parsedPolicy + } } - // surviving bidders are not privacy blocked - for _, b := range parsedReq.Bidders { - adapterSyncs[openrtb_ext.BidderName(b)] = false + syncTypeFilter, err := parseTypeFilter(request.FilterSettings) + if err != nil { + return emptyUserSyncRequest, emptyPrivacyPolicies, err } - for b, g := range adapterSyncs { - deps.metrics.RecordAdapterCookieSync(b, g) // how to record now that 1:many mapping for keys? multiple bidders benefit from the same sync. + + rx := usersync.Request{ + Bidders: request.Bidders, + Cooperative: usersync.Cooperative{ + Enabled: (request.CooperativeSync != nil && *request.CooperativeSync) || (request.CooperativeSync == nil && c.config.Cooperative.EnabledByDefault), + PriorityGroups: c.config.Cooperative.PriorityGroups, + }, + Limit: request.Limit, + Privacy: usersyncPrivacy{ + gdprPermissions: c.privacyConfig.gdprPermissions, + gdprSignal: gdprSignal, + gdprConsent: request.GDPRConsent, + ccpaParsedPolicy: ccpaParsedPolicy, + }, + SyncTypeFilter: syncTypeFilter, } - parsedReq.filterToLimit() + return rx, privacyPolicies, nil +} - csResp := cookieSyncResponse{ - Status: cookieSyncStatus(userSyncCookie.HasAnyLiveSyncs()), - BidderStatus: make([]*usersync.CookieSyncBidders, 0, len(parsedReq.Bidders)), +func parseTypeFilter(request *cookieSyncRequestFilterSettings) (usersync.SyncTypeFilter, error) { + syncTypeFilter := usersync.SyncTypeFilter{ + IFrame: cookieSyncBidderFilterAllowAll, + Redirect: cookieSyncBidderFilterAllowAll, } - for i := 0; i < len(parsedReq.Bidders); i++ { - bidder := parsedReq.Bidders[i] - syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy) - if err == nil { - newSync := &usersync.CookieSyncBidders{ - BidderCode: bidder, - NoCookie: true, - UsersyncInfo: syncInfo, - } - csResp.BidderStatus = append(csResp.BidderStatus, newSync) + + if request != nil { + if filter, err := parseBidderFilter(request.IFrame); err == nil { + syncTypeFilter.IFrame = filter } else { - glog.Errorf("Failed to get usersync info for %s: %v", bidder, err) + return emptySyncTypeFilter, fmt.Errorf("error parsing filtersettings.iframe: %v", err) } - } - if len(csResp.BidderStatus) > 0 { - co.BidderStatus = append(co.BidderStatus, csResp.BidderStatus...) + if filter, err := parseBidderFilter(request.Redirect); err == nil { + syncTypeFilter.Redirect = filter + } else { + return emptySyncTypeFilter, fmt.Errorf("error parsing filtersettings.image: %v", err) + } } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - enc.Encode(csResp) + return syncTypeFilter, nil } -func parseRequest(parsedReq *cookieSyncRequest, bodyBytes []byte, usersyncIfAmbiguous bool) error { - if err := json.Unmarshal(bodyBytes, parsedReq); err != nil { - return fmt.Errorf("JSON parsing failed: %s", err.Error()) +func parseBidderFilter(filter *cookieSyncRequestFilter) (usersync.BidderFilter, error) { + if filter == nil { + return cookieSyncBidderFilterAllowAll, nil } - if parsedReq.GDPR != nil && *parsedReq.GDPR == 1 && parsedReq.Consent == "" { - return errors.New("gdpr_consent is required if gdpr=1") + var mode usersync.BidderFilterMode + switch filter.Mode { + case "include": + mode = usersync.BidderFilterModeInclude + case "exclude": + mode = usersync.BidderFilterModeExclude + default: + return emptySyncBidderFilter, fmt.Errorf("invalid filter value '%s'. must be either 'include' or 'exclude'", filter.Mode) } - // If GDPR is ambiguous, lets untangle it here. - if parsedReq.GDPR == nil { - var gdpr = new(int) - *gdpr = 1 - if usersyncIfAmbiguous { - *gdpr = 0 + + switch v := filter.Bidders.(type) { + case string: + if v == "*" { + return usersync.NewBidderFilterForAll(mode), nil + } + return emptySyncBidderFilter, fmt.Errorf("invalid bidders value `%s`. must either be '*' or a string array", v) + case []interface{}: + bidders := make([]string, len(v)) + for i, x := range v { + if bidder, ok := x.(string); ok { + bidders[i] = bidder + } else { + return emptySyncBidderFilter, errCookieSyncInvalidBiddersType + } } - parsedReq.GDPR = gdpr + return usersync.NewBidderFilter(bidders, mode), nil + default: + return emptySyncBidderFilter, errCookieSyncInvalidBiddersType } - return nil } -func gdprToString(gdpr *int) string { - if gdpr == nil { - return "" +func (c *cookieSyncEndpoint) handleError(w http.ResponseWriter, err error, httpStatus int) { + http.Error(w, err.Error(), httpStatus) + c.pbsAnalytics.LogCookieSyncObject(&analytics.CookieSyncObject{ + Status: httpStatus, + Errors: []error{err}, + BidderStatus: []*analytics.CookieSyncBidder{}, + }) +} + +func (c *cookieSyncEndpoint) writeBidderMetrics(biddersEvaluated []usersync.BidderEvaluation) { + for _, bidder := range biddersEvaluated { + switch bidder.Status { + case usersync.StatusOK: + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerOK) + case usersync.StatusBlockedByGDPR: + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerPrivacyBlocked) + case usersync.StatusBlockedByCCPA: + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerPrivacyBlocked) + case usersync.StatusAlreadySynced: + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerAlreadySynced) + case usersync.StatusTypeNotSupported: + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerTypeNotSupported) + } } - return strconv.Itoa(*gdpr) } -func parseBidders(request []byte) ([]byte, error) { - value, valueType, _, err := jsonparser.Get(request, "bidders") - if err == nil && valueType != jsonparser.NotExist { - return value, nil - } else if err != jsonparser.KeyPathNotFoundError { - return nil, err +func (c *cookieSyncEndpoint) handleResponse(w http.ResponseWriter, tf usersync.SyncTypeFilter, co *usersync.Cookie, p privacy.Policies, s []usersync.SyncerChoice) { + status := "no_cookie" + if co.HasAnyLiveSyncs() { + status = "ok" + } + + response := cookieSyncResponse{ + Status: status, + BidderStatus: make([]cookieSyncResponseBidder, 0, len(s)), + } + + for _, syncerChoice := range s { + syncTypes := tf.ForBidder(syncerChoice.Bidder) + sync, err := syncerChoice.Syncer.GetSync(syncTypes, p) + if err != nil { + glog.Errorf("Failed to get usersync info for %s: %v", syncerChoice.Bidder, err) + continue + } + + response.BidderStatus = append(response.BidderStatus, cookieSyncResponseBidder{ + BidderCode: syncerChoice.Bidder, + NoCookie: true, + UsersyncInfo: cookieSyncResponseSync{ + URL: sync.URL, + Type: sync.Type.String(), + SupportCORS: sync.SupportsCORS, + }, + }) } - return nil, nil + + c.pbsAnalytics.LogCookieSyncObject(&analytics.CookieSyncObject{ + Status: http.StatusOK, + BidderStatus: mapBidderStatusToAnalytics(response.BidderStatus), + }) + + w.Header().Set("Content-Type", "application/json; charset=utf-8") + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + enc.Encode(response) } -func cookieSyncStatus(hasSyncs bool) string { - if hasSyncs { - return "ok" +func mapBidderStatusToAnalytics(from []cookieSyncResponseBidder) []*analytics.CookieSyncBidder { + to := make([]*analytics.CookieSyncBidder, len(from)) + for i, b := range from { + to[i] = &analytics.CookieSyncBidder{ + BidderCode: b.BidderCode, + NoCookie: b.NoCookie, + UsersyncInfo: &analytics.UsersyncInfo{ + URL: b.UsersyncInfo.URL, + Type: b.UsersyncInfo.Type, + SupportCORS: b.UsersyncInfo.SupportCORS, + }, + } } - return "no_cookie" + return to } type cookieSyncRequest struct { - Bidders []string `json:"bidders"` - GDPR *int `json:"gdpr"` - Consent string `json:"gdpr_consent"` - USPrivacy string `json:"us_privacy"` - Limit int `json:"limit"` + Bidders []string `json:"bidders"` + GDPR string `json:"gdpr"` + GDPRConsent string `json:"gdpr_consent"` + USPrivacy string `json:"us_privacy"` + Limit int `json:"limit"` + CooperativeSync *bool `json:"coopSync"` + FilterSettings *cookieSyncRequestFilterSettings `json:"filterSettings"` } -func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderName]usersync.Usersyncer, cookie *usersync.Cookie, needSyncupForSameSite bool) { - for i := 0; i < len(req.Bidders); i++ { - thisBidder := req.Bidders[i] - if syncer, isValid := valid[openrtb_ext.BidderName(thisBidder)]; !isValid || (cookie.HasLiveSync(syncer.FamilyName()) && !needSyncupForSameSite) { - req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) - i-- - } - } +type cookieSyncRequestFilterSettings struct { + IFrame *cookieSyncRequestFilter `json:"iframe"` + Redirect *cookieSyncRequestFilter `json:"image"` } -func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) { - if req.GDPR != nil && *req.GDPR == 0 { - return - } +type cookieSyncRequestFilter struct { + Bidders interface{} `json:"bidders"` + Mode string `json:"filter"` +} - // At this point we know the gdpr signal is Yes because the upstream call to parseRequest already denormalized the signal if it was ambiguous - if allowSync, err := permissions.HostCookiesAllowed(context.Background(), gdpr.SignalYes, req.Consent); err != nil || !allowSync { - req.Bidders = nil - return - } +type cookieSyncResponse struct { + Status string `json:"status"` + BidderStatus []cookieSyncResponseBidder `json:"bidder_status"` +} - for i := 0; i < len(req.Bidders); i++ { - if allowSync, err := permissions.BidderSyncAllowed(context.Background(), openrtb_ext.BidderName(req.Bidders[i]), gdpr.SignalYes, req.Consent); err != nil || !allowSync { - req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) - i-- - } - } +type cookieSyncResponseBidder struct { + BidderCode string `json:"bidder"` + NoCookie bool `json:"no_cookie,omitempty"` + UsersyncInfo cookieSyncResponseSync `json:"usersync,omitempty"` } -func (req *cookieSyncRequest) filterForCCPA(bidderMap map[string]struct{}) { - ccpaPolicy := &ccpa.Policy{Consent: req.USPrivacy} - ccpaParsedPolicy, err := ccpaPolicy.Parse(bidderMap) +type cookieSyncResponseSync struct { + URL string `json:"url,omitempty"` + Type string `json:"type,omitempty"` + SupportCORS bool `json:"supportCORS,omitempty"` +} - if err == nil { - for i := 0; i < len(req.Bidders); i++ { - if ccpaParsedPolicy.ShouldEnforce(req.Bidders[i]) { - req.Bidders = append(req.Bidders[:i], req.Bidders[i+1:]...) - i-- - } - } - } +type usersyncPrivacyConfig struct { + gdprConfig config.GDPR + gdprPermissions gdpr.Permissions + ccpaEnforce bool + bidderHashSet map[string]struct{} } -// filterToLimit will enforce a max limit on cookiesyncs supplied, picking a random subset of syncs to get to the limit if over. -func (req *cookieSyncRequest) filterToLimit() { - if req.Limit <= 0 { - return - } - if req.Limit >= len(req.Bidders) { - return - } +type usersyncPrivacy struct { + gdprPermissions gdpr.Permissions + gdprSignal gdpr.Signal + gdprConsent string + ccpaParsedPolicy ccpa.ParsedPolicy +} - // Modified Fisher and Yates' shuffle. We don't need the bidder list shuffled, so we stop shuffling once the final values beyond limit have been set. - // We also don't bother saving the values that should go into the entries beyond limit, as they will be discarded. - for i := len(req.Bidders) - 1; i >= req.Limit; i-- { - j := rand.Intn(i + 1) - if i != j { - req.Bidders[j] = req.Bidders[i] - // Don't complete the swap as the new value for req.Bidders[i] will be discarded below, and will never again be accessed as part of the swapping. - } - } - req.Bidders = req.Bidders[:req.Limit] - return +func (p usersyncPrivacy) GDPRAllowsHostCookie() bool { + allowCookie, err := p.gdprPermissions.HostCookiesAllowed(context.Background(), p.gdprSignal, p.gdprConsent) + return err == nil && allowCookie } -type cookieSyncResponse struct { - Status string `json:"status"` - BidderStatus []*usersync.CookieSyncBidders `json:"bidder_status"` +func (p usersyncPrivacy) GDPRAllowsBidderSync(bidder string) bool { + allowSync, err := p.gdprPermissions.BidderSyncAllowed(context.Background(), openrtb_ext.BidderName(bidder), p.gdprSignal, p.gdprConsent) + return err == nil && allowSync +} + +func (p usersyncPrivacy) CCPAAllowsBidderSync(bidder string) bool { + return p.ccpaParsedPolicy.ShouldEnforce(bidder) } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 84e9545e1ff..327604779ca 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -2,225 +2,1313 @@ package endpoints import ( "context" + "errors" + "io" "net/http" "net/http/httptest" "strings" "testing" - "text/template" + "testing/iotest" "time" - "github.com/buger/jsonparser" - "github.com/julienschmidt/httprouter" - "github.com/prebid/prebid-server/adapters/appnexus" - "github.com/prebid/prebid-server/adapters/audienceNetwork" - "github.com/prebid/prebid-server/adapters/lifestreet" - "github.com/prebid/prebid-server/adapters/pubmatic" - analyticsConf "github.com/prebid/prebid-server/analytics/config" + "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/gdpr" - metricsConf "github.com/prebid/prebid-server/metrics/config" + "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" + "github.com/prebid/prebid-server/privacy/ccpa" + gdprPrivacy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) -func TestCookieSyncNoCookies(t *testing.T) { - rr := doPost(`{"bidders":["appnexus", "audienceNetwork", "random"]}`, nil, true, syncersForTest()) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"appnexus", "audienceNetwork"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} +// end to end test? -func TestGDPRPreventsCookie(t *testing.T) { - rr := doPost(`{"bidders":["appnexus", "pubmatic"]}`, nil, false, syncersForTest()) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.Empty(t, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} +func TestNewCookieSyncEndpoint(t *testing.T) { + var ( + syncers = map[string]usersync.Syncer{"a": &MockSyncer{}} + gdprPerms = MockGDPRPerms{} + configUserSync = config.UserSync{Cooperative: config.UserSyncCooperative{EnabledByDefault: true}} + configHostCookie = config.HostCookie{Family: "foo"} + configGDPR = config.GDPR{HostVendorID: 42} + configCCPAEnforce = true + metrics = metrics.MetricsEngineMock{} + analytics = MockAnalytics{} + bidders = map[string]openrtb_ext.BidderName{"bidderA": openrtb_ext.BidderName("bidderA"), "bidderB": openrtb_ext.BidderName("bidderB")} + ) -func TestGDPRPreventsBidders(t *testing.T) { - rr := doPost(`{"gdpr":1,"bidders":["appnexus", "pubmatic", "lifestreet"],"gdpr_consent":"BOONs2HOONs2HABABBENAGgAAAAPrABACGA"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{ - openrtb_ext.BidderLifestreet: lifestreet.NewLifestreetSyncer(template.Must(template.New("sync").Parse("someurl.com"))), - }) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"lifestreet"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} + endpoint := NewCookieSyncEndpoint( + syncers, + &config.Configuration{ + UserSync: configUserSync, + HostCookie: configHostCookie, + GDPR: configGDPR, + CCPA: config.CCPA{Enforce: configCCPAEnforce}, + }, + &gdprPerms, + &metrics, + &analytics, + bidders, + ) + + expected := &cookieSyncEndpoint{ + chooser: usersync.NewChooser(syncers), + config: configUserSync, + hostCookieConfig: &configHostCookie, + privacyConfig: usersyncPrivacyConfig{ + gdprConfig: configGDPR, + gdprPermissions: &gdprPerms, + ccpaEnforce: configCCPAEnforce, + bidderHashSet: map[string]struct{}{"bidderA": {}, "bidderB": {}}, + }, + metrics: &metrics, + pbsAnalytics: &analytics, + } -func TestGDPRIgnoredIfZero(t *testing.T) { - rr := doPost(`{"gdpr":0,"bidders":["appnexus", "pubmatic"]}`, nil, false, nil) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"appnexus", "pubmatic"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) + assert.Equal(t, expected, endpoint) } -func TestGDPRConsentRequired(t *testing.T) { - rr := doPost(`{"gdpr":1,"bidders":["appnexus", "pubmatic"]}`, nil, false, nil) - assert.Equal(t, rr.Header().Get("Content-Type"), "text/plain; charset=utf-8") - assert.Equal(t, http.StatusBadRequest, rr.Code) - assert.Equal(t, "gdpr_consent is required if gdpr=1\n", rr.Body.String()) +// usersyncPrivacy +func TestCookieSyncHandle(t *testing.T) { + syncTypeExpected := []usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect} + sync := usersync.Sync{URL: "aURL", Type: usersync.SyncTypeRedirect, SupportsCORS: true} + syncer := MockSyncer{} + syncer.On("GetSync", syncTypeExpected, privacy.Policies{}).Return(sync, nil).Maybe() + + cookieWithSyncs := usersync.NewCookie() + cookieWithSyncs.TrySync("foo", "anyID") + + testCases := []struct { + description string + givenCookie *usersync.Cookie + givenBody io.Reader + givenChooserResult usersync.Result + expectedStatusCode int + expectedBody string + setMetricsExpectations func(*metrics.MetricsEngineMock) + setAnalyticsExpectations func(*MockAnalytics) + }{ + { + description: "Request With Cookie", + givenCookie: cookieWithSyncs, + givenBody: strings.NewReader(`{}`), + givenChooserResult: usersync.Result{ + Status: usersync.StatusOK, + BiddersEvaluated: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, + }, + expectedStatusCode: 200, + expectedBody: `{"status":"ok","bidder_status":[` + + `{"bidder":"a","no_cookie":true,"usersync":{"url":"aURL","type":"redirect","supportCORS":true}}` + + `]}` + "\n", + setMetricsExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordCookieSync", metrics.CookieSyncOK).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + }, + setAnalyticsExpectations: func(a *MockAnalytics) { + expected := analytics.CookieSyncObject{ + Status: 200, + Errors: nil, + BidderStatus: []*analytics.CookieSyncBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "aURL", Type: "redirect", SupportCORS: true}, + }, + }, + } + a.On("LogCookieSyncObject", &expected).Once() + }, + }, + { + description: "Request Without Cookie", + givenCookie: nil, + givenBody: strings.NewReader(`{}`), + givenChooserResult: usersync.Result{ + Status: usersync.StatusOK, + BiddersEvaluated: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, + }, + expectedStatusCode: 200, + expectedBody: `{"status":"no_cookie","bidder_status":[` + + `{"bidder":"a","no_cookie":true,"usersync":{"url":"aURL","type":"redirect","supportCORS":true}}` + + `]}` + "\n", + setMetricsExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordCookieSync", metrics.CookieSyncOK).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + }, + setAnalyticsExpectations: func(a *MockAnalytics) { + expected := analytics.CookieSyncObject{ + Status: 200, + Errors: nil, + BidderStatus: []*analytics.CookieSyncBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "aURL", Type: "redirect", SupportCORS: true}, + }, + }, + } + a.On("LogCookieSyncObject", &expected).Once() + }, + }, + { + description: "Malformed Request", + givenCookie: cookieWithSyncs, + givenBody: strings.NewReader(`malformed`), + givenChooserResult: usersync.Result{ + Status: usersync.StatusOK, + BiddersEvaluated: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, + }, + expectedStatusCode: 400, + expectedBody: `JSON parsing failed: invalid character 'm' looking for beginning of value` + "\n", + setMetricsExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordCookieSync", metrics.CookieSyncBadRequest).Once() + }, + setAnalyticsExpectations: func(a *MockAnalytics) { + expected := analytics.CookieSyncObject{ + Status: 400, + Errors: []error{errors.New("JSON parsing failed: invalid character 'm' looking for beginning of value")}, + BidderStatus: []*analytics.CookieSyncBidder{}, + } + a.On("LogCookieSyncObject", &expected).Once() + }, + }, + { + description: "Request Blocked By Opt Out", + givenCookie: cookieWithSyncs, + givenBody: strings.NewReader(`{}`), + givenChooserResult: usersync.Result{ + Status: usersync.StatusBlockedByUserOptOut, + BiddersEvaluated: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, + }, + expectedStatusCode: 401, + expectedBody: `User has opted out` + "\n", + setMetricsExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordCookieSync", metrics.CookieSyncOptOut).Once() + }, + setAnalyticsExpectations: func(a *MockAnalytics) { + expected := analytics.CookieSyncObject{ + Status: 401, + Errors: []error{errors.New("User has opted out")}, + BidderStatus: []*analytics.CookieSyncBidder{}, + } + a.On("LogCookieSyncObject", &expected).Once() + }, + }, + { + description: "Request Blocked By GDPR Host Cookie Restriction", + givenCookie: cookieWithSyncs, + givenBody: strings.NewReader(`{}`), + givenChooserResult: usersync.Result{ + Status: usersync.StatusBlockedByGDPR, + BiddersEvaluated: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + SyncersChosen: []usersync.SyncerChoice{{Bidder: "a", Syncer: &syncer}}, + }, + expectedStatusCode: 200, + expectedBody: `{"status":"ok","bidder_status":[]}` + "\n", + setMetricsExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordCookieSync", metrics.CookieSyncGDPRHostCookieBlocked).Once() + }, + setAnalyticsExpectations: func(a *MockAnalytics) { + expected := analytics.CookieSyncObject{ + Status: 200, + Errors: nil, + BidderStatus: []*analytics.CookieSyncBidder{}, + } + a.On("LogCookieSyncObject", &expected).Once() + }, + }, + } + + for _, test := range testCases { + mockMetrics := metrics.MetricsEngineMock{} + test.setMetricsExpectations(&mockMetrics) + + mockAnalytics := MockAnalytics{} + test.setAnalyticsExpectations(&mockAnalytics) + + request := httptest.NewRequest("POST", "/cookiesync", test.givenBody) + if test.givenCookie != nil { + request.AddCookie(test.givenCookie.ToHTTPCookie(24 * time.Hour)) + } + + writer := httptest.NewRecorder() + + endpoint := cookieSyncEndpoint{ + chooser: FakeChooser{Result: test.givenChooserResult}, + hostCookieConfig: &config.HostCookie{}, + privacyConfig: usersyncPrivacyConfig{ + gdprConfig: config.GDPR{ + Enabled: true, + UsersyncIfAmbiguous: true, + }, + ccpaEnforce: true, + }, + metrics: &mockMetrics, + pbsAnalytics: &mockAnalytics, + } + endpoint.Handle(writer, request, nil) + + assert.Equal(t, test.expectedStatusCode, writer.Code, test.description+":status_code") + assert.Equal(t, test.expectedBody, writer.Body.String(), test.description+":body") + mockMetrics.AssertExpectations(t) + mockAnalytics.AssertExpectations(t) + } } -func TestCCPA(t *testing.T) { +func TestCookieSyncParseRequest(t *testing.T) { + expectedCCPAParsedPolicy, _ := ccpa.Policy{Consent: "1NYN"}.Parse(map[string]struct{}{}) + testCases := []struct { - description string - requestBody string - enforceCCPA bool - expectedSyncs []string + description string + givenConfig config.UserSync + givenBody io.Reader + givenGDPRConfig config.GDPR + givenCCPAEnabled bool + expectedError string + expectedPrivacy privacy.Policies + expectedRequest usersync.Request }{ { - description: "Feature Flag On & Opt-Out Yes", - requestBody: `{"bidders":["appnexus"], "us_privacy":"1-Y-"}`, - enforceCCPA: true, - expectedSyncs: []string{}, + description: "Complete Request", + givenBody: strings.NewReader(`{` + + `"bidders":["a", "b"],` + + `"gdpr":"1",` + + `"gdpr_consent":"anyGDPRConsent",` + + `"us_privacy":"1NYN",` + + `"limit":42,` + + `"coopSync":true,` + + `"filterSettings":{"iframe":{"bidders":"*","filter":"include"}, "image":{"bidders":["b"],"filter":"exclude"}}` + + `}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{ + GDPR: gdprPrivacy.Policy{ + Signal: "1", + Consent: "anyGDPRConsent", + }, + CCPA: ccpa.Policy{ + Consent: "1NYN", + }, + }, + expectedRequest: usersync.Request{ + Bidders: []string{"a", "b"}, + Cooperative: usersync.Cooperative{ + Enabled: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Limit: 42, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalYes, + gdprConsent: "anyGDPRConsent", + ccpaParsedPolicy: expectedCCPAParsedPolicy, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + }, + }, + }, + { + description: "Complete Request - Legacy Fields Only", + givenBody: strings.NewReader(`{` + + `"bidders":["a", "b"],` + + `"gdpr":"1",` + + `"gdpr_consent":"anyGDPRConsent",` + + `"us_privacy":"1NYN",` + + `"limit":42` + + `}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{ + GDPR: gdprPrivacy.Policy{ + Signal: "1", + Consent: "anyGDPRConsent", + }, + CCPA: ccpa.Policy{ + Consent: "1NYN", + }, + }, + expectedRequest: usersync.Request{ + Bidders: []string{"a", "b"}, + Cooperative: usersync.Cooperative{ + Enabled: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Limit: 42, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalYes, + gdprConsent: "anyGDPRConsent", + ccpaParsedPolicy: expectedCCPAParsedPolicy, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Empty Request", + givenBody: strings.NewReader(`{}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Cooperative Unspecified - Default True", + givenBody: strings.NewReader(`{}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Cooperative Unspecified - Default False", + givenBody: strings.NewReader(`{}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Cooperative False - Default True", + givenBody: strings.NewReader(`{"coopSync":false}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Cooperative False - Default False", + givenBody: strings.NewReader(`{"coopSync":false}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, }, { - description: "Feature Flag Off & Opt-Out Yes", - requestBody: `{"bidders":["appnexus"], "us_privacy":"1-Y-"}`, - enforceCCPA: false, - expectedSyncs: []string{"appnexus"}, + description: "Cooperative True - Default True", + givenBody: strings.NewReader(`{"coopSync":true}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, }, { - description: "Feature Flag On & Opt-Out No", - requestBody: `{"bidders":["appnexus"], "us_privacy":"1-N-"}`, - enforceCCPA: false, - expectedSyncs: []string{"appnexus"}, + description: "Cooperative True - Default False", + givenBody: strings.NewReader(`{"coopSync":true}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + givenConfig: config.UserSync{ + Cooperative: config.UserSyncCooperative{ + EnabledByDefault: false, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + }, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Cooperative: usersync.Cooperative{ + Enabled: true, + PriorityGroups: [][]string{{"a", "b", "c"}}, + }, + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, }, { - description: "Feature Flag On & Opt-Out Unknown", - requestBody: `{"bidders":["appnexus"], "us_privacy":"1---"}`, - enforceCCPA: false, - expectedSyncs: []string{"appnexus"}, + description: "CCPA Consent Invalid", + givenBody: strings.NewReader(`{"us_privacy":"invalid"}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedPrivacy: privacy.Policies{}, + expectedRequest: usersync.Request{ + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, }, { - description: "Feature Flag On & Opt-Out Invalid", - requestBody: `{"bidders":["appnexus"], "us_privacy":"invalid"}`, - enforceCCPA: false, - expectedSyncs: []string{"appnexus"}, + description: "CCPA Disabled", + givenBody: strings.NewReader(`{"us_privacy":"1NYN"}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: false, + expectedPrivacy: privacy.Policies{ + CCPA: ccpa.Policy{ + Consent: "1NYN"}, + }, + expectedRequest: usersync.Request{ + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, }, { - description: "Feature Flag On & Opt-Out Not Provided", - requestBody: `{"bidders":["appnexus"]}`, - enforceCCPA: false, - expectedSyncs: []string{"appnexus"}, + description: "Invalid JSON", + givenBody: strings.NewReader(`malformed`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedError: "JSON parsing failed: invalid character 'm' looking for beginning of value", + }, + { + description: "Invalid Type Filter", + givenBody: strings.NewReader(`{"filterSettings":{"iframe":{"bidders":"invalid","filter":"exclude"}}}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedError: "error parsing filtersettings.iframe: invalid bidders value `invalid`. must either be '*' or a string array", + }, + { + description: "Invalid GDPR Signal", + givenBody: strings.NewReader(`{"gdpr":"invalid"}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedError: "GDPR signal should be integer 0 or 1", + }, + { + description: "Missing GDPR Consent - Explicit Signal 0", + givenBody: strings.NewReader(`{"gdpr":"0"}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedPrivacy: privacy.Policies{ + GDPR: gdprPrivacy.Policy{Signal: "0"}, + }, + expectedRequest: usersync.Request{ + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalNo, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Missing GDPR Consent - Explicit Signal 1", + givenBody: strings.NewReader(`{"gdpr":"1"}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedError: "gdpr_consent is required if gdpr=1", + }, + { + description: "Missing GDPR Consent - Ambiguous Signal - Default Value 0", + givenBody: strings.NewReader(`{}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedPrivacy: privacy.Policies{ + GDPR: gdprPrivacy.Policy{Signal: ""}, + }, + expectedRequest: usersync.Request{ + Privacy: usersyncPrivacy{ + gdprSignal: gdpr.SignalAmbiguous, + }, + SyncTypeFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + }, + { + description: "Missing GDPR Consent - Ambiguous Signal - Default Value 1", + givenBody: strings.NewReader(`{}`), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: false}, + givenCCPAEnabled: true, + expectedError: "gdpr_consent is required. gdpr is not specified and is assumed to be 1 by the server. set gdpr=0 to exempt this request", + }, + { + description: "HTTP Read Error", + givenBody: iotest.ErrReader(errors.New("anyError")), + givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, + givenCCPAEnabled: true, + expectedError: "Failed to read request body", }, } for _, test := range testCases { - gdpr := config.GDPR{UsersyncIfAmbiguous: true} - ccpa := config.CCPA{Enforce: test.enforceCCPA} - rr := doConfigurablePost(test.requestBody, nil, true, syncersForTest(), gdpr, ccpa) - assert.Equal(t, http.StatusOK, rr.Code, test.description+":httpResponseCode") - assert.ElementsMatch(t, test.expectedSyncs, parseSyncs(t, rr.Body.Bytes()), test.description+":syncs") - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes()), test.description+":status") + httpRequest := httptest.NewRequest("POST", "/cookiesync", test.givenBody) + + endpoint := cookieSyncEndpoint{ + config: test.givenConfig, + privacyConfig: usersyncPrivacyConfig{ + gdprConfig: test.givenGDPRConfig, + ccpaEnforce: test.givenCCPAEnabled, + }, + } + request, privacyPolicies, err := endpoint.parseRequest(httpRequest) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedRequest, request, test.description+":request") + assert.Equal(t, test.expectedPrivacy, privacyPolicies, test.description+":privacy") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, request, test.description+":request") + assert.Empty(t, privacyPolicies, test.description+":privacy") + } } } -func TestCookieSyncHasCookies(t *testing.T) { - rr := doPost(`{"bidders":["appnexus", "audienceNetwork", "random"]}`, map[string]string{ - "adnxs": "1234", - "audienceNetwork": "2345", - }, true, syncersForTest()) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.Empty(t, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "ok", parseStatus(t, rr.Body.Bytes())) -} +func TestParseTypeFilter(t *testing.T) { + testCases := []struct { + description string + given *cookieSyncRequestFilterSettings + expectedError string + expectedFilter usersync.SyncTypeFilter + }{ + { + description: "Nil", + given: nil, + expectedFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + { + description: "Nil Object", + given: &cookieSyncRequestFilterSettings{}, + expectedFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + { + description: "Given IFrame Only", + given: &cookieSyncRequestFilterSettings{ + IFrame: &cookieSyncRequestFilter{Bidders: []interface{}{"a"}, Mode: "exclude"}, + }, + expectedFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + }, + { + description: "Given Redirect Only", + given: &cookieSyncRequestFilterSettings{ + Redirect: &cookieSyncRequestFilter{Bidders: []interface{}{"b"}, Mode: "exclude"}, + }, + expectedFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + }, + }, + { + description: "Given Both", + given: &cookieSyncRequestFilterSettings{ + IFrame: &cookieSyncRequestFilter{Bidders: []interface{}{"a"}, Mode: "exclude"}, + Redirect: &cookieSyncRequestFilter{Bidders: []interface{}{"b"}, Mode: "exclude"}, + }, + expectedFilter: usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), + Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + }, + }, + { + description: "IFrame Error", + given: &cookieSyncRequestFilterSettings{ + IFrame: &cookieSyncRequestFilter{Bidders: 42, Mode: "exclude"}, + Redirect: &cookieSyncRequestFilter{Bidders: []interface{}{"b"}, Mode: "exclude"}, + }, + expectedError: "error parsing filtersettings.iframe: invalid bidders type. must either be a string '*' or a string array of bidders", + }, + { + description: "Redirect Error", + given: &cookieSyncRequestFilterSettings{ + IFrame: &cookieSyncRequestFilter{Bidders: []interface{}{"a"}, Mode: "exclude"}, + Redirect: &cookieSyncRequestFilter{Bidders: 42, Mode: "exclude"}, + }, + expectedError: "error parsing filtersettings.image: invalid bidders type. must either be a string '*' or a string array of bidders", + }, + } -func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) { - rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true}, config.CCPA{}) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"appnexus", "audienceNetwork"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} + for _, test := range testCases { + result, err := parseTypeFilter(test.given) -func doPost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer) *httptest.ResponseRecorder { - return doConfigurablePost(body, existingSyncs, gdprHostConsent, gdprBidders, config.GDPR{}, config.CCPA{}) + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedFilter, result, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result, test.description+":result") + } + } } -func doConfigurablePost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer, cfgGDPR config.GDPR, cfgCCPA config.CCPA) *httptest.ResponseRecorder { - endpoint := testableEndpoint(mockPermissions(gdprHostConsent, gdprBidders), cfgGDPR, cfgCCPA) - router := httprouter.New() - router.POST("/cookie_sync", endpoint) - req, _ := http.NewRequest("POST", "/cookie_sync", strings.NewReader(body)) - if len(existingSyncs) > 0 { +func TestParseBidderFilter(t *testing.T) { + testCases := []struct { + description string + given *cookieSyncRequestFilter + expectedError string + expectedFilter usersync.BidderFilter + }{ + { + description: "Nil", + given: nil, + expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + { + description: "All Bidders - Include", + given: &cookieSyncRequestFilter{Bidders: "*", Mode: "include"}, + expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + }, + { + description: "All Bidders - Exclude", + given: &cookieSyncRequestFilter{Bidders: "*", Mode: "exclude"}, + expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeExclude), + }, + { + description: "All Bidders - Invalid Mode", + given: &cookieSyncRequestFilter{Bidders: "*", Mode: "invalid"}, + expectedError: "invalid filter value 'invalid'. must be either 'include' or 'exclude'", + }, + { + description: "All Bidders - Unexpected Bidders Value", + given: &cookieSyncRequestFilter{Bidders: "invalid", Mode: "include"}, + expectedError: "invalid bidders value `invalid`. must either be '*' or a string array", + }, + { + description: "Specific Bidders - Include", + given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", "b"}, Mode: "include"}, + expectedFilter: usersync.NewBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeInclude), + }, + { + description: "Specific Bidders - Exclude", + given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", "b"}, Mode: "exclude"}, + expectedFilter: usersync.NewBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeExclude), + }, + { + description: "Specific Bidders - Invalid Mode", + given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", "b"}, Mode: "invalid"}, + expectedError: "invalid filter value 'invalid'. must be either 'include' or 'exclude'", + }, + { + description: "Invalid Bidders Type", + given: &cookieSyncRequestFilter{Bidders: 42, Mode: "include"}, + expectedError: "invalid bidders type. must either be a string '*' or a string array of bidders", + }, + { + description: "Invalid Bidders Type Of Array Element", + given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", 42}, Mode: "include"}, + expectedError: "invalid bidders type. must either be a string '*' or a string array of bidders", + }, + } + + for _, test := range testCases { + result, err := parseBidderFilter(test.given) - pcs := usersync.NewCookie() - for bidder, uid := range existingSyncs { - pcs.TrySync(bidder, uid) + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedFilter, result, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result, test.description+":result") } - req.AddCookie(pcs.ToHTTPCookie(90 * 24 * time.Hour)) } +} + +func TestCookieSyncHandleError(t *testing.T) { + err := errors.New("anyError") + + mockAnalytics := MockAnalytics{} + mockAnalytics.On("LogCookieSyncObject", mock.Anything) + writer := httptest.NewRecorder() + + endpoint := cookieSyncEndpoint{pbsAnalytics: &mockAnalytics} + endpoint.handleError(writer, err, 418) - rr := httptest.NewRecorder() - endpoint(rr, req, nil) - return rr + assert.Equal(t, writer.Code, 418) + assert.Equal(t, writer.Body.String(), "anyError\n") + mockAnalytics.AssertCalled(t, "LogCookieSyncObject", &analytics.CookieSyncObject{ + Status: 418, + Errors: []error{err}, + BidderStatus: []*analytics.CookieSyncBidder{}, + }) } -func testableEndpoint(perms gdpr.Permissions, cfgGDPR config.GDPR, cfgCCPA config.CCPA) httprouter.Handle { - return NewCookieSyncEndpoint(syncersForTest(), &config.Configuration{GDPR: cfgGDPR, CCPA: cfgCCPA}, perms, &metricsConf.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), openrtb_ext.BuildBidderMap()) +func TestCookieSyncWriteBidderMetrics(t *testing.T) { + testCases := []struct { + description string + given []usersync.BidderEvaluation + setExpectations func(*metrics.MetricsEngineMock) + }{ + { + description: "None", + given: []usersync.BidderEvaluation{}, + setExpectations: func(m *metrics.MetricsEngineMock) { + }, + }, + { + description: "One - OK", + given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + }, + }, + { + description: "One - Blocked By GDPR", + given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusBlockedByGDPR}}, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerPrivacyBlocked).Once() + }, + }, + { + description: "One - Blocked By CCPA", + given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusBlockedByCCPA}}, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerPrivacyBlocked).Once() + }, + }, + { + description: "One - Already Synced", + given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusAlreadySynced}}, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerAlreadySynced).Once() + }, + }, + { + description: "One - Type Not Supported", + given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusTypeNotSupported}}, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerTypeNotSupported).Once() + }, + }, + { + description: "Many", + given: []usersync.BidderEvaluation{ + {Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}, + {Bidder: "b", SyncerKey: "bSyncer", Status: usersync.StatusAlreadySynced}, + }, + setExpectations: func(m *metrics.MetricsEngineMock) { + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + m.On("RecordSyncerRequest", "bSyncer", metrics.SyncerAlreadySynced).Once() + }, + }, + } + + for _, test := range testCases { + mockMetrics := metrics.MetricsEngineMock{} + test.setExpectations(&mockMetrics) + + endpoint := &cookieSyncEndpoint{metrics: &mockMetrics} + endpoint.writeBidderMetrics(test.given) + + mockMetrics.AssertExpectations(t) + } } -func syncersForTest() map[openrtb_ext.BidderName]usersync.Usersyncer { - return map[openrtb_ext.BidderName]usersync.Usersyncer{ - openrtb_ext.BidderAppnexus: appnexus.NewAppnexusSyncer(template.Must(template.New("sync").Parse("someurl.com"))), - openrtb_ext.BidderAudienceNetwork: audienceNetwork.NewFacebookSyncer(template.Must(template.New("sync").Parse("https://www.facebook.com/audiencenetwork/idsync/?partner=partnerId&callback=localhost%2Fsetuid%3Fbidder%3DaudienceNetwork%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID"))), - openrtb_ext.BidderLifestreet: lifestreet.NewLifestreetSyncer(template.Must(template.New("sync").Parse("anotherurl.com"))), - openrtb_ext.BidderPubmatic: pubmatic.NewPubmaticSyncer(template.Must(template.New("sync").Parse("thaturl.com"))), +func TestCookieSyncHandleResponse(t *testing.T) { + syncTypeFilter := usersync.SyncTypeFilter{ + IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeExclude), + Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + } + syncTypeExpected := []usersync.SyncType{usersync.SyncTypeRedirect} + privacyPolicies := privacy.Policies{CCPA: ccpa.Policy{Consent: "anyConsent"}} + + // The & in the URL is necessary to test proper JSON encoding. + syncA := usersync.Sync{URL: "https://syncA.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportsCORS: true} + syncerA := MockSyncer{} + syncerA.On("GetSync", syncTypeExpected, privacyPolicies).Return(syncA, nil).Maybe() + + // The & in the URL is necessary to test proper JSON encoding. + syncB := usersync.Sync{URL: "https://syncB.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportsCORS: false} + syncerB := MockSyncer{} + syncerB.On("GetSync", syncTypeExpected, privacyPolicies).Return(syncB, nil).Maybe() + + syncWithError := usersync.Sync{} + syncerWithError := MockSyncer{} + syncerWithError.On("GetSync", syncTypeExpected, privacyPolicies).Return(syncWithError, errors.New("anyError")).Maybe() + + testCases := []struct { + description string + givenCookieHasSyncs bool + givenSyncersChosen []usersync.SyncerChoice + expectedJSON string + expectedAnalytics analytics.CookieSyncObject + }{ + { + description: "None", + givenCookieHasSyncs: true, + givenSyncersChosen: []usersync.SyncerChoice{}, + expectedJSON: `{"status":"ok","bidder_status":[]}` + "\n", + expectedAnalytics: analytics.CookieSyncObject{Status: 200, BidderStatus: []*analytics.CookieSyncBidder{}}, + }, + { + description: "One", + givenCookieHasSyncs: true, + givenSyncersChosen: []usersync.SyncerChoice{{Bidder: "foo", Syncer: &syncerA}}, + expectedJSON: `{"status":"ok","bidder_status":[` + + `{"bidder":"foo","no_cookie":true,"usersync":{"url":"https://syncA.com/sync?a=1&b=2","type":"redirect","supportCORS":true}}` + + `]}` + "\n", + expectedAnalytics: analytics.CookieSyncObject{ + Status: 200, + BidderStatus: []*analytics.CookieSyncBidder{ + { + BidderCode: "foo", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "https://syncA.com/sync?a=1&b=2", Type: "redirect", SupportCORS: true}, + }, + }, + }, + }, + { + description: "Many", + givenCookieHasSyncs: true, + givenSyncersChosen: []usersync.SyncerChoice{{Bidder: "foo", Syncer: &syncerA}, {Bidder: "bar", Syncer: &syncerB}}, + expectedJSON: `{"status":"ok","bidder_status":[` + + `{"bidder":"foo","no_cookie":true,"usersync":{"url":"https://syncA.com/sync?a=1&b=2","type":"redirect","supportCORS":true}},` + + `{"bidder":"bar","no_cookie":true,"usersync":{"url":"https://syncB.com/sync?a=1&b=2","type":"redirect"}}` + + `]}` + "\n", + expectedAnalytics: analytics.CookieSyncObject{ + Status: 200, + BidderStatus: []*analytics.CookieSyncBidder{ + { + BidderCode: "foo", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "https://syncA.com/sync?a=1&b=2", Type: "redirect", SupportCORS: true}, + }, + { + BidderCode: "bar", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "https://syncB.com/sync?a=1&b=2", Type: "redirect", SupportCORS: false}, + }, + }, + }, + }, + { + description: "Many With One GetSync Error", + givenCookieHasSyncs: true, + givenSyncersChosen: []usersync.SyncerChoice{{Bidder: "foo", Syncer: &syncerWithError}, {Bidder: "bar", Syncer: &syncerB}}, + expectedJSON: `{"status":"ok","bidder_status":[` + + `{"bidder":"bar","no_cookie":true,"usersync":{"url":"https://syncB.com/sync?a=1&b=2","type":"redirect"}}` + + `]}` + "\n", + expectedAnalytics: analytics.CookieSyncObject{ + Status: 200, + BidderStatus: []*analytics.CookieSyncBidder{ + { + BidderCode: "bar", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "https://syncB.com/sync?a=1&b=2", Type: "redirect", SupportCORS: false}, + }, + }, + }, + }, + { + description: "No Existing Syncs", + givenCookieHasSyncs: false, + givenSyncersChosen: []usersync.SyncerChoice{}, + expectedJSON: `{"status":"no_cookie","bidder_status":[]}` + "\n", + expectedAnalytics: analytics.CookieSyncObject{Status: 200, BidderStatus: []*analytics.CookieSyncBidder{}}, + }, + } + + for _, test := range testCases { + mockAnalytics := MockAnalytics{} + mockAnalytics.On("LogCookieSyncObject", &test.expectedAnalytics).Once() + + cookie := usersync.NewCookie() + if test.givenCookieHasSyncs { + if err := cookie.TrySync("foo", "anyID"); err != nil { + assert.FailNow(t, test.description+":set_cookie") + } + } + + writer := httptest.NewRecorder() + endpoint := cookieSyncEndpoint{pbsAnalytics: &mockAnalytics} + endpoint.handleResponse(writer, syncTypeFilter, cookie, privacyPolicies, test.givenSyncersChosen) + + if assert.Equal(t, writer.Code, http.StatusOK, test.description+":http_status") { + assert.Equal(t, writer.Header().Get("Content-Type"), "application/json; charset=utf-8", test.description+":http_header") + assert.Equal(t, test.expectedJSON, writer.Body.String(), test.description+":http_response") + } + mockAnalytics.AssertExpectations(t) } } -func parseStatus(t *testing.T, responseBody []byte) string { - t.Helper() - val, err := jsonparser.GetString(responseBody, "status") - if err != nil { - t.Fatalf("response.status was not a string. Error was %v", err) +func TestMapBidderStatusToAnalytics(t *testing.T) { + testCases := []struct { + description string + given []cookieSyncResponseBidder + expected []*analytics.CookieSyncBidder + }{ + { + description: "None", + given: []cookieSyncResponseBidder{}, + expected: []*analytics.CookieSyncBidder{}, + }, + { + description: "One", + given: []cookieSyncResponseBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: cookieSyncResponseSync{URL: "aURL", Type: "aType", SupportCORS: false}, + }, + }, + expected: []*analytics.CookieSyncBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "aURL", Type: "aType", SupportCORS: false}, + }, + }, + }, + { + description: "Many", + given: []cookieSyncResponseBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: cookieSyncResponseSync{URL: "aURL", Type: "aType", SupportCORS: false}, + }, + { + BidderCode: "b", + NoCookie: false, + UsersyncInfo: cookieSyncResponseSync{URL: "bURL", Type: "bType", SupportCORS: true}, + }, + }, + expected: []*analytics.CookieSyncBidder{ + { + BidderCode: "a", + NoCookie: true, + UsersyncInfo: &analytics.UsersyncInfo{URL: "aURL", Type: "aType", SupportCORS: false}, + }, + { + BidderCode: "b", + NoCookie: false, + UsersyncInfo: &analytics.UsersyncInfo{URL: "bURL", Type: "bType", SupportCORS: true}, + }, + }, + }, + } + + for _, test := range testCases { + result := mapBidderStatusToAnalytics(test.given) + assert.ElementsMatch(t, test.expected, result, test.description) } - return val } -func parseSyncs(t *testing.T, response []byte) []string { - t.Helper() - var syncs []string - jsonparser.ArrayEach(response, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { - if dataType != jsonparser.Object { - t.Errorf("response.bidder_status contained unexpected element of type %v.", dataType) +func TestUsersyncPrivacyGDPRAllowsHostCookie(t *testing.T) { + testCases := []struct { + description string + givenResponse bool + givenError error + expected bool + }{ + { + description: "Allowed - No Error", + givenResponse: true, + givenError: nil, + expected: true, + }, + { + description: "Allowed - Error", + givenResponse: true, + givenError: errors.New("anyError"), + expected: false, + }, + { + description: "Not Allowed - No Error", + givenResponse: false, + givenError: nil, + expected: false, + }, + { + description: "Not Allowed - Error", + givenResponse: false, + givenError: errors.New("anyError"), + expected: false, + }, + } + + for _, test := range testCases { + mockPerms := MockGDPRPerms{} + mockPerms.On("HostCookiesAllowed", mock.Anything, gdpr.SignalYes, "anyConsent").Return(test.givenResponse, test.givenError) + + privacy := usersyncPrivacy{ + gdprPermissions: &mockPerms, + gdprSignal: gdpr.SignalYes, + gdprConsent: "anyConsent", } - if val, err := jsonparser.GetString(value, "bidder"); err != nil { - t.Errorf("response.bidder_status[?].bidder was not a string. Value was %s", string(value)) - } else { - syncs = append(syncs, val) + + result := privacy.GDPRAllowsHostCookie() + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestUsersyncPrivacyGDPRAllowsBidderSync(t *testing.T) { + testCases := []struct { + description string + givenResponse bool + givenError error + expected bool + }{ + { + description: "Allowed - No Error", + givenResponse: true, + givenError: nil, + expected: true, + }, + { + description: "Allowed - Error", + givenResponse: true, + givenError: errors.New("anyError"), + expected: false, + }, + { + description: "Not Allowed - No Error", + givenResponse: false, + givenError: nil, + expected: false, + }, + { + description: "Not Allowed - Error", + givenResponse: false, + givenError: errors.New("anyError"), + expected: false, + }, + } + + for _, test := range testCases { + mockPerms := MockGDPRPerms{} + mockPerms.On("BidderSyncAllowed", mock.Anything, openrtb_ext.BidderName("foo"), gdpr.SignalYes, "anyConsent").Return(test.givenResponse, test.givenError) + + privacy := usersyncPrivacy{ + gdprPermissions: &mockPerms, + gdprSignal: gdpr.SignalYes, + gdprConsent: "anyConsent", } - }, "bidder_status") - return syncs + + result := privacy.GDPRAllowsBidderSync("foo") + assert.Equal(t, test.expected, result, test.description) + } } -func mockPermissions(allowHost bool, allowedBidders map[openrtb_ext.BidderName]usersync.Usersyncer) gdpr.Permissions { - return &gdprPerms{ - allowHost: allowHost, - allowedBidders: allowedBidders, +func TestUsersyncPrivacyCCPAAllowsBidderSync(t *testing.T) { + testCases := []struct { + description string + givenConsent string + expected bool + }{ + { + description: "Allowed", + givenConsent: "1YYY", + expected: true, + }, + { + description: "Not Allowed", + givenConsent: "1NNN", + expected: false, + }, } + + for _, test := range testCases { + validBidders := map[string]struct{}{"foo": {}} + parsedPolicy, err := ccpa.Policy{Consent: test.givenConsent}.Parse(validBidders) + + if assert.NoError(t, err) { + privacy := usersyncPrivacy{ccpaParsedPolicy: parsedPolicy} + result := privacy.CCPAAllowsBidderSync("foo") + assert.Equal(t, test.expected, result, test.description) + } + } +} + +type FakeChooser struct { + Result usersync.Result +} + +func (c FakeChooser) Choose(request usersync.Request, cookie *usersync.Cookie) usersync.Result { + return c.Result +} + +type MockSyncer struct { + mock.Mock +} + +func (m *MockSyncer) Key() string { + args := m.Called() + return args.String(0) +} + +func (m *MockSyncer) SupportsType(syncTypes []usersync.SyncType) bool { + args := m.Called(syncTypes) + return args.Bool(0) +} + +func (m *MockSyncer) GetSync(syncTypes []usersync.SyncType, privacyPolicies privacy.Policies) (usersync.Sync, error) { + args := m.Called(syncTypes, privacyPolicies) + return args.Get(0).(usersync.Sync), args.Error(1) +} + +type MockAnalytics struct { + mock.Mock +} + +func (m *MockAnalytics) LogAuctionObject(obj *analytics.AuctionObject) { + m.Called(obj) +} + +func (m *MockAnalytics) LogVideoObject(obj *analytics.VideoObject) { + m.Called(obj) +} + +func (m *MockAnalytics) LogCookieSyncObject(obj *analytics.CookieSyncObject) { + m.Called(obj) +} + +func (m *MockAnalytics) LogSetUIDObject(obj *analytics.SetUIDObject) { + m.Called(obj) +} + +func (m *MockAnalytics) LogAmpObject(obj *analytics.AmpObject) { + m.Called(obj) +} + +func (m *MockAnalytics) LogNotificationEventObject(obj *analytics.NotificationEvent) { + m.Called(obj) } -type gdprPerms struct { - allowHost bool - allowedBidders map[openrtb_ext.BidderName]usersync.Usersyncer +type MockGDPRPerms struct { + mock.Mock } -func (g *gdprPerms) HostCookiesAllowed(ctx context.Context, gdprSignal gdpr.Signal, consent string) (bool, error) { - return g.allowHost, nil +func (m *MockGDPRPerms) HostCookiesAllowed(ctx context.Context, gdprSignal gdpr.Signal, consent string) (bool, error) { + args := m.Called(ctx, gdprSignal, consent) + return args.Bool(0), args.Error(1) } -func (g *gdprPerms) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, gdprSignal gdpr.Signal, consent string) (bool, error) { - _, ok := g.allowedBidders[bidder] - return ok, nil +func (m *MockGDPRPerms) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, gdprSignal gdpr.Signal, consent string) (bool, error) { + args := m.Called(ctx, bidder, gdprSignal, consent) + return args.Bool(0), args.Error(1) } -func (g *gdprPerms) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, gdprSignal gdpr.Signal, consent string) (bool, bool, bool, error) { - return true, true, true, nil +func (m *MockGDPRPerms) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, gdprSignal gdpr.Signal, consent string) (bool, bool, bool, error) { + args := m.Called(ctx, bidder, PublisherID, gdprSignal, consent) + return args.Bool(0), args.Bool(1), args.Bool(2), args.Error(3) } diff --git a/endpoints/httprouterhandler.go b/endpoints/httprouterhandler.go new file mode 100644 index 00000000000..118b1c31b06 --- /dev/null +++ b/endpoints/httprouterhandler.go @@ -0,0 +1,11 @@ +package endpoints + +import ( + "net/http" + + "github.com/julienschmidt/httprouter" +) + +type HTTPRouterHandler interface { + Handle(http.ResponseWriter, *http.Request, httprouter.Params) +} diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 0074ed63e3f..461dbf9a0ca 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -26,12 +26,12 @@ const ( chromeiOSStrLen = len(chromeiOSStr) ) -func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[openrtb_ext.BidderName]usersync.Usersyncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { +func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour - validFamilyNameMap := make(map[string]struct{}) + validKeyLookup := make(map[string]struct{}) for _, s := range syncers { - validFamilyNameMap[s.FamilyName()] = struct{}{} + validKeyLookup[s.Key()] = struct{}{} } return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -54,7 +54,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[openrtb_ext.BidderName query := r.URL.Query() - familyName, err := getFamilyName(query, validFamilyNameMap) + familyName, err := getFamilyName(query, validKeyLookup) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 6851b582ce6..db80f6ff97e 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -399,9 +399,9 @@ func doRequest(req *http.Request, metrics metrics.MetricsEngine, validFamilyName allowPI: true, } analytics := analyticsConf.NewPBSAnalytics(&cfg.Analytics) - syncers := make(map[openrtb_ext.BidderName]usersync.Usersyncer) + syncers := make(map[string]usersync.Syncer) for _, name := range validFamilyNames { - syncers[openrtb_ext.BidderName(name)] = newFakeSyncer(name) + syncers[name] = newFakeSyncer(name) } endpoint := NewSetUIDEndpoint(cfg.HostCookie, syncers, perms, analytics, metrics) @@ -448,22 +448,26 @@ func (g *mockPermsSetUID) PersonalInfoAllowed(ctx context.Context, bidder openrt return g.allowPI, g.allowPI, g.allowPI, nil } -func newFakeSyncer(familyName string) usersync.Usersyncer { +func newFakeSyncer(key string) usersync.Syncer { return fakeSyncer{ - familyName: familyName, + key: key, } } type fakeSyncer struct { - familyName string + key string } // FamilyNames implements the Usersyncer interface. -func (s fakeSyncer) FamilyName() string { - return s.familyName +func (s fakeSyncer) Key() string { + return s.key +} + +func (s fakeSyncer) SupportsType(syncTypes []usersync.SyncType) bool { + return true } // GetUsersyncInfo implements the Usersyncer interface with a no-op. -func (s fakeSyncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.UsersyncInfo, error) { - return nil, nil +func (s fakeSyncer) GetSync(syncTypes []usersync.SyncType, privacyPolicies privacy.Policies) (usersync.Sync, error) { + return usersync.Sync{}, nil } diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 0f0ae871a4e..be2ca0dfd1e 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -729,7 +729,8 @@ func TestGetBidCacheInfoEndToEnd(t *testing.T) { } adapterList := make([]openrtb_ext.BidderName, 0, 2) - testEngine := metricsConf.NewMetricsEngine(cfg, adapterList) + syncerKeys := []string{} + testEngine := metricsConf.NewMetricsEngine(cfg, adapterList, syncerKeys) /* 2) Init new exchange with said configuration */ handlerNoBidServer := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) } server := httptest.NewServer(http.HandlerFunc(handlerNoBidServer)) @@ -1763,7 +1764,7 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string] return &exchange{ adapterMap: bidderAdapters, - me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames()), + me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames(), []string{}), cache: &wellBehavedCache{}, cacheTime: 0, gDPR: gdpr.AlwaysFail{}, diff --git a/exchange/legacy_test.go b/exchange/legacy_test.go index 2cef4feae40..0cf34b4b619 100644 --- a/exchange/legacy_test.go +++ b/exchange/legacy_test.go @@ -16,7 +16,6 @@ import ( "github.com/prebid/prebid-server/currency" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbs" - "github.com/prebid/prebid-server/usersync" ) func TestSiteVideo(t *testing.T) { @@ -494,10 +493,6 @@ func (a *mockLegacyAdapter) SkipNoCookies() bool { return false } -func (a *mockLegacyAdapter) GetUsersyncInfo() (*usersync.UsersyncInfo, error) { - return nil, nil -} - func (a *mockLegacyAdapter) Call(ctx context.Context, req *pbs.PBSRequest, bidder *pbs.PBSBidder) (pbs.PBSBidSlice, error) { a.gotRequest = req a.gotBidder = bidder diff --git a/gdpr/gdpr.go b/gdpr/gdpr.go index 36dac06c0ab..bb947392e9e 100644 --- a/gdpr/gdpr.go +++ b/gdpr/gdpr.go @@ -3,11 +3,9 @@ package gdpr import ( "context" "net/http" - "strconv" "github.com/prebid/go-gdpr/vendorlist" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" ) @@ -67,21 +65,3 @@ type ErrorMalformedConsent struct { func (e *ErrorMalformedConsent) Error() string { return "malformed consent string " + e.consent + ": " + e.cause.Error() } - -// SignalParse parses a raw signal and returns -func SignalParse(rawSignal string) (Signal, error) { - if rawSignal == "" { - return SignalAmbiguous, nil - } - - i, err := strconv.Atoi(rawSignal) - - if err != nil { - return SignalAmbiguous, err - } - if i != 0 && i != 1 { - return SignalAmbiguous, &errortypes.BadInput{Message: "GDPR signal should be integer 0 or 1"} - } - - return Signal(i), nil -} diff --git a/gdpr/gdpr_test.go b/gdpr/gdpr_test.go index 5048cf118f5..9e4f0940f03 100644 --- a/gdpr/gdpr_test.go +++ b/gdpr/gdpr_test.go @@ -7,7 +7,6 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/stretchr/testify/assert" ) @@ -48,55 +47,3 @@ func TestNewPermissions(t *testing.T) { assert.IsType(t, tt.wantType, perms, tt.description) } } - -func TestSignalParse(t *testing.T) { - tests := []struct { - description string - rawSignal string - wantSignal Signal - wantError bool - }{ - { - description: "valid raw signal is 0", - rawSignal: "0", - wantSignal: SignalNo, - wantError: false, - }, - { - description: "Valid signal - raw signal is 1", - rawSignal: "1", - wantSignal: SignalYes, - wantError: false, - }, - { - description: "Valid signal - raw signal is empty", - rawSignal: "", - wantSignal: SignalAmbiguous, - wantError: false, - }, - { - description: "Invalid signal - raw signal is -1", - rawSignal: "-1", - wantSignal: SignalAmbiguous, - wantError: true, - }, - { - description: "Invalid signal - raw signal is abc", - rawSignal: "abc", - wantSignal: SignalAmbiguous, - wantError: true, - }, - } - - for _, tt := range tests { - signal, err := SignalParse(tt.rawSignal) - - assert.Equal(t, tt.wantSignal, signal, tt.description) - - if tt.wantError { - assert.NotNil(t, err, tt.description) - } else { - assert.Nil(t, err, tt.description) - } - } -} diff --git a/gdpr/impl.go b/gdpr/impl.go index 5b1349ffe60..095b88cd7aa 100644 --- a/gdpr/impl.go +++ b/gdpr/impl.go @@ -14,19 +14,6 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) -// This file implements GDPR permissions for the app. -// For more info, see https://github.com/prebid/prebid-server/issues/501 -// -// Nothing in this file is exported. Public APIs can be found in gdpr.go - -type Signal int - -const ( - SignalAmbiguous Signal = -1 - SignalNo Signal = 0 - SignalYes Signal = 1 -) - type permissionsImpl struct { cfg config.GDPR vendorIDs map[openrtb_ext.BidderName]uint16 @@ -34,7 +21,7 @@ type permissionsImpl struct { } func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, gdprSignal Signal, consent string) (bool, error) { - gdprSignal = p.normalizeGDPR(gdprSignal) + gdprSignal = SignalNormalize(gdprSignal, p.cfg) if gdprSignal == SignalNo { return true, nil @@ -44,7 +31,7 @@ func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, gdprSignal Sig } func (p *permissionsImpl) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, gdprSignal Signal, consent string) (bool, error) { - gdprSignal = p.normalizeGDPR(gdprSignal) + gdprSignal = SignalNormalize(gdprSignal, p.cfg) if gdprSignal == SignalNo { return true, nil @@ -63,7 +50,7 @@ func (p *permissionsImpl) PersonalInfoAllowed(ctx context.Context, bidder openrt return true, true, true, nil } - gdprSignal = p.normalizeGDPR(gdprSignal) + gdprSignal = SignalNormalize(gdprSignal, p.cfg) if gdprSignal == SignalNo { return true, true, true, nil @@ -84,18 +71,6 @@ func (p *permissionsImpl) defaultVendorPermissions() (allowPI bool, allowGeo boo return false, false, false, nil } -func (p *permissionsImpl) normalizeGDPR(gdprSignal Signal) Signal { - if gdprSignal != SignalAmbiguous { - return gdprSignal - } - - if p.cfg.UsersyncIfAmbiguous { - return SignalNo - } - - return SignalYes -} - func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, consent string) (bool, error) { if consent == "" { diff --git a/gdpr/impl_test.go b/gdpr/impl_test.go index 737ed14a300..4510c7576a7 100644 --- a/gdpr/impl_test.go +++ b/gdpr/impl_test.go @@ -762,61 +762,3 @@ func assertStringsEqual(t *testing.T, expected string, actual string) { t.Errorf("Expected %s, got %s", expected, actual) } } - -func TestNormalizeGDPR(t *testing.T) { - tests := []struct { - description string - userSyncIfAmbiguous bool - giveSignal Signal - wantSignal Signal - }{ - { - description: "Don't normalize - Signal No and userSyncIfAmbiguous false", - userSyncIfAmbiguous: false, - giveSignal: SignalNo, - wantSignal: SignalNo, - }, - { - description: "Don't normalize - Signal No and userSyncIfAmbiguous true", - userSyncIfAmbiguous: true, - giveSignal: SignalNo, - wantSignal: SignalNo, - }, - { - description: "Don't normalize - Signal Yes and userSyncIfAmbiguous false", - userSyncIfAmbiguous: false, - giveSignal: SignalYes, - wantSignal: SignalYes, - }, - { - description: "Don't normalize - Signal Yes and userSyncIfAmbiguous true", - userSyncIfAmbiguous: true, - giveSignal: SignalYes, - wantSignal: SignalYes, - }, - { - description: "Normalize - Signal Ambiguous and userSyncIfAmbiguous false", - userSyncIfAmbiguous: false, - giveSignal: SignalAmbiguous, - wantSignal: SignalYes, - }, - { - description: "Normalize - Signal Ambiguous and userSyncIfAmbiguous true", - userSyncIfAmbiguous: true, - giveSignal: SignalAmbiguous, - wantSignal: SignalNo, - }, - } - - for _, tt := range tests { - perms := permissionsImpl{ - cfg: config.GDPR{ - UsersyncIfAmbiguous: tt.userSyncIfAmbiguous, - }, - } - - normalizedSignal := perms.normalizeGDPR(tt.giveSignal) - - assert.Equal(t, tt.wantSignal, normalizedSignal, tt.description) - } -} diff --git a/gdpr/signal.go b/gdpr/signal.go new file mode 100644 index 00000000000..8bfe66a50d5 --- /dev/null +++ b/gdpr/signal.go @@ -0,0 +1,46 @@ +package gdpr + +import ( + "strconv" + + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" +) + +type Signal int + +const ( + SignalAmbiguous Signal = -1 + SignalNo Signal = 0 + SignalYes Signal = 1 +) + +var gdprSignalError = &errortypes.BadInput{Message: "GDPR signal should be integer 0 or 1"} + +// SignalParse returns a parsed GDPR signal or a parse error. +func SignalParse(rawSignal string) (Signal, error) { + if rawSignal == "" { + return SignalAmbiguous, nil + } + + i, err := strconv.Atoi(rawSignal) + + if err != nil || (i != 0 && i != 1) { + return SignalAmbiguous, gdprSignalError + } + + return Signal(i), nil +} + +// SignalNormalize normalizes a GDPR signal to ensure it's always either SignalYes or SignalNo. +func SignalNormalize(signal Signal, config config.GDPR) Signal { + if signal != SignalAmbiguous { + return signal + } + + if config.UsersyncIfAmbiguous { + return SignalNo + } + + return SignalYes +} diff --git a/gdpr/signal_test.go b/gdpr/signal_test.go new file mode 100644 index 00000000000..2c70d845f40 --- /dev/null +++ b/gdpr/signal_test.go @@ -0,0 +1,116 @@ +package gdpr + +import ( + "testing" + + "github.com/prebid/prebid-server/config" + "github.com/stretchr/testify/assert" +) + +func TestSignalParse(t *testing.T) { + tests := []struct { + description string + rawSignal string + wantSignal Signal + wantError bool + }{ + { + description: "valid raw signal is 0", + rawSignal: "0", + wantSignal: SignalNo, + wantError: false, + }, + { + description: "Valid signal - raw signal is 1", + rawSignal: "1", + wantSignal: SignalYes, + wantError: false, + }, + { + description: "Valid signal - raw signal is empty", + rawSignal: "", + wantSignal: SignalAmbiguous, + wantError: false, + }, + { + description: "Invalid signal - raw signal is -1", + rawSignal: "-1", + wantSignal: SignalAmbiguous, + wantError: true, + }, + { + description: "Invalid signal - raw signal is abc", + rawSignal: "abc", + wantSignal: SignalAmbiguous, + wantError: true, + }, + } + + for _, test := range tests { + signal, err := SignalParse(test.rawSignal) + + assert.Equal(t, test.wantSignal, signal, test.description) + + if test.wantError { + assert.NotNil(t, err, test.description) + } else { + assert.Nil(t, err, test.description) + } + } +} + +func TestSignalNormalize(t *testing.T) { + tests := []struct { + description string + userSyncIfAmbiguous bool + giveSignal Signal + wantSignal Signal + }{ + { + description: "Don't normalize - Signal No and userSyncIfAmbiguous false", + userSyncIfAmbiguous: false, + giveSignal: SignalNo, + wantSignal: SignalNo, + }, + { + description: "Don't normalize - Signal No and userSyncIfAmbiguous true", + userSyncIfAmbiguous: true, + giveSignal: SignalNo, + wantSignal: SignalNo, + }, + { + description: "Don't normalize - Signal Yes and userSyncIfAmbiguous false", + userSyncIfAmbiguous: false, + giveSignal: SignalYes, + wantSignal: SignalYes, + }, + { + description: "Don't normalize - Signal Yes and userSyncIfAmbiguous true", + userSyncIfAmbiguous: true, + giveSignal: SignalYes, + wantSignal: SignalYes, + }, + { + description: "Normalize - Signal Ambiguous and userSyncIfAmbiguous false", + userSyncIfAmbiguous: false, + giveSignal: SignalAmbiguous, + wantSignal: SignalYes, + }, + { + description: "Normalize - Signal Ambiguous and userSyncIfAmbiguous true", + userSyncIfAmbiguous: true, + giveSignal: SignalAmbiguous, + wantSignal: SignalNo, + }, + } + + for _, test := range tests { + config := config.GDPR{ + UsersyncIfAmbiguous: test.userSyncIfAmbiguous, + } + + normalizedSignal := SignalNormalize(test.giveSignal, config) + + assert.Equal(t, test.wantSignal, normalizedSignal, test.description) + } +} diff --git a/macros/macros.go b/macros/macros.go index 5d6bd7af65e..4440f2e242d 100644 --- a/macros/macros.go +++ b/macros/macros.go @@ -23,7 +23,7 @@ type UserSyncTemplateParams struct { } // ResolveMacros resolves macros in the given template with the provided params -func ResolveMacros(aTemplate template.Template, params interface{}) (string, error) { +func ResolveMacros(aTemplate *template.Template, params interface{}) (string, error) { strBuf := bytes.Buffer{} err := aTemplate.Execute(&strBuf, params) diff --git a/macros/macros_test.go b/macros/macros_test.go index cd8c4b557b9..055624b611f 100644 --- a/macros/macros_test.go +++ b/macros/macros_test.go @@ -10,27 +10,37 @@ import ( const validEndpointTemplate = "http://{{.Host}}/publisher/{{.PublisherID}}" func TestResolveMacros(t *testing.T) { - endpointTemplate, _ := template.New("endpointTemplate").Parse(validEndpointTemplate) + endpointTemplate := template.Must(template.New("endpointTemplate").Parse(validEndpointTemplate)) testCases := []struct { - aTemplate template.Template - params interface{} - result string - hasError bool + givenTemplate *template.Template + givenParams interface{} + expectedResult string + expectedError bool }{ - {aTemplate: *endpointTemplate, params: EndpointTemplateParams{Host: "SomeHost", PublisherID: "1"}, result: "http://SomeHost/publisher/1", hasError: false}, - {aTemplate: *endpointTemplate, params: UserSyncTemplateParams{GDPR: "SomeGDPR", GDPRConsent: "SomeGDPRConsent"}, result: "", hasError: true}, + { + givenTemplate: endpointTemplate, + givenParams: EndpointTemplateParams{Host: "SomeHost", PublisherID: "1"}, + expectedResult: "http://SomeHost/publisher/1", + expectedError: false, + }, + { + givenTemplate: endpointTemplate, + givenParams: UserSyncTemplateParams{GDPR: "SomeGDPR", GDPRConsent: "SomeGDPRConsent"}, + expectedResult: "", + expectedError: true, + }, } for _, test := range testCases { - res, err := ResolveMacros(test.aTemplate, test.params) + result, err := ResolveMacros(test.givenTemplate, test.givenParams) - if test.hasError { + if test.expectedError { assert.NotNil(t, err, "Error shouldn't be nil") - assert.Empty(t, res, "Result should be empty") + assert.Empty(t, result, "Result should be empty") } else { assert.Nil(t, err, "Err should be nil") - assert.Equal(t, res, test.result, "String after resolving macros should be %s", test.result) + assert.Equal(t, result, test.expectedResult, "String after resolving macros should be %s", test.expectedResult) } } } diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index c1f726e904e..e68a9e1e7c1 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -13,7 +13,7 @@ import ( // NewMetricsEngine reads the configuration and returns the appropriate metrics engine // for this instance. -func NewMetricsEngine(cfg *config.Configuration, adapterList []openrtb_ext.BidderName) *DetailedMetricsEngine { +func NewMetricsEngine(cfg *config.Configuration, adapterList []openrtb_ext.BidderName, syncerKeys []string) *DetailedMetricsEngine { // Create a list of metrics engines to use. // Capacity of 2, as unlikely to have more than 2 metrics backends, and in the case // of 1 we won't use the list so it will be garbage collected. @@ -37,7 +37,7 @@ func NewMetricsEngine(cfg *config.Configuration, adapterList []openrtb_ext.Bidde } if cfg.Metrics.Prometheus.Port != 0 { // Set up the Prometheus metrics. - returnEngine.PrometheusMetrics = prometheusmetrics.NewMetrics(cfg.Metrics.Prometheus, cfg.Metrics.Disabled) + returnEngine.PrometheusMetrics = prometheusmetrics.NewMetrics(cfg.Metrics.Prometheus, cfg.Metrics.Disabled, syncerKeys) engineList = append(engineList, returnEngine.PrometheusMetrics) } @@ -175,9 +175,9 @@ func (me *MultiMetricsEngine) RecordAdapterTime(labels metrics.AdapterLabels, le } // RecordCookieSync across all engines -func (me *MultiMetricsEngine) RecordCookieSync() { +func (me *MultiMetricsEngine) RecordCookieSync(status metrics.CookieSyncStatus) { for _, thisME := range *me { - thisME.RecordCookieSync() + thisME.RecordCookieSync(status) } } @@ -202,10 +202,10 @@ func (me *MultiMetricsEngine) RecordAccountCacheResult(cacheResult metrics.Cache } } -// RecordAdapterCookieSync across all engines -func (me *MultiMetricsEngine) RecordAdapterCookieSync(adapter openrtb_ext.BidderName, gdprBlocked bool) { +// RecordSyncerRequest across all engines +func (me *MultiMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerStatus) { for _, thisME := range *me { - thisME.RecordAdapterCookieSync(adapter, gdprBlocked) + thisME.RecordSyncerRequest(key, status) } } @@ -312,11 +312,11 @@ func (me *DummyMetricsEngine) RecordAdapterTime(labels metrics.AdapterLabels, le } // RecordCookieSync as a noop -func (me *DummyMetricsEngine) RecordCookieSync() { +func (me *DummyMetricsEngine) RecordCookieSync(status metrics.CookieSyncStatus) { } -// RecordAdapterCookieSync as a noop -func (me *DummyMetricsEngine) RecordAdapterCookieSync(adapter openrtb_ext.BidderName, gdprBlocked bool) { +// RecordSyncerCRecordSyncerRequestookieSync as a noop +func (me *DummyMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerStatus) { } // RecordUserIDSet as a noop diff --git a/metrics/config/metrics_test.go b/metrics/config/metrics_test.go index 5b70b53bb1a..62fc7992415 100644 --- a/metrics/config/metrics_test.go +++ b/metrics/config/metrics_test.go @@ -14,7 +14,8 @@ import ( func TestDummyMetricsEngine(t *testing.T) { cfg := mainConfig.Configuration{} adapterList := make([]openrtb_ext.BidderName, 0, 2) - testEngine := NewMetricsEngine(&cfg, adapterList) + syncerKeys := []string{"keyA", "keyB"} + testEngine := NewMetricsEngine(&cfg, adapterList, syncerKeys) _, ok := testEngine.MetricsEngine.(*DummyMetricsEngine) if !ok { t.Error("Expected a DummyMetricsEngine, but didn't get it") @@ -25,7 +26,8 @@ func TestGoMetricsEngine(t *testing.T) { cfg := mainConfig.Configuration{} cfg.Metrics.Influxdb.Host = "localhost" adapterList := make([]openrtb_ext.BidderName, 0, 2) - testEngine := NewMetricsEngine(&cfg, adapterList) + syncerKeys := []string{"keyA", "keyB"} + testEngine := NewMetricsEngine(&cfg, adapterList, syncerKeys) _, ok := testEngine.MetricsEngine.(*metrics.Metrics) if !ok { t.Error("Expected a legacy Metrics as MetricsEngine, but didn't get it") diff --git a/metrics/config/todo b/metrics/config/todo new file mode 100644 index 00000000000..c6b4b2f2bed --- /dev/null +++ b/metrics/config/todo @@ -0,0 +1,6 @@ +todo + - add proper samesite behavior - cookie should just know what to do + - add new cookiesync metrics to go metrics + - load syncer from configs / handle parsing errors + - /setuid endpoint changes + - migrate all adapter user sync configs \ No newline at end of file diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index ac7ed0691c4..2621369bd9e 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -608,16 +608,17 @@ func (me *Metrics) RecordAdapterTime(labels AdapterLabels, length time.Duration) } // RecordCookieSync implements a part of the MetricsEngine interface. Records a cookie sync request -func (me *Metrics) RecordCookieSync() { +func (me *Metrics) RecordCookieSync(status CookieSyncStatus) { + // TODO: add influx db metrics me.CookieSyncMeter.Mark(1) } -// RecordAdapterCookieSync implements a part of the MetricsEngine interface. Records a cookie sync adpter sync request and gdpr status -func (me *Metrics) RecordAdapterCookieSync(adapter openrtb_ext.BidderName, gdprBlocked bool) { - me.CookieSyncGen[adapter].Mark(1) - if gdprBlocked { - me.CookieSyncGDPRPrevent[adapter].Mark(1) - } +// RecordSyncerRequest implements a part of the MetricsEngine interface. Records a cookie sync syncer request and status +func (me *Metrics) RecordSyncerRequest(key string, status SyncerStatus) { + // me.CookieSyncGen[adapter].Mark(1) + // if blockedByPrivacy { + // me.CookieSyncGDPRPrevent[adapter].Mark(1) + // } } // RecordUserIDSet implements a part of the MetricsEngine interface. Records a cookie setuid request diff --git a/metrics/metrics.go b/metrics/metrics.go index 62de3afac21..cda780ab9e2 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -323,6 +323,44 @@ func TCFVersionToValue(version int) TCFVersionValue { return TCFVersionErr } +type CookieSyncStatus string + +const ( + CookieSyncOK CookieSyncStatus = "ok" + CookieSyncBadRequest CookieSyncStatus = "bad_request" + CookieSyncOptOut CookieSyncStatus = "opt_out" + CookieSyncGDPRHostCookieBlocked CookieSyncStatus = "gdpr_blocked_host_cookie" +) + +// CookieSyncStatuses returns possible cookie sync statuses. +func CookieSyncStatuses() []CookieSyncStatus { + return []CookieSyncStatus{ + CookieSyncOK, + CookieSyncBadRequest, + CookieSyncOptOut, + CookieSyncGDPRHostCookieBlocked, + } +} + +type SyncerStatus string + +const ( + SyncerOK SyncerStatus = "ok" + SyncerPrivacyBlocked SyncerStatus = "privacy_blocked" + SyncerAlreadySynced SyncerStatus = "already_synced" + SyncerTypeNotSupported SyncerStatus = "type_not_supported" +) + +// SyncerStatuses returns possible syncer statuses. +func SyncerStatuses() []SyncerStatus { + return []SyncerStatus{ + SyncerOK, + SyncerPrivacyBlocked, + SyncerAlreadySynced, + SyncerTypeNotSupported, + } +} + // MetricsEngine is a generic interface to record PBS metrics into the desired backend // The first three metrics function fire off once per incoming request, so total metrics // will equal the total number of incoming requests. The remaining 5 fire off per outgoing @@ -346,8 +384,8 @@ type MetricsEngine interface { RecordAdapterBidReceived(labels AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) RecordAdapterPrice(labels AdapterLabels, cpm float64) RecordAdapterTime(labels AdapterLabels, length time.Duration) - RecordCookieSync() - RecordAdapterCookieSync(adapter openrtb_ext.BidderName, gdprBlocked bool) + RecordCookieSync(status CookieSyncStatus) + RecordSyncerRequest(key string, status SyncerStatus) RecordUserIDSet(userLabels UserLabels) // Function should verify bidder values RecordStoredReqCacheResult(cacheResult CacheResult, inc int) RecordStoredImpCacheResult(cacheResult CacheResult, inc int) diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index c852eb47d24..77768178a1a 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -92,13 +92,13 @@ func (me *MetricsEngineMock) RecordAdapterTime(labels AdapterLabels, length time } // RecordCookieSync mock -func (me *MetricsEngineMock) RecordCookieSync() { - me.Called() +func (me *MetricsEngineMock) RecordCookieSync(status CookieSyncStatus) { + me.Called(status) } -// RecordAdapterCookieSync mock -func (me *MetricsEngineMock) RecordAdapterCookieSync(adapter openrtb_ext.BidderName, gdprBlocked bool) { - me.Called(adapter, gdprBlocked) +// RecordSyncerRequest mock +func (me *MetricsEngineMock) RecordSyncerRequest(key string, status SyncerStatus) { + me.Called(key, status) } // RecordUserIDSet mock diff --git a/metrics/prometheus/preload.go b/metrics/prometheus/preload.go index f4dfe43469d..d33614b1bb8 100644 --- a/metrics/prometheus/preload.go +++ b/metrics/prometheus/preload.go @@ -5,7 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func preloadLabelValues(m *Metrics) { +func preloadLabelValues(m *Metrics, syncerKeys []string) { var ( actionValues = actionsAsString() adapterErrorValues = adapterErrorsAsString() @@ -15,10 +15,12 @@ func preloadLabelValues(m *Metrics) { cacheResultValues = cacheResultsAsString() connectionErrorValues = []string{connectionAcceptError, connectionCloseError} cookieValues = cookieTypesAsString() + cookieSyncStatusValues = cookieSyncStatusesAsString() requestStatusValues = requestStatusesAsString() requestTypeValues = requestTypesAsString() storedDataFetchTypeValues = storedDataFetchTypesAsString() storedDataErrorValues = storedDataErrorsAsString() + syncerStatusValues = syncerStatusesAsString() sourceValues = []string{sourceRequest} ) @@ -26,6 +28,10 @@ func preloadLabelValues(m *Metrics) { connectionErrorLabel: connectionErrorValues, }) + preloadLabelValuesForCounter(m.cookieSync, map[string][]string{ + cookiesyncStatusLabel: cookieSyncStatusValues, + }) + preloadLabelValuesForCounter(m.impressions, map[string][]string{ isBannerLabel: boolValues, isVideoLabel: boolValues, @@ -107,11 +113,6 @@ func preloadLabelValues(m *Metrics) { markupDeliveryLabel: bidTypeValues, }) - preloadLabelValuesForCounter(m.adapterCookieSync, map[string][]string{ - adapterLabel: adapterValues, - privacyBlockedLabel: boolValues, - }) - preloadLabelValuesForCounter(m.adapterErrors, map[string][]string{ adapterLabel: adapterValues, adapterErrorLabel: adapterErrorValues, @@ -154,6 +155,11 @@ func preloadLabelValues(m *Metrics) { actionLabel: actionValues, }) + preloadLabelValuesForCounter(m.syncerRequests, map[string][]string{ + syncerLabel: syncerKeys, + syncerStatusLabel: syncerStatusValues, + }) + //to minimize memory usage, queuedTimeout metric is now supported for video endpoint only //boolean value represents 2 general request statuses: accepted and rejected preloadLabelValuesForHistogram(m.requestsQueueTimer, map[string][]string{ diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 1e384c0e438..5a83496c8f5 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -18,7 +18,7 @@ type Metrics struct { connectionsClosed prometheus.Counter connectionsError *prometheus.CounterVec connectionsOpened prometheus.Counter - cookieSync prometheus.Counter + cookieSync *prometheus.CounterVec impressions *prometheus.CounterVec impressionsLegacy prometheus.Counter prebidCacheWriteTimer *prometheus.HistogramVec @@ -49,7 +49,6 @@ type Metrics struct { // Adapter Metrics adapterBids *prometheus.CounterVec - adapterCookieSync *prometheus.CounterVec adapterErrors *prometheus.CounterVec adapterPanics *prometheus.CounterVec adapterPrices *prometheus.HistogramVec @@ -60,6 +59,9 @@ type Metrics struct { adapterCreatedConnections *prometheus.CounterVec adapterConnectionWaitTime *prometheus.HistogramVec + // Syncer Metrics + syncerRequests *prometheus.CounterVec + // Account Metrics accountRequests *prometheus.CounterVec @@ -67,26 +69,29 @@ type Metrics struct { } const ( - accountLabel = "account" - actionLabel = "action" - adapterErrorLabel = "adapter_error" - adapterLabel = "adapter" - bidTypeLabel = "bid_type" - cacheResultLabel = "cache_result" - connectionErrorLabel = "connection_error" - cookieLabel = "cookie" - hasBidsLabel = "has_bids" - isAudioLabel = "audio" - isBannerLabel = "banner" - isNativeLabel = "native" - isVideoLabel = "video" - markupDeliveryLabel = "delivery" - optOutLabel = "opt_out" - privacyBlockedLabel = "privacy_blocked" - requestStatusLabel = "request_status" - requestTypeLabel = "request_type" - successLabel = "success" - versionLabel = "version" + accountLabel = "account" + actionLabel = "action" + adapterErrorLabel = "adapter_error" + adapterLabel = "adapter" + bidTypeLabel = "bid_type" + cacheResultLabel = "cache_result" + connectionErrorLabel = "connection_error" + cookieLabel = "cookie" + cookiesyncStatusLabel = "status" + hasBidsLabel = "has_bids" + isAudioLabel = "audio" + isBannerLabel = "banner" + isNativeLabel = "native" + isVideoLabel = "video" + markupDeliveryLabel = "delivery" + optOutLabel = "opt_out" + privacyBlockedLabel = "privacy_blocked" + requestStatusLabel = "request_status" + requestTypeLabel = "request_type" + successLabel = "success" + syncerLabel = "syncer" + syncerStatusLabel = "status" + versionLabel = "version" ) const ( @@ -120,7 +125,7 @@ const ( ) // NewMetrics initializes a new Prometheus metrics instance with preloaded label values. -func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMetrics) *Metrics { +func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMetrics, syncerKeys []string) *Metrics { standardTimeBuckets := []float64{0.05, 0.1, 0.15, 0.20, 0.25, 0.3, 0.4, 0.5, 0.75, 1} cacheWriteTimeBuckets := []float64{0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1} priceBuckets := []float64{250, 500, 750, 1000, 1500, 2000, 2500, 3000, 3500, 4000} @@ -143,9 +148,10 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet "connections_opened", "Count of successful connections opened to Prebid Server.") - metrics.cookieSync = newCounterWithoutLabels(cfg, metrics.Registry, + metrics.cookieSync = newCounter(cfg, metrics.Registry, "cookie_sync_requests", - "Count of cookie sync requests to Prebid Server.") + "Count of cookie sync requests to Prebid Server.", + []string{cookiesyncStatusLabel}) metrics.impressions = newCounter(cfg, metrics.Registry, "impressions_requests", @@ -288,11 +294,6 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet "Count of bids labeled by adapter and markup delivery type (adm or nurl).", []string{adapterLabel, markupDeliveryLabel}) - metrics.adapterCookieSync = newCounter(cfg, metrics.Registry, - "adapter_cookie_sync", - "Count of cookie sync requests received labeled by adapter and if the sync was blocked due to privacy regulation (GDPR, CCPA, etc...).", - []string{adapterLabel, privacyBlockedLabel}) - metrics.adapterErrors = newCounter(cfg, metrics.Registry, "adapter_errors", "Count of errors labeled by adapter and error type.", @@ -343,6 +344,11 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet "Count of user ID sync requests received labeled by adapter and action.", []string{adapterLabel, actionLabel}) + metrics.syncerRequests = newCounter(cfg, metrics.Registry, + "syncer_requests", + "Count of cookie sync requests received labeled by syncer key and status.", + []string{syncerLabel, syncerStatusLabel}) + metrics.accountRequests = newCounter(cfg, metrics.Registry, "account_requests", "Count of total requests to Prebid Server labeled by account.", @@ -354,7 +360,7 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet []string{requestTypeLabel, requestStatusLabel}, queuedRequestTimeBuckets) - preloadLabelValues(&metrics) + preloadLabelValues(&metrics, syncerKeys) return &metrics } @@ -596,14 +602,16 @@ func (m *Metrics) RecordAdapterTime(labels metrics.AdapterLabels, length time.Du } } -func (m *Metrics) RecordCookieSync() { - m.cookieSync.Inc() +func (m *Metrics) RecordCookieSync(status metrics.CookieSyncStatus) { + m.cookieSync.With(prometheus.Labels{ + cookiesyncStatusLabel: string(status), + }).Inc() } -func (m *Metrics) RecordAdapterCookieSync(adapter openrtb_ext.BidderName, privacyBlocked bool) { - m.adapterCookieSync.With(prometheus.Labels{ - adapterLabel: string(adapter), - privacyBlockedLabel: strconv.FormatBool(privacyBlocked), +func (m *Metrics) RecordSyncerRequest(key string, status metrics.SyncerStatus) { + m.syncerRequests.With(prometheus.Labels{ + syncerLabel: key, + syncerStatusLabel: string(status), }).Inc() } diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index b2ffff34850..71b7d646448 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -14,11 +14,12 @@ import ( ) func createMetricsForTesting() *Metrics { + syncerKeys := []string{} return NewMetrics(config.PrometheusMetrics{ Port: 8080, Namespace: "prebid", Subsystem: "server", - }, config.DisabledMetrics{}) + }, config.DisabledMetrics{}, syncerKeys) } func TestMetricCountGatekeeping(t *testing.T) { @@ -922,19 +923,18 @@ func TestAdapterTimeMetric(t *testing.T) { } } -func TestAdapterCookieSyncMetric(t *testing.T) { +func TestSyncerRequests(t *testing.T) { m := createMetricsForTesting() - adapterName := "anyName" - privacyBlocked := true + syncerKey := "anyKey" + syncerStatus := metrics.SyncerAlreadySynced - m.RecordAdapterCookieSync(openrtb_ext.BidderName(adapterName), privacyBlocked) + m.RecordSyncerRequest(syncerKey, syncerStatus) - expectedCount := float64(1) - assertCounterVecValue(t, "", "adapterCookieSync", m.adapterCookieSync, - expectedCount, + assertCounterVecValue(t, "", "syncer_requests", m.syncerRequests, + float64(1), prometheus.Labels{ - adapterLabel: adapterName, - privacyBlockedLabel: "true", + syncerLabel: syncerKey, + syncerStatusLabel: string(syncerStatus), }) } @@ -1051,13 +1051,40 @@ func TestAccountCacheResultMetric(t *testing.T) { } func TestCookieMetric(t *testing.T) { - m := createMetricsForTesting() + tests := []struct { + status metrics.CookieSyncStatus + label string + }{ + { + status: metrics.CookieSyncOK, + label: "ok", + }, + { + status: metrics.CookieSyncBadRequest, + label: "bad_request", + }, + { + status: metrics.CookieSyncOptOut, + label: "opt_out", + }, + { + status: metrics.CookieSyncGDPRHostCookieBlocked, + label: "gdpr_blocked_host_cookie", + }, + } - m.RecordCookieSync() + for _, test := range tests { + m := createMetricsForTesting() + + m.RecordCookieSync(test.status) + + assertCounterVecValue(t, "", "cookie_sync_requests:"+test.label, m.cookieSync, + float64(1), + prometheus.Labels{ + cookiesyncStatusLabel: string(test.status), + }) + } - expectedCount := float64(1) - assertCounterValue(t, "", "cookieSync", m.cookieSync, - expectedCount) } func TestPrebidCacheRequestTimeMetric(t *testing.T) { @@ -1342,11 +1369,12 @@ func TestRecordAdapterConnections(t *testing.T) { } func TestDisableAdapterConnections(t *testing.T) { + syncerKeys := []string{} prometheusMetrics := NewMetrics(config.PrometheusMetrics{ Port: 8080, Namespace: "prebid", Subsystem: "server", - }, config.DisabledMetrics{AdapterConnectionMetrics: true}) + }, config.DisabledMetrics{AdapterConnectionMetrics: true}, syncerKeys) // Assert counter vector was not initialized assert.Nil(t, prometheusMetrics.adapterReusedConnections, "Counter Vector adapterReusedConnections should be nil") diff --git a/metrics/prometheus/type_conversion.go b/metrics/prometheus/type_conversion.go index 0e5c80636db..2a43e991c1a 100644 --- a/metrics/prometheus/type_conversion.go +++ b/metrics/prometheus/type_conversion.go @@ -50,6 +50,24 @@ func cookieTypesAsString() []string { return valuesAsString } +func cookieSyncStatusesAsString() []string { + values := metrics.CookieSyncStatuses() + valuesAsString := make([]string, len(values)) + for i, v := range values { + valuesAsString[i] = string(v) + } + return valuesAsString +} + +func syncerStatusesAsString() []string { + values := metrics.SyncerStatuses() + valuesAsString := make([]string, len(values)) + for i, v := range values { + valuesAsString[i] = string(v) + } + return valuesAsString +} + func cacheResultsAsString() []string { values := metrics.CacheResults() valuesAsString := make([]string, len(values)) diff --git a/pbs/pbsrequest.go b/pbs/pbsrequest.go index 8da6c2f51a8..42d1259e668 100644 --- a/pbs/pbsrequest.go +++ b/pbs/pbsrequest.go @@ -121,19 +121,25 @@ type SDK struct { } type PBSBidder struct { - BidderCode string `json:"bidder"` - AdUnitCode string `json:"ad_unit,omitempty"` // for index to dedup responses - ResponseTime int `json:"response_time_ms,omitempty"` - NumBids int `json:"num_bids,omitempty"` - Error string `json:"error,omitempty"` - NoCookie bool `json:"no_cookie,omitempty"` - NoBid bool `json:"no_bid,omitempty"` - UsersyncInfo *usersync.UsersyncInfo `json:"usersync,omitempty"` - Debug []*BidderDebug `json:"debug,omitempty"` + BidderCode string `json:"bidder"` + AdUnitCode string `json:"ad_unit,omitempty"` // for index to dedup responses + ResponseTime int `json:"response_time_ms,omitempty"` + NumBids int `json:"num_bids,omitempty"` + Error string `json:"error,omitempty"` + NoCookie bool `json:"no_cookie,omitempty"` + NoBid bool `json:"no_bid,omitempty"` + UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` + Debug []*BidderDebug `json:"debug,omitempty"` AdUnits []PBSAdUnit `json:"-"` } +type UsersyncInfo struct { + URL string `json:"url,omitempty"` + Type string `json:"type,omitempty"` + SupportCORS bool `json:"supportCORS,omitempty"` +} + func (bidder *PBSBidder) LookupBidID(Code string) string { for _, unit := range bidder.AdUnits { if unit.Code == Code { @@ -217,7 +223,7 @@ func ParseMediaTypes(types []string) []MediaType { return mtypes } -var ipv4Validator iputil.IPValidator = iputil.VersionIPValidator{iputil.IPv4} +var ipv4Validator iputil.IPValidator = iputil.VersionIPValidator{Version: iputil.IPv4} func ParsePBSRequest(r *http.Request, cfg *config.AuctionTimeouts, cache cache.Cache, hostCookieConfig *config.HostCookie) (*PBSRequest, error) { defer r.Body.Close() diff --git a/router/router.go b/router/router.go index 0920f1d61b6..dbcff2d974a 100644 --- a/router/router.go +++ b/router/router.go @@ -12,12 +12,6 @@ import ( "strings" "time" - "github.com/prebid/prebid-server/currency" - "github.com/prebid/prebid-server/endpoints/events" - "github.com/prebid/prebid-server/errortypes" - - "github.com/prebid/prebid-server/metrics" - "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/adapters/adform" "github.com/prebid/prebid-server/adapters/appnexus" @@ -34,11 +28,15 @@ import ( "github.com/prebid/prebid-server/cache/filecache" "github.com/prebid/prebid-server/cache/postgrescache" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/currency" "github.com/prebid/prebid-server/endpoints" + "github.com/prebid/prebid-server/endpoints/events" infoEndpoints "github.com/prebid/prebid-server/endpoints/info" "github.com/prebid/prebid-server/endpoints/openrtb2" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/gdpr" + "github.com/prebid/prebid-server/metrics" metricsConf "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbs" @@ -46,7 +44,7 @@ import ( "github.com/prebid/prebid-server/router/aspects" "github.com/prebid/prebid-server/server/ssl" storedRequestsConf "github.com/prebid/prebid-server/stored_requests/config" - "github.com/prebid/prebid-server/usersync/usersyncers" + "github.com/prebid/prebid-server/usersync" "github.com/golang/glog" "github.com/julienschmidt/httprouter" @@ -215,7 +213,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R legacyBidderList = append(legacyBidderList, openrtb_ext.BidderName("districtm")) // Metrics engine - r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList) + r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList, nil) // todo: write up syncers db, shutdown, fetcher, ampFetcher, accounts, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) // todo(zachbadgett): better shutdown r.Shutdown = shutdown @@ -244,7 +242,8 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } - syncers := usersyncers.NewSyncerMap(cfg) + //todo: syncers := usersyncers.NewSyncerMap(cfg) + syncers := make(map[string]usersync.Syncer) gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, gvlVendorIDs, generalHttpClient) @@ -286,7 +285,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R r.GET("/info/bidders", infoEndpoints.NewBiddersEndpoint(bidderInfos, defaultAliases)) r.GET("/info/bidders/:bidderName", infoEndpoints.NewBiddersDetailEndpoint(bidderInfos, cfg.Adapters, defaultAliases)) r.GET("/bidders/params", NewJsonDirectoryServer(schemaDirectory, paramsValidator, defaultAliases)) - r.POST("/cookie_sync", endpoints.NewCookieSyncEndpoint(syncers, cfg, gdprPerms, r.MetricsEngine, pbsAnalytics, activeBidders)) + r.POST("/cookie_sync", endpoints.NewCookieSyncEndpoint(syncers, cfg, gdprPerms, r.MetricsEngine, pbsAnalytics, activeBidders).Handle) r.GET("/status", endpoints.NewStatusEndpoint(cfg.StatusResponse)) r.GET("/", serveIndex) r.ServeFiles("/static/*filepath", http.Dir("static")) diff --git a/usersync/bidderchooser.go b/usersync/bidderchooser.go index 3dac0625ec8..5e5ea81edf3 100644 --- a/usersync/bidderchooser.go +++ b/usersync/bidderchooser.go @@ -1,18 +1,18 @@ package usersync -import "github.com/prebid/prebid-server/config" - +// bidderChooser determines which bidders to consider for user syncing. type bidderChooser interface { // choose returns an ordered collection of potentially non-unique bidders. - choose(requested, available []string, cooperative config.UserSyncCooperative) []string + choose(requested, available []string, cooperative Cooperative) []string } +// standardBidderChooser implements the bidder choosing algorithm per official Prebid specification. type standardBidderChooser struct { shuffler shuffler } -func (c standardBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { - if requested == nil { +func (c standardBidderChooser) choose(requested, available []string, cooperative Cooperative) []string { + if len(requested) == 0 { return c.shuffledCopy(available) } diff --git a/usersync/bidderchooser_test.go b/usersync/bidderchooser_test.go index cb636586960..da598c6c526 100644 --- a/usersync/bidderchooser_test.go +++ b/usersync/bidderchooser_test.go @@ -3,7 +3,6 @@ package usersync import ( "testing" - "github.com/prebid/prebid-server/config" "github.com/stretchr/testify/assert" ) @@ -18,37 +17,37 @@ func TestBidderChooserChoose(t *testing.T) { testCases := []struct { description string givenRequested []string - givenCooperative config.UserSyncCooperative + givenCooperative Cooperative expected []string }{ { description: "No Coop - Nil", givenRequested: nil, - givenCooperative: config.UserSyncCooperative{Enabled: false}, + givenCooperative: Cooperative{Enabled: false}, expected: []string{"a2", "a1"}, }, { description: "No Coop - Empty", givenRequested: []string{}, - givenCooperative: config.UserSyncCooperative{Enabled: false}, - expected: []string{}, + givenCooperative: Cooperative{Enabled: false}, + expected: []string{"a2", "a1"}, }, { description: "No Coop - One", givenRequested: []string{"r"}, - givenCooperative: config.UserSyncCooperative{Enabled: false}, + givenCooperative: Cooperative{Enabled: false}, expected: []string{"r"}, }, { description: "No Coop - Many", givenRequested: []string{"r1", "r2"}, - givenCooperative: config.UserSyncCooperative{Enabled: false}, + givenCooperative: Cooperative{Enabled: false}, expected: []string{"r2", "r1"}, }, { description: "Coop - Integration Test", givenRequested: []string{"r1", "r2"}, - givenCooperative: config.UserSyncCooperative{Enabled: true, PriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}}, + givenCooperative: Cooperative{Enabled: true, PriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}}, expected: []string{"r2", "r1", "pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, }, } diff --git a/usersync/bidderfilter.go b/usersync/bidderfilter.go new file mode 100644 index 00000000000..cf3b985fdc0 --- /dev/null +++ b/usersync/bidderfilter.go @@ -0,0 +1,53 @@ +package usersync + +// BidderFilterMode represents the comparison approach of a BidderFilter. +type BidderFilterMode int + +const ( + BidderFilterModeInclude BidderFilterMode = iota + BidderFilterModeExclude +) + +// BidderFilter determines if a bidder has permission to perform a user sync activity. +type BidderFilter struct { + biddersAll bool + biddersLookup map[string]struct{} + mode BidderFilterMode +} + +// Allowed returns true if the bidder has permission per the filter settings and returns false if either +// the bidder is denied permission or if the BidderFilter is configured for an unsupported filter mode. +func (t BidderFilter) Allowed(bidder string) bool { + switch t.mode { + case BidderFilterModeInclude: + return t.bidderIncluded(bidder) + case BidderFilterModeExclude: + return !t.bidderIncluded(bidder) + default: + return false + } +} + +func (t BidderFilter) bidderIncluded(bidder string) bool { + if t.biddersAll { + return true + } + + _, exists := t.biddersLookup[bidder] + return exists +} + +// NewBidderFilter returns a new BidderFilter which applies the same mode for a list of specific bidders. +func NewBidderFilter(bidders []string, mode BidderFilterMode) BidderFilter { + biddersLookup := make(map[string]struct{}, len(bidders)) + for _, bidder := range bidders { + biddersLookup[bidder] = struct{}{} + } + + return BidderFilter{biddersLookup: biddersLookup, mode: mode} +} + +// NewBidderFilterForAll returns a new BidderFilter which applies the same mode for all bidders. +func NewBidderFilterForAll(mode BidderFilterMode) BidderFilter { + return BidderFilter{biddersAll: true, mode: mode} +} diff --git a/usersync/bidderfilter_test.go b/usersync/bidderfilter_test.go new file mode 100644 index 00000000000..de742bf14b2 --- /dev/null +++ b/usersync/bidderfilter_test.go @@ -0,0 +1,104 @@ +package usersync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBidderFilter(t *testing.T) { + bidder := "foo" + + testCases := []struct { + description string + bidders []string + mode BidderFilterMode + expected bool + }{ + { + description: "Include - None", + bidders: []string{}, + mode: BidderFilterModeInclude, + expected: false, + }, + { + description: "Include - One", + bidders: []string{bidder}, + mode: BidderFilterModeInclude, + expected: true, + }, + { + description: "Include - Many", + bidders: []string{"other", bidder}, + mode: BidderFilterModeInclude, + expected: true, + }, + { + description: "Include - Other", + bidders: []string{"other"}, + mode: BidderFilterModeInclude, + expected: false, + }, + { + description: "Exclude - None", + bidders: []string{}, + mode: BidderFilterModeExclude, + expected: true, + }, + { + description: "Exclude - One", + bidders: []string{bidder}, + mode: BidderFilterModeExclude, + expected: false, + }, + { + description: "Exclude - Many", + bidders: []string{"other", bidder}, + mode: BidderFilterModeExclude, + expected: false, + }, + { + description: "Exclude - Other", + bidders: []string{"other"}, + mode: BidderFilterModeExclude, + expected: true, + }, + { + description: "Invalid Mode", + bidders: []string{bidder}, + mode: BidderFilterMode(-1), + expected: false, + }, + } + + for _, test := range testCases { + filter := NewBidderFilter(test.bidders, test.mode) + assert.Equal(t, test.expected, filter.Allowed(bidder), test.description) + } +} + +func TestBidderFilterForAll(t *testing.T) { + bidder := "foo" + + testCases := []struct { + description string + mode BidderFilterMode + expected bool + }{ + { + description: "Include", + mode: BidderFilterModeInclude, + expected: true, + }, + { + description: "Exclude", + mode: BidderFilterModeExclude, + expected: false, + }, + } + + for _, test := range testCases { + filter := NewBidderFilterForAll(test.mode) + assert.Equal(t, test.expected, filter.Allowed(bidder), test.description) + } +} diff --git a/usersync/chooser.go b/usersync/chooser.go index e9532661142..48c1b897bd8 100644 --- a/usersync/chooser.go +++ b/usersync/chooser.go @@ -1,11 +1,13 @@ package usersync -import "github.com/prebid/prebid-server/config" - +// Chooser determines which user syncers are eligible for a given user sync request. type Chooser interface { - Choose(request Request, cookie Cookie) Result + // Choose considers bidders to sync, filters the bidders, and returns the result of the + // user sync selection. + Choose(request Request, cookie *Cookie) Result } +// NewChooser returns a new instance of the standard chooser implementation. func NewChooser(bidderSyncerLookup map[string]Syncer) Chooser { bidders := make([]string, 0, len(bidderSyncerLookup)) for k := range bidderSyncerLookup { @@ -19,51 +21,90 @@ func NewChooser(bidderSyncerLookup map[string]Syncer) Chooser { } } +// Request specifies a user sync request from an end user. type Request struct { - Bidders []string - Kind Kind - Privacy Privacy - Cooperative config.UserSyncCooperative - Limit int + Bidders []string + Cooperative Cooperative + Limit int + Privacy Privacy + SyncTypeFilter SyncTypeFilter +} + +// Cooperative specifies the settings for cooperative syncing for a given request, where bidders +// other than used by the publisher are considered for user syncing. +type Cooperative struct { + Enabled bool + PriorityGroups [][]string } +// Result specifies which bidders were included in the evaluation and which syncers were ultimately chosen. type Result struct { - Status Status BiddersEvaluated []BidderEvaluation - SyncersChosen []Syncer + Status Status + SyncersChosen []SyncerChoice } +// BidderEvaluation specifies the result of a bidder evaluation for a user sync. type BidderEvaluation struct { + Bidder string + SyncerKey string + Status Status +} + +// SyncerChoice specifies a syncer chosen for a user sync. +type SyncerChoice struct { Bidder string - Status Status + Syncer Syncer } +// Status specifies the result of a user sync. type Status int const ( + // StatusOK specifies user syncing is permitted. StatusOK Status = iota + + // StatusBlockedByUserOptOut specifies a user's cookie explicitly signals an opt-out. StatusBlockedByUserOptOut + + // StatusBlockedByGDPR specifies a user's GDPR TCF consent explicitly forbids host cookies + // or specific bidder syncing. StatusBlockedByGDPR + + // StatusBlockedByCCPA specifiers a user's CCPA consent explicitly forbids bidder syncing. StatusBlockedByCCPA + + // StatusAlreadySynced specifies a user's cookie has an existing non-expired sync for a specific bidder. StatusAlreadySynced + + // StatusUnknownBidder specifies a requested bidder is unknown to Prebid Server. StatusUnknownBidder - StatusIncompatibleKind + + // StatusTypeNotSupported specifies a requested sync type is not supported by a specific bidder. + StatusTypeNotSupported + + // StatusDuplicate specifies the requested bidders included a duplicate value either explicitly + // or through cooperative syncing. StatusDuplicate ) +// Privacy determines which privacy policies should be enforced for a user sync request. type Privacy interface { GDPRAllowsHostCookie() bool GDPRAllowsBidderSync(bidder string) bool CCPAAllowsBidderSync(bidder string) bool } +// standardChooser implements the user syncer algorithm per official Prebid specification. type standardChooser struct { bidderSyncerLookup map[string]Syncer biddersAvailable []string bidderChooser bidderChooser } -func (c standardChooser) Choose(request Request, cookie Cookie) Result { +// Choose randomly selects user syncers which are permitted by the user's privacy settings and +// which don't already have a valid user sync. +func (c standardChooser) Choose(request Request, cookie *Cookie) Result { if !cookie.AllowSyncs() { return Result{Status: StatusBlockedByUserOptOut} } @@ -76,22 +117,22 @@ func (c standardChooser) Choose(request Request, cookie Cookie) Result { limitDisabled := request.Limit <= 0 biddersEvaluated := make([]BidderEvaluation, 0) - syncersChosen := make([]Syncer, 0) + syncersChosen := make([]SyncerChoice, 0) bidders := c.bidderChooser.choose(request.Bidders, c.biddersAvailable, request.Cooperative) for i := 0; i < len(bidders) && (limitDisabled || len(syncersChosen) < request.Limit); i++ { - syncer, evaluation := c.evaluate(bidders[i], syncersSeen, request.Kind, request.Privacy, cookie) + syncer, evaluation := c.evaluate(bidders[i], syncersSeen, request.SyncTypeFilter, request.Privacy, cookie) biddersEvaluated = append(biddersEvaluated, evaluation) if evaluation.Status == StatusOK { - syncersChosen = append(syncersChosen, syncer) + syncersChosen = append(syncersChosen, SyncerChoice{Bidder: bidders[i], Syncer: syncer}) } } return Result{Status: StatusOK, BiddersEvaluated: biddersEvaluated, SyncersChosen: syncersChosen} } -func (c standardChooser) evaluate(bidder string, syncersSeen map[string]struct{}, kind Kind, privacy Privacy, cookie Cookie) (Syncer, BidderEvaluation) { +func (c standardChooser) evaluate(bidder string, syncersSeen map[string]struct{}, syncTypeFilter SyncTypeFilter, privacy Privacy, cookie *Cookie) (Syncer, BidderEvaluation) { syncer, exists := c.bidderSyncerLookup[bidder] if !exists { return nil, BidderEvaluation{Bidder: bidder, Status: StatusUnknownBidder} @@ -103,8 +144,8 @@ func (c standardChooser) evaluate(bidder string, syncersSeen map[string]struct{} } syncersSeen[syncer.Key()] = struct{}{} - if !syncer.SupportsKind(kind) { - return nil, BidderEvaluation{Bidder: bidder, Status: StatusIncompatibleKind} + if !syncer.SupportsType(syncTypeFilter.ForBidder(bidder)) { + return nil, BidderEvaluation{Bidder: bidder, Status: StatusTypeNotSupported} } if cookie.HasLiveSync(syncer.Key()) { diff --git a/usersync/chooser_test.go b/usersync/chooser_test.go index 078cd07dbbb..000bb8b2911 100644 --- a/usersync/chooser_test.go +++ b/usersync/chooser_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/privacy" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -45,12 +44,17 @@ func TestNewChooser(t *testing.T) { } func TestChooserChoose(t *testing.T) { - fakeSyncerA := fakeSyncer{key: "keyA", supportsKind: true} - fakeSyncerB := fakeSyncer{key: "keyB", supportsKind: true} - fakeSyncerC := fakeSyncer{key: "keyC", supportsKind: false} + fakeSyncerA := fakeSyncer{key: "keyA", supportsIFrame: true} + fakeSyncerB := fakeSyncer{key: "keyB", supportsIFrame: true} + fakeSyncerC := fakeSyncer{key: "keyC", supportsIFrame: false} bidderSyncerLookup := map[string]Syncer{"a": fakeSyncerA, "b": fakeSyncerB, "c": fakeSyncerC} + syncerChoiceA := SyncerChoice{Bidder: "a", Syncer: fakeSyncerA} + syncerChoiceB := SyncerChoice{Bidder: "b", Syncer: fakeSyncerB} + syncTypeFilter := SyncTypeFilter{ + IFrame: NewBidderFilterForAll(BidderFilterModeInclude), + Redirect: NewBidderFilterForAll(BidderFilterModeExclude)} - cooperativeConfig := config.UserSyncCooperative{Enabled: true} + cooperativeConfig := Cooperative{Enabled: true} testCases := []struct { description string @@ -98,7 +102,7 @@ func TestChooserChoose(t *testing.T) { expected: Result{ Status: StatusOK, BiddersEvaluated: []BidderEvaluation{}, - SyncersChosen: []Syncer{}, + SyncersChosen: []SyncerChoice{}, }, }, { @@ -112,7 +116,7 @@ func TestChooserChoose(t *testing.T) { expected: Result{ Status: StatusOK, BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}}, - SyncersChosen: []Syncer{fakeSyncerA}, + SyncersChosen: []SyncerChoice{syncerChoiceA}, }, }, { @@ -125,8 +129,8 @@ func TestChooserChoose(t *testing.T) { givenCookie: Cookie{}, expected: Result{ Status: StatusOK, - BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusIncompatibleKind}}, - SyncersChosen: []Syncer{}, + BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusTypeNotSupported}}, + SyncersChosen: []SyncerChoice{}, }, }, { @@ -140,7 +144,7 @@ func TestChooserChoose(t *testing.T) { expected: Result{ Status: StatusOK, BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "b", Status: StatusOK}}, - SyncersChosen: []Syncer{fakeSyncerA, fakeSyncerB}, + SyncersChosen: []SyncerChoice{syncerChoiceA, syncerChoiceB}, }, }, { @@ -154,7 +158,7 @@ func TestChooserChoose(t *testing.T) { expected: Result{ Status: StatusOK, BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "b", Status: StatusOK}}, - SyncersChosen: []Syncer{fakeSyncerA, fakeSyncerB}, + SyncersChosen: []SyncerChoice{syncerChoiceA, syncerChoiceB}, }, }, { @@ -168,7 +172,7 @@ func TestChooserChoose(t *testing.T) { expected: Result{ Status: StatusOK, BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}}, - SyncersChosen: []Syncer{fakeSyncerA}, + SyncersChosen: []SyncerChoice{syncerChoiceA}, }, }, { @@ -181,8 +185,8 @@ func TestChooserChoose(t *testing.T) { givenCookie: Cookie{}, expected: Result{ Status: StatusOK, - BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusIncompatibleKind}, {Bidder: "a", Status: StatusOK}}, - SyncersChosen: []Syncer{fakeSyncerA}, + BiddersEvaluated: []BidderEvaluation{{Bidder: "c", Status: StatusTypeNotSupported}, {Bidder: "a", Status: StatusOK}}, + SyncersChosen: []SyncerChoice{syncerChoiceA}, }, }, { @@ -195,8 +199,8 @@ func TestChooserChoose(t *testing.T) { givenCookie: Cookie{}, expected: Result{ Status: StatusOK, - BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "c", Status: StatusIncompatibleKind}}, - SyncersChosen: []Syncer{fakeSyncerA}, + BiddersEvaluated: []BidderEvaluation{{Bidder: "a", Status: StatusOK}, {Bidder: "c", Status: StatusTypeNotSupported}}, + SyncersChosen: []SyncerChoice{syncerChoiceA}, }, }, } @@ -206,7 +210,7 @@ func TestChooserChoose(t *testing.T) { for _, test := range testCases { // set request values which don't need to be specified for each test test.givenRequest.Bidders = bidders - test.givenRequest.Kind = KindRedirect + test.givenRequest.SyncTypeFilter = syncTypeFilter test.givenRequest.Cooperative = cooperativeConfig mockBidderChooser := &mockBidderChooser{} @@ -220,18 +224,22 @@ func TestChooserChoose(t *testing.T) { bidderChooser: mockBidderChooser, } - result := chooser.Choose(test.givenRequest, test.givenCookie) + result := chooser.Choose(test.givenRequest, &test.givenCookie) assert.Equal(t, test.expected, result, test.description) } } func TestChooserEvaluate(t *testing.T) { - fakeSyncerA := fakeSyncer{key: "keyA", supportsKind: true} - fakeSyncerB := fakeSyncer{key: "keyB", supportsKind: false} + fakeSyncerA := fakeSyncer{key: "keyA", supportsIFrame: true} + fakeSyncerB := fakeSyncer{key: "keyB", supportsIFrame: false} bidderSyncerLookup := map[string]Syncer{"a": fakeSyncerA, "b": fakeSyncerB} + syncTypeFilter := SyncTypeFilter{ + IFrame: NewBidderFilterForAll(BidderFilterModeInclude), + Redirect: NewBidderFilterForAll(BidderFilterModeExclude)} cookieNeedsSync := Cookie{} - cookieAlreadyHasSync := Cookie{uids: map[string]uidWithExpiry{"keyA": {Expires: time.Now().Add(time.Duration(24) * time.Hour)}}} + cookieAlreadyHasSyncForA := Cookie{uids: map[string]uidWithExpiry{"keyA": {Expires: time.Now().Add(time.Duration(24) * time.Hour)}}} + cookieAlreadyHasSyncForB := Cookie{uids: map[string]uidWithExpiry{"keyB": {Expires: time.Now().Add(time.Duration(24) * time.Hour)}}} testCases := []struct { description string @@ -281,18 +289,28 @@ func TestChooserEvaluate(t *testing.T) { givenCookie: cookieNeedsSync, expectedSyncer: nil, expectedBidder: "b", - expectedStatus: StatusIncompatibleKind, + expectedStatus: StatusTypeNotSupported, }, { description: "Already Synced", givenBidder: "a", givenSyncersSeen: map[string]struct{}{}, givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, - givenCookie: cookieAlreadyHasSync, + givenCookie: cookieAlreadyHasSyncForA, expectedSyncer: nil, expectedBidder: "a", expectedStatus: StatusAlreadySynced, }, + { + description: "Different Bidder Already Synced", + givenBidder: "a", + givenSyncersSeen: map[string]struct{}{}, + givenPrivacy: fakePrivacy{gdprAllowsHostCookie: true, gdprAllowsBidderSync: true, ccpaAllowsBidderSync: true}, + givenCookie: cookieAlreadyHasSyncForB, + expectedSyncer: fakeSyncerA, + expectedBidder: "a", + expectedStatus: StatusOK, + }, { description: "Blocked By GDPR", givenBidder: "a", @@ -317,7 +335,7 @@ func TestChooserEvaluate(t *testing.T) { for _, test := range testCases { chooser, _ := NewChooser(bidderSyncerLookup).(standardChooser) - sync, evaluation := chooser.evaluate(test.givenBidder, test.givenSyncersSeen, KindBidderPreference, test.givenPrivacy, test.givenCookie) + sync, evaluation := chooser.evaluate(test.givenBidder, test.givenSyncersSeen, syncTypeFilter, test.givenPrivacy, &test.givenCookie) assert.Equal(t, test.expectedSyncer, sync, test.description+":syncer") @@ -330,26 +348,35 @@ type mockBidderChooser struct { mock.Mock } -func (m *mockBidderChooser) choose(requested, available []string, cooperative config.UserSyncCooperative) []string { +func (m *mockBidderChooser) choose(requested, available []string, cooperative Cooperative) []string { args := m.Called(requested, available, cooperative) return args.Get(0).([]string) } type fakeSyncer struct { - key string - supportsKind bool + key string + supportsIFrame bool + supportsRedirect bool } func (s fakeSyncer) Key() string { return s.key } -func (s fakeSyncer) SupportsKind(kind Kind) bool { - return s.supportsKind +func (s fakeSyncer) SupportsType(syncTypes []SyncType) bool { + for _, syncType := range syncTypes { + if syncType == SyncTypeIFrame && s.supportsIFrame { + return true + } + if syncType == SyncTypeRedirect && s.supportsRedirect { + return true + } + } + return false } -func (fakeSyncer) GetSync(kind Kind, privacyPolicies privacy.Policies) Sync { - return Sync{} +func (fakeSyncer) GetSync(syncTypes []SyncType, privacyPolicies privacy.Policies) (Sync, error) { + return Sync{}, nil } type fakePrivacy struct { diff --git a/usersync/cookie.go b/usersync/cookie.go index b66278651b7..243b61dc8c2 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -12,6 +12,8 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) +// todo: add "useSameSite" into the cookie itself, read from the request via user agent headers + const uidCookieName = "uids" // uidTTL is the default amount of time a uid stored within a cookie is considered valid. This is @@ -26,7 +28,7 @@ var bidderToFamilyNames = map[openrtb_ext.BidderName]string{ // Cookie is the cookie used in Prebid Server. // -// To get an instance of this from a request, use ParsePBSCookieFromRequest. +// To get an instance of this from a request, use ParseCookieFromRequest. // To write an instance onto a response, use SetCookieOnResponse. type Cookie struct { uids map[string]uidWithExpiry @@ -43,6 +45,7 @@ type uidWithExpiry struct { Expires time.Time `json:"expires"` } +// todo: add samesite if cookie is already same site or detect by browser version // ParseCookieFromRequest parses the UserSyncMap from an HTTP Request. func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie { if cookie.OptOutCookie.Name != "" { @@ -86,7 +89,7 @@ func ParseCookie(httpCookie *http.Cookie) *Cookie { return &cookie } -// NewCookie returns an empty PBSCookie +// NewCookie returns a new empty cookie. func NewCookie() *Cookie { return &Cookie{ uids: make(map[string]uidWithExpiry), @@ -186,14 +189,11 @@ func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie b currSize = len([]byte(httpCookie.String())) } - var uidsCookieStr string if setSiteCookie { httpCookie.Secure = true httpCookie.SameSite = http.SameSiteNoneMode - } else { - uidsCookieStr = httpCookie.String() } - w.Header().Add("Set-Cookie", uidsCookieStr) + w.Header().Add("Set-Cookie", httpCookie.String()) } // Unsync removes the user's ID for the given family from this cookie. @@ -223,7 +223,7 @@ func (cookie *Cookie) HasAnyLiveSyncs() bool { // TrySync tries to set the UID for some family name. It returns an error if the set didn't happen. func (cookie *Cookie) TrySync(familyName string, uid string) error { if !cookie.AllowSyncs() { - return errors.New("The user has opted out of prebid server PBSCookie syncs.") + return errors.New("The user has opted out of prebid server cookie syncs.") } // At the moment, Facebook calls /setuid with a UID of 0 if the user isn't logged into Facebook. @@ -242,8 +242,8 @@ func (cookie *Cookie) TrySync(familyName string, uid string) error { // cookieJson defines the JSON contract for the cookie data's storage format. // -// This exists so that PBSCookie (which is public) can have private fields, and the rest of -// PBS doesn't have to worry about the cookie data storage format. +// This exists so that Cookie (which is public) can have private fields, and the rest of +// the code doesn't have to worry about the cookie data storage format. type cookieJson struct { LegacyUIDs map[string]string `json:"uids,omitempty"` UIDs map[string]uidWithExpiry `json:"tempUIDs,omitempty"` diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index df46640449a..cc464827fc5 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -160,6 +160,89 @@ func TestParseOtherCookie(t *testing.T) { } } +func TestParseCookieFromRequestOptOut(t *testing.T) { + optOutCookieName := "optOutCookieName" + optOutCookieValue := "optOutCookieValue" + + existingCookie := *(&Cookie{ + uids: map[string]uidWithExpiry{ + "foo": newTempId("fooID", 1), + "bar": newTempId("barID", 2), + }, + optOut: false, + birthday: timestamp(), + }).ToHTTPCookie(24 * time.Hour) + + testCases := []struct { + description string + givenExistingCookies []http.Cookie + expectedEmpty bool + expectedSetOptOut bool + }{ + { + description: "Opt Out Cookie", + givenExistingCookies: []http.Cookie{ + existingCookie, + {Name: optOutCookieName, Value: optOutCookieValue}}, + expectedEmpty: true, + expectedSetOptOut: true, + }, + { + description: "No Opt Out Cookie", + givenExistingCookies: []http.Cookie{ + existingCookie}, + expectedEmpty: false, + expectedSetOptOut: false, + }, + { + description: "Opt Out Cookie - Wrong Value", + givenExistingCookies: []http.Cookie{ + existingCookie, + {Name: optOutCookieName, Value: "wrong"}}, + expectedEmpty: false, + expectedSetOptOut: false, + }, + { + description: "Opt Out Cookie - Wrong Name", + givenExistingCookies: []http.Cookie{ + existingCookie, + {Name: "wrong", Value: optOutCookieValue}}, + expectedEmpty: false, + expectedSetOptOut: false, + }, + { + description: "Opt Out Cookie - No Host Cookies", + givenExistingCookies: []http.Cookie{ + {Name: optOutCookieName, Value: optOutCookieValue}}, + expectedEmpty: true, + expectedSetOptOut: true, + }, + } + + for _, test := range testCases { + req := httptest.NewRequest("POST", "http://www.prebid.com", nil) + + for _, c := range test.givenExistingCookies { + req.AddCookie(&c) + } + + parsed := ParseCookieFromRequest(req, &config.HostCookie{ + Family: "foo", + OptOutCookie: config.Cookie{ + Name: optOutCookieName, + Value: optOutCookieValue, + }, + }) + + if test.expectedEmpty { + assert.Empty(t, parsed.uids, test.description+":empty") + } else { + assert.NotEmpty(t, parsed.uids, test.description+":not-empty") + } + assert.Equal(t, parsed.optOut, test.expectedSetOptOut, test.description+":opt-out") + } +} + func TestCookieReadWrite(t *testing.T) { cookie := newSampleCookie() @@ -265,33 +348,41 @@ func TestGetUIDsWithNilCookie(t *testing.T) { } func TestTrimCookiesClosestExpirationDates(t *testing.T) { - cookieToSend, cookieToSendLen := newTestCookie() - closestToExpirationDate := "key7" + cookieToSend := &Cookie{ + uids: map[string]uidWithExpiry{ + "k1": newTempId("12345678901234567890123456789012345678901234567890", 7), + "k2": newTempId("abcdefghijklmnopqrstuvwxyz", 6), + "k3": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 6), + "k4": newTempId("12345678901234567890123456789612345678901234567890", 5), + "k5": newTempId("aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ", 4), + "k6": newTempId("12345678901234567890123456789012345678901234567890", 3), + "k7": newTempId("abcdefghijklmnopqrstuvwxyz", 2), + }, + optOut: false, + birthday: timestamp(), + } type aTest struct { maxCookieSize int - expAction string + expKeys []string } testCases := []aTest{ - {maxCookieSize: 2000, expAction: "equal"}, //1 don't trim, set - {maxCookieSize: 0, expAction: "equal"}, //2 unlimited size: don't trim, set - {maxCookieSize: 800, expAction: "trim"}, //3 trim to size and set - {maxCookieSize: 500, expAction: "trim"}, //4 trim to size and set - {maxCookieSize: 200, expAction: "empty"}, //5 insufficient size, trim to zero length and set - {maxCookieSize: -100, expAction: "empty"}, //6 invalid size, trim to zero length and set + {maxCookieSize: 2000, expKeys: []string{"k1", "k2", "k3", "k4", "k5", "k6", "k7"}}, //1 don't trim, set + {maxCookieSize: 0, expKeys: []string{"k1", "k2", "k3", "k4", "k5", "k6", "k7"}}, //2 unlimited size: don't trim, set + {maxCookieSize: 800, expKeys: []string{"k1", "k2", "k3", "k4"}}, //3 trim to size and set + {maxCookieSize: 500, expKeys: []string{"k1", "k3"}}, //4 trim to size and set + {maxCookieSize: 200, expKeys: []string{}}, //5 insufficient size, trim to zero length and set + {maxCookieSize: -100, expKeys: []string{}}, //6 invalid size, trim to zero length and set } for i := range testCases { processedCookie := writeThenRead(cookieToSend, testCases[i].maxCookieSize) - switch testCases[i].expAction { - case "equal": - assert.Equal(t, cookieToSendLen, len(processedCookie.uids), "[Test %d] MaxCookieSizeBytes equal to zero or bigger than %d bytes should be enough to set and remain cookie unchanged \n", i+1, len(processedCookie.uids)) - assert.Containsf(t, processedCookie.uids, closestToExpirationDate, "[Test %d] Oldest entry in cookie should not have been eliminated", i+1) - case "trim": - assert.Equal(t, cookieToSendLen > len(processedCookie.uids), true, "[Test %d] MaxCookieSizeBytes of %d is smaller than %d bytes and cookie entries should have been removed\n", i+1, testCases[i].maxCookieSize, cookieToSendLen) - assert.NotContainsf(t, processedCookie.uids, closestToExpirationDate, "[Test %d] Oldest entry in cookie should not have been eliminated", i+1) - case "empty": - assert.Equal(t, len(processedCookie.uids), 0, "[Test %d] MaxCookieSizeBytes of %d is too small, processedCookie.uids should be empty\n", i+1) + + actualKeys := make([]string, 0, 7) + for key, _ := range processedCookie.uids { + actualKeys = append(actualKeys, key) } + + assert.ElementsMatch(t, testCases[i].expKeys, actualKeys, "[Test %d]", i+1) } } @@ -383,23 +474,6 @@ func newSampleCookie() *Cookie { } } -func newTestCookie() (*Cookie, int) { - var mediumSizeCookie *Cookie = &Cookie{ - uids: map[string]uidWithExpiry{ - "key1": newTempId("12345678901234567890123456789012345678901234567890", 7), - "key2": newTempId("abcdefghijklmnopqrstuvwxyz", 6), - "key3": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 6), - "key4": newTempId("12345678901234567890123456789612345678901234567890", 5), - "key5": newTempId("aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ", 4), - "key6": newTempId("12345678901234567890123456789012345678901234567890", 3), - "key7": newTempId("abcdefghijklmnopqrstuvwxyz", 2), - }, - optOut: false, - birthday: timestamp(), - } - return mediumSizeCookie, len(mediumSizeCookie.uids) -} - func writeThenRead(cookie *Cookie, maxCookieSize int) *Cookie { w := httptest.NewRecorder() hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: maxCookieSize} @@ -412,23 +486,20 @@ func writeThenRead(cookie *Cookie, maxCookieSize int) *Cookie { return ParseCookieFromRequest(&request, hostCookie) } -// func TestSetCookieOnResponseForSameSiteNone(t *testing.T) { -// cookie := newSampleCookie() -// w := httptest.NewRecorder() -// req := httptest.NewRequest("GET", "http://www.prebid.com", nil) -// ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" -// req.Header.Set("User-Agent", ua) -// hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: 0} -// cookie.SetCookieOnResponse(w, true, hostCookie, 90*24*time.Hour) -// writtenCookie := w.HeaderMap.Get("Set-Cookie") -// t.Log("Set-Cookie is: ", writtenCookie) -// if !strings.Contains(writtenCookie, "SSCookie=1") { -// t.Error("Set-Cookie should contain SSCookie=1") -// } -// if !strings.Contains(writtenCookie, "; Secure;") { -// t.Error("Set-Cookie should contain Secure") -// } -// } +func TestSetCookieOnResponseForSameSiteNone(t *testing.T) { + cookie := newSampleCookie() + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://www.prebid.com", nil) + ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36" + req.Header.Set("User-Agent", ua) + hostCookie := &config.HostCookie{Domain: "mock-domain", MaxCookieSizeBytes: 0} + cookie.SetCookieOnResponse(w, true, hostCookie, 90*24*time.Hour) + writtenCookie := w.HeaderMap.Get("Set-Cookie") + t.Log("Set-Cookie is: ", writtenCookie) + if !strings.Contains(writtenCookie, "; Secure;") { + t.Error("Set-Cookie should contain Secure") + } +} func TestSetCookieOnResponseForOlderChromeVersion(t *testing.T) { cookie := newSampleCookie() diff --git a/usersync/shuffler.go b/usersync/shuffler.go index b75fa3e105b..0185817ec5e 100644 --- a/usersync/shuffler.go +++ b/usersync/shuffler.go @@ -2,10 +2,12 @@ package usersync import "math/rand" +// shuffler changes the order of elements in the slice. type shuffler interface { shuffle(v []string) } +// randomShuffler randombly changes the order of elements in the slice. type randomShuffler struct{} func (randomShuffler) shuffle(v []string) { diff --git a/usersync/syncer.go b/usersync/syncer.go index 51a6861c172..ea829fc6494 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -1,26 +1,202 @@ package usersync -import "github.com/prebid/prebid-server/privacy" +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "text/template" + validator "github.com/asaskevich/govalidator" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/macros" + "github.com/prebid/prebid-server/privacy" +) + +// Syncer represents the user sync configuration for a bidder or a shared set of bidders. type Syncer interface { + // Key is the name of the syncer as stored in the user's cookie. This is not necessarily a + // one-to-one relationship with a bidder. Key() string - SupportsKind(kind Kind) bool - GetSync(kind Kind, privacyPolicies privacy.Policies) Sync + + // SupportsType returns true if the syncer supports at least one of the specified sync types. + SupportsType(syncTypes []SyncType) bool + + // GetSync returns a user sync for the user's device to perform, or an error if the none of the + // sync types are supported or if macro substitution fails. + GetSync(syncTypes []SyncType, privacyPolicies privacy.Policies) (Sync, error) } +// Sync represents a user sync for the user's device to perform. type Sync struct { - URL string - Kind Kind - SupportCORS bool + URL string + Type SyncType + SupportsCORS bool } -type Kind int +type standardSyncer struct { + key string + defaultSyncType SyncType + iframe *template.Template + redirect *template.Template + supportCORS bool +} const ( - KindBidderPreference Kind = iota - KindIFrame - KindRedirect + setuidSyncTypeIFrame = "b" + setuidSyncTypeRedirect = "i" ) -// todo: syncer from config -// - builds up the url template per bidder +// NewSyncer creates a new Syncer instance from the provided configuration, or an error if macro +// substition fails or the url specified is invalid. +func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { + syncer := standardSyncer{ + key: syncerConfig.Key, + } + + if syncerConfig.IFrame != nil { + var err error + syncer.iframe, err = composeTemplate(syncerConfig.Key, setuidSyncTypeIFrame, hostConfig, *syncerConfig.IFrame) + if err != nil { + return nil, err + } + } + + if syncerConfig.Redirect != nil { + var err error + syncer.redirect, err = composeTemplate(syncerConfig.Key, setuidSyncTypeRedirect, hostConfig, *syncerConfig.Redirect) + if err != nil { + return nil, err + } + } + + return syncer, nil +} + +var externalHostRegex = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) +var syncerKeyRegex = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) +var syncTypeRegex = regexp.MustCompile(`{{\s*.SyncType\s*}}`) +var userMacroRegex = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) +var redirectRegex = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) + +func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncerEndpoint config.SyncerEndpoint) (*template.Template, error) { + redirectTemplate := syncerEndpoint.RedirectURL + if redirectTemplate == "" { + redirectTemplate = hostConfig.RedirectURL + } + + externalURL := syncerEndpoint.ExternalURL + if externalURL == "" { + externalURL = hostConfig.ExternalURL + } + + redirectURL := externalHostRegex.ReplaceAllString(redirectTemplate, externalURL) + redirectURL = syncerKeyRegex.ReplaceAllString(redirectURL, key) + redirectURL = syncTypeRegex.ReplaceAllString(redirectURL, syncTypeValue) + redirectURL = userMacroRegex.ReplaceAllString(redirectURL, syncerEndpoint.UserMacro) + redirectURL = url.PathEscape(redirectURL) + + url := redirectRegex.ReplaceAllString(externalURL, redirectURL) + + templateName := strings.ToLower(key) + "_usersync_url" + template, err := template.New(templateName).Parse(url) + if err != nil { + return nil, err + } + + if err := validateTemplate(template); err != nil { + return nil, err + } + + return template, nil +} + +func validateTemplate(template *template.Template) error { + testValues := macros.UserSyncTemplateParams{ + GDPR: "anyGDPR", + GDPRConsent: "anyGDPRConsent", + USPrivacy: "anyCCPAConsent", + } + + url, err := macros.ResolveMacros(template, testValues) + if err != nil { + return err + } + + if !validator.IsURL(url) || !validator.IsRequestURL(url) { + return fmt.Errorf("composed url %s is invalid", url) + } + + return nil +} + +func (s standardSyncer) Key() string { + return s.key +} + +func (s standardSyncer) SupportsType(syncTypes []SyncType) bool { + for _, syncType := range syncTypes { + switch syncType { + case SyncTypeIFrame: + if s.iframe != nil { + return true + } + case SyncTypeRedirect: + if s.redirect != nil { + return true + } + } + } + return false +} + +func (s standardSyncer) GetSync(syncTypes []SyncType, privacyPolicies privacy.Policies) (Sync, error) { + syncType, err := s.chooseSyncType(syncTypes) + if err != nil { + return Sync{}, err + } + + syncTemplate := s.chooseTemplate(syncType) + + url, err := macros.ResolveMacros(syncTemplate, macros.UserSyncTemplateParams{ + GDPR: privacyPolicies.GDPR.Signal, + GDPRConsent: privacyPolicies.GDPR.Consent, + USPrivacy: privacyPolicies.CCPA.Consent, + }) + if err != nil { + return Sync{}, err + } + + sync := Sync{ + URL: url, + Type: syncType, + SupportsCORS: s.supportCORS, + } + return sync, nil +} + +func (s standardSyncer) chooseSyncType(syncTypes []SyncType) (SyncType, error) { + if len(syncTypes) == 0 { + return SyncTypeUnknown, errors.New("no sync types provided") + } + + for _, syncType := range syncTypes { + if syncType == s.defaultSyncType { + return syncType, nil + } + } + + return syncTypes[0], nil +} + +func (s standardSyncer) chooseTemplate(syncType SyncType) *template.Template { + switch syncType { + case SyncTypeIFrame: + return s.iframe + case SyncTypeRedirect: + return s.redirect + default: + return nil + } +} diff --git a/usersync/synctype.go b/usersync/synctype.go new file mode 100644 index 00000000000..3a1db225eca --- /dev/null +++ b/usersync/synctype.go @@ -0,0 +1,49 @@ +package usersync + +// SyncType specifies the protocol used to perform a user sync. +type SyncType int + +const ( + // SyncTypeUnknown specifies the user sync type is invalid or not specified. + SyncTypeUnknown SyncType = -1 + + // SyncTypeIFrame specifies the user sync is to be performed within an HTML `iframe` + // and to expect the server to return a valid HTML page with an embedded script. + SyncTypeIFrame SyncType = 0 + + // SyncTypeRedirect specifies the user sync is to be performed within an HTML image + // and to expect the server to return a 302 redirect. + SyncTypeRedirect SyncType = 1 +) + +// SyncTypeFilter determines which sync types, if any, the bidder is permitted to use. +type SyncTypeFilter struct { + IFrame BidderFilter + Redirect BidderFilter +} + +// ForBidder returns a slice of sync types the bidder is permitted to use. +func (t SyncTypeFilter) ForBidder(bidder string) []SyncType { + var syncTypes []SyncType + + if t.IFrame.Allowed(bidder) { + syncTypes = append(syncTypes, SyncTypeIFrame) + } + + if t.Redirect.Allowed(bidder) { + syncTypes = append(syncTypes, SyncTypeRedirect) + } + + return syncTypes +} + +func (t SyncType) String() string { + switch t { + case SyncTypeIFrame: + return "iframe" + case SyncTypeRedirect: + return "redirect" + default: + return "" + } +} diff --git a/usersync/synctype_test.go b/usersync/synctype_test.go new file mode 100644 index 00000000000..8d89c66363f --- /dev/null +++ b/usersync/synctype_test.go @@ -0,0 +1,81 @@ +package usersync + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSyncTypeFilter(t *testing.T) { + bidder := "foo" + + bidderFilterAllowed := NewBidderFilterForAll(BidderFilterModeInclude) + bidderFilterNotAllowed := NewBidderFilterForAll(BidderFilterModeExclude) + + testCases := []struct { + description string + givenIFrameFilter BidderFilter + givenRedirectFilter BidderFilter + expectedSyncTypes []SyncType + }{ + { + description: "None", + givenIFrameFilter: bidderFilterNotAllowed, + givenRedirectFilter: bidderFilterNotAllowed, + expectedSyncTypes: []SyncType{}, + }, + { + description: "IFrame Only", + givenIFrameFilter: bidderFilterAllowed, + givenRedirectFilter: bidderFilterNotAllowed, + expectedSyncTypes: []SyncType{SyncTypeIFrame}, + }, + { + description: "Redirect Only", + givenIFrameFilter: bidderFilterNotAllowed, + givenRedirectFilter: bidderFilterAllowed, + expectedSyncTypes: []SyncType{SyncTypeRedirect}, + }, + { + description: "All", + givenIFrameFilter: bidderFilterAllowed, + givenRedirectFilter: bidderFilterAllowed, + expectedSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + }, + } + + for _, test := range testCases { + syncTypeFilter := SyncTypeFilter{IFrame: test.givenIFrameFilter, Redirect: test.givenRedirectFilter} + syncTypes := syncTypeFilter.ForBidder(bidder) + assert.ElementsMatch(t, test.expectedSyncTypes, syncTypes, test.description) + } +} + +func TestString(t *testing.T) { + testCases := []struct { + description string + given SyncType + expected string + }{ + { + description: "IFrame", + given: SyncTypeIFrame, + expected: "iframe", + }, + { + description: "Redirect", + given: SyncTypeRedirect, + expected: "redirect", + }, + { + description: "Unknown", + given: SyncType(42), + expected: "", + }, + } + + for _, test := range testCases { + result := test.given.String() + assert.Equal(t, test.expected, result) + } +} diff --git a/usersync/usersync.go b/usersync/usersync.go deleted file mode 100644 index dffc1832f01..00000000000 --- a/usersync/usersync.go +++ /dev/null @@ -1,29 +0,0 @@ -package usersync - -import "github.com/prebid/prebid-server/privacy" - -type Usersyncer interface { - // GetUsersyncInfo returns basic info the browser needs in order to run a user sync. - // The returned UsersyncInfo object must not be mutated by callers. - // - // For more information about user syncs, see http://clearcode.cc/2015/12/cookie-syncing/ - GetUsersyncInfo(privacyPolicies privacy.Policies) (*UsersyncInfo, error) - - // FamilyName should be the same as the `BidderName` for this Usersyncer. - // This function only exists for legacy reasons. - // TODO #362: when the appnexus usersyncer is consistent, delete this and use the key - // of NewSyncerMap() here instead. - FamilyName() string -} - -type UsersyncInfo struct { - URL string `json:"url,omitempty"` - Type string `json:"type,omitempty"` - SupportCORS bool `json:"supportCORS,omitempty"` -} - -type CookieSyncBidders struct { - BidderCode string `json:"bidder"` - NoCookie bool `json:"no_cookie,omitempty"` - UsersyncInfo *UsersyncInfo `json:"usersync,omitempty"` -} diff --git a/usersync/usersyncers/syncer.go b/usersync/usersyncers/syncer.go deleted file mode 100644 index 7467eb261fa..00000000000 --- a/usersync/usersyncers/syncer.go +++ /dev/null @@ -1,195 +0,0 @@ -package usersyncers - -import ( - "strings" - "text/template" - - "github.com/golang/glog" - ttx "github.com/prebid/prebid-server/adapters/33across" - "github.com/prebid/prebid-server/adapters/acuityads" - "github.com/prebid/prebid-server/adapters/adform" - "github.com/prebid/prebid-server/adapters/adkernel" - "github.com/prebid/prebid-server/adapters/adkernelAdn" - "github.com/prebid/prebid-server/adapters/adman" - "github.com/prebid/prebid-server/adapters/admixer" - "github.com/prebid/prebid-server/adapters/adocean" - "github.com/prebid/prebid-server/adapters/adpone" - "github.com/prebid/prebid-server/adapters/adtarget" - "github.com/prebid/prebid-server/adapters/adtelligent" - "github.com/prebid/prebid-server/adapters/advangelists" - "github.com/prebid/prebid-server/adapters/adyoulike" - "github.com/prebid/prebid-server/adapters/aja" - "github.com/prebid/prebid-server/adapters/amx" - "github.com/prebid/prebid-server/adapters/appnexus" - "github.com/prebid/prebid-server/adapters/audienceNetwork" - "github.com/prebid/prebid-server/adapters/avocet" - "github.com/prebid/prebid-server/adapters/beachfront" - "github.com/prebid/prebid-server/adapters/beintoo" - "github.com/prebid/prebid-server/adapters/between" - "github.com/prebid/prebid-server/adapters/brightroll" - "github.com/prebid/prebid-server/adapters/colossus" - "github.com/prebid/prebid-server/adapters/connectad" - "github.com/prebid/prebid-server/adapters/consumable" - "github.com/prebid/prebid-server/adapters/conversant" - "github.com/prebid/prebid-server/adapters/cpmstar" - "github.com/prebid/prebid-server/adapters/datablocks" - "github.com/prebid/prebid-server/adapters/deepintent" - "github.com/prebid/prebid-server/adapters/dmx" - "github.com/prebid/prebid-server/adapters/emx_digital" - "github.com/prebid/prebid-server/adapters/engagebdr" - "github.com/prebid/prebid-server/adapters/eplanning" - "github.com/prebid/prebid-server/adapters/gamma" - "github.com/prebid/prebid-server/adapters/gamoshi" - "github.com/prebid/prebid-server/adapters/grid" - "github.com/prebid/prebid-server/adapters/gumgum" - "github.com/prebid/prebid-server/adapters/improvedigital" - "github.com/prebid/prebid-server/adapters/invibes" - "github.com/prebid/prebid-server/adapters/ix" - "github.com/prebid/prebid-server/adapters/jixie" - "github.com/prebid/prebid-server/adapters/krushmedia" - "github.com/prebid/prebid-server/adapters/lifestreet" - "github.com/prebid/prebid-server/adapters/lockerdome" - "github.com/prebid/prebid-server/adapters/logicad" - "github.com/prebid/prebid-server/adapters/lunamedia" - "github.com/prebid/prebid-server/adapters/marsmedia" - "github.com/prebid/prebid-server/adapters/mediafuse" - "github.com/prebid/prebid-server/adapters/mgid" - "github.com/prebid/prebid-server/adapters/nanointeractive" - "github.com/prebid/prebid-server/adapters/ninthdecimal" - "github.com/prebid/prebid-server/adapters/nobid" - "github.com/prebid/prebid-server/adapters/onetag" - "github.com/prebid/prebid-server/adapters/openx" - "github.com/prebid/prebid-server/adapters/pubmatic" - "github.com/prebid/prebid-server/adapters/pulsepoint" - "github.com/prebid/prebid-server/adapters/rhythmone" - "github.com/prebid/prebid-server/adapters/rtbhouse" - "github.com/prebid/prebid-server/adapters/rubicon" - "github.com/prebid/prebid-server/adapters/sharethrough" - "github.com/prebid/prebid-server/adapters/smartadserver" - "github.com/prebid/prebid-server/adapters/smartrtb" - "github.com/prebid/prebid-server/adapters/smartyads" - "github.com/prebid/prebid-server/adapters/somoaudience" - "github.com/prebid/prebid-server/adapters/sonobi" - "github.com/prebid/prebid-server/adapters/sovrn" - "github.com/prebid/prebid-server/adapters/synacormedia" - "github.com/prebid/prebid-server/adapters/tappx" - "github.com/prebid/prebid-server/adapters/telaria" - "github.com/prebid/prebid-server/adapters/triplelift" - "github.com/prebid/prebid-server/adapters/triplelift_native" - "github.com/prebid/prebid-server/adapters/trustx" - "github.com/prebid/prebid-server/adapters/ucfunnel" - "github.com/prebid/prebid-server/adapters/unruly" - "github.com/prebid/prebid-server/adapters/valueimpression" - "github.com/prebid/prebid-server/adapters/verizonmedia" - "github.com/prebid/prebid-server/adapters/visx" - "github.com/prebid/prebid-server/adapters/vrtcal" - "github.com/prebid/prebid-server/adapters/yieldlab" - "github.com/prebid/prebid-server/adapters/yieldmo" - "github.com/prebid/prebid-server/adapters/yieldone" - "github.com/prebid/prebid-server/adapters/zeroclickfraud" - "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/usersync" -) - -// NewSyncerMap returns a map of all the usersyncer objects. -// The same keys should exist in this map as in the exchanges map. -// Static syncer map will be removed when adapter isolation is complete. -func NewSyncerMap(cfg *config.Configuration) map[openrtb_ext.BidderName]usersync.Usersyncer { - syncers := make(map[openrtb_ext.BidderName]usersync.Usersyncer, len(cfg.Adapters)) - - insertIntoMap(cfg, syncers, openrtb_ext.Bidder33Across, ttx.New33AcrossSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAcuityAds, acuityads.NewAcuityAdsSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdform, adform.NewAdformSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdkernel, adkernel.NewAdkernelSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdkernelAdn, adkernelAdn.NewAdkernelAdnSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdman, adman.NewAdmanSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdmixer, admixer.NewAdmixerSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdOcean, adocean.NewAdOceanSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdpone, adpone.NewadponeSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdtarget, adtarget.NewAdtargetSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdtelligent, adtelligent.NewAdtelligentSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdvangelists, advangelists.NewAdvangelistsSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAdyoulike, adyoulike.NewAdyoulikeSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAJA, aja.NewAJASyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAMX, amx.NewAMXSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAppnexus, appnexus.NewAppnexusSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAvocet, avocet.NewAvocetSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderBeachfront, beachfront.NewBeachfrontSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderBeintoo, beintoo.NewBeintooSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderBrightroll, brightroll.NewBrightrollSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderColossus, colossus.NewColossusSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderConnectAd, connectad.NewConnectAdSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderConsumable, consumable.NewConsumableSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderConversant, conversant.NewConversantSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderCpmstar, cpmstar.NewCpmstarSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderDatablocks, datablocks.NewDatablocksSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderDeepintent, deepintent.NewDeepintentSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderDmx, dmx.NewDmxSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderEmxDigital, emx_digital.NewEMXDigitalSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderEngageBDR, engagebdr.NewEngageBDRSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderEPlanning, eplanning.NewEPlanningSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderAudienceNetwork, audienceNetwork.NewFacebookSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderGamma, gamma.NewGammaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderGamoshi, gamoshi.NewGamoshiSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderGrid, grid.NewGridSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderGumGum, gumgum.NewGumGumSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderImprovedigital, improvedigital.NewImprovedigitalSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderInvibes, invibes.NewInvibesSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderIx, ix.NewIxSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderJixie, jixie.NewJixieSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderKrushmedia, krushmedia.NewKrushmediaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderLifestreet, lifestreet.NewLifestreetSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderLockerDome, lockerdome.NewLockerDomeSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderLogicad, logicad.NewLogicadSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderLunaMedia, lunamedia.NewLunaMediaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderMarsmedia, marsmedia.NewMarsmediaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderMediafuse, mediafuse.NewMediafuseSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderMgid, mgid.NewMgidSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderNanoInteractive, nanointeractive.NewNanoInteractiveSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderNinthDecimal, ninthdecimal.NewNinthDecimalSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderNoBid, nobid.NewNoBidSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderOneTag, onetag.NewSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderOpenx, openx.NewOpenxSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderPubmatic, pubmatic.NewPubmaticSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderPulsepoint, pulsepoint.NewPulsepointSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderRhythmone, rhythmone.NewRhythmoneSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderRTBHouse, rtbhouse.NewRTBHouseSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderRubicon, rubicon.NewRubiconSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSharethrough, sharethrough.NewSharethroughSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSomoaudience, somoaudience.NewSomoaudienceSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSonobi, sonobi.NewSonobiSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSovrn, sovrn.NewSovrnSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSmartAdserver, smartadserver.NewSmartadserverSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSmartRTB, smartrtb.NewSmartRTBSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSmartyAds, smartyads.NewSmartyAdsSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderSynacormedia, synacormedia.NewSynacorMediaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderTappx, tappx.NewTappxSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderTelaria, telaria.NewTelariaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderTriplelift, triplelift.NewTripleliftSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderTripleliftNative, triplelift_native.NewTripleliftSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderTrustX, trustx.NewTrustXSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderUcfunnel, ucfunnel.NewUcfunnelSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderUnruly, unruly.NewUnrulySyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderValueImpression, valueimpression.NewValueImpressionSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderVerizonMedia, verizonmedia.NewVerizonMediaSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderVisx, visx.NewVisxSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderVrtcal, vrtcal.NewVrtcalSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderYieldlab, yieldlab.NewYieldlabSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderYieldmo, yieldmo.NewYieldmoSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderYieldone, yieldone.NewYieldoneSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderZeroClickFraud, zeroclickfraud.NewZeroClickFraudSyncer) - insertIntoMap(cfg, syncers, openrtb_ext.BidderBetween, between.NewBetweenSyncer) - - return syncers -} - -func insertIntoMap(cfg *config.Configuration, syncers map[openrtb_ext.BidderName]usersync.Usersyncer, bidder openrtb_ext.BidderName, syncerFactory func(*template.Template) usersync.Usersyncer) { - lowercased := strings.ToLower(string(bidder)) - urlString := cfg.Adapters[lowercased].UserSyncURL - if urlString == "" { - glog.Warningf("adapters." + string(bidder) + ".usersync_url was not defined, and their usersync API isn't flexible enough for Prebid Server to choose a good default. No usersyncs will be performed with " + string(bidder)) - return - } - syncers[bidder] = syncerFactory(template.Must(template.New(lowercased + "_usersync_url").Parse(urlString))) -} From c454f5a00b0a82219ddfa6d05cebc4bb22e16a48 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 28 May 2021 04:37:53 -0400 Subject: [PATCH 06/50] Fixed Syncer Template Composition --- config/bidderinfo.go | 3 + endpoints/auction.go | 2 +- endpoints/cookie_sync.go | 2 +- endpoints/cookie_sync_test.go | 6 +- metrics/config/todo | 3 +- usersync/syncer.go | 104 +++++--- usersync/syncer_test.go | 450 ++++++++++++++++++++++++++++++++++ 7 files changed, 532 insertions(+), 38 deletions(-) create mode 100644 usersync/syncer_test.go diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 3fb3ce86242..4813534d77f 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -62,6 +62,9 @@ type Syncer struct { // Redirect configures an redirect endpoint for user syncing. This is also known as an image // endpoint in the Prebid.js project. Redirect *SyncerEndpoint `yaml:"redirect"` + + // SupportCORS identifies if CORS is supported for user syncing. + SupportCORS bool } // SyncerEndpoint specifies the configuration of the URL returned by the /cookie_sync endpoint diff --git a/endpoints/auction.go b/endpoints/auction.go index 6d47d2b9f51..e2e1cf5fb59 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -496,7 +496,7 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl bidder.UsersyncInfo = &pbs.UsersyncInfo{ URL: sync.URL, Type: sync.Type.String(), - SupportCORS: sync.SupportsCORS, + SupportCORS: sync.SupportCORS, } } else { glog.Errorf("Failed to get usersync info for %s: %v", syncerCode, err) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index ed55fd6da28..a914ea2d800 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -280,7 +280,7 @@ func (c *cookieSyncEndpoint) handleResponse(w http.ResponseWriter, tf usersync.S UsersyncInfo: cookieSyncResponseSync{ URL: sync.URL, Type: sync.Type.String(), - SupportCORS: sync.SupportsCORS, + SupportCORS: sync.SupportCORS, }, }) } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 327604779ca..c28b9a8e213 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -73,7 +73,7 @@ func TestNewCookieSyncEndpoint(t *testing.T) { // usersyncPrivacy func TestCookieSyncHandle(t *testing.T) { syncTypeExpected := []usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect} - sync := usersync.Sync{URL: "aURL", Type: usersync.SyncTypeRedirect, SupportsCORS: true} + sync := usersync.Sync{URL: "aURL", Type: usersync.SyncTypeRedirect, SupportCORS: true} syncer := MockSyncer{} syncer.On("GetSync", syncTypeExpected, privacy.Policies{}).Return(sync, nil).Maybe() @@ -931,12 +931,12 @@ func TestCookieSyncHandleResponse(t *testing.T) { privacyPolicies := privacy.Policies{CCPA: ccpa.Policy{Consent: "anyConsent"}} // The & in the URL is necessary to test proper JSON encoding. - syncA := usersync.Sync{URL: "https://syncA.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportsCORS: true} + syncA := usersync.Sync{URL: "https://syncA.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportCORS: true} syncerA := MockSyncer{} syncerA.On("GetSync", syncTypeExpected, privacyPolicies).Return(syncA, nil).Maybe() // The & in the URL is necessary to test proper JSON encoding. - syncB := usersync.Sync{URL: "https://syncB.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportsCORS: false} + syncB := usersync.Sync{URL: "https://syncB.com/sync?a=1&b=2", Type: usersync.SyncTypeRedirect, SupportCORS: false} syncerB := MockSyncer{} syncerB.On("GetSync", syncTypeExpected, privacyPolicies).Return(syncB, nil).Maybe() diff --git a/metrics/config/todo b/metrics/config/todo index c6b4b2f2bed..c4286f93724 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,6 +1,7 @@ todo + - finish syncer test cases + - load syncer from configs / handle parsing errors - add proper samesite behavior - cookie should just know what to do - add new cookiesync metrics to go metrics - - load syncer from configs / handle parsing errors - /setuid endpoint changes - migrate all adapter user sync configs \ No newline at end of file diff --git a/usersync/syncer.go b/usersync/syncer.go index ea829fc6494..ef0f2c2fdf2 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -30,9 +30,9 @@ type Syncer interface { // Sync represents a user sync for the user's device to perform. type Sync struct { - URL string - Type SyncType - SupportsCORS bool + URL string + Type SyncType + SupportCORS bool } type standardSyncer struct { @@ -51,15 +51,32 @@ const ( // NewSyncer creates a new Syncer instance from the provided configuration, or an error if macro // substition fails or the url specified is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { + if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { + return nil, errors.New("at least one iframe or redirect is required") + } + + // var defaultSyncType SyncType + // if syncerConfig.Default == "" { + // // error if more than 1 defined, otherwise choose that one + // } else { + // // parse. verify it's defined + // } + syncer := standardSyncer{ - key: syncerConfig.Key, + key: syncerConfig.Key, + supportCORS: syncerConfig.SupportCORS, } + // todo: default sync + if syncerConfig.IFrame != nil { var err error syncer.iframe, err = composeTemplate(syncerConfig.Key, setuidSyncTypeIFrame, hostConfig, *syncerConfig.IFrame) if err != nil { - return nil, err + return nil, fmt.Errorf("iframe: %v", err) + } + if err := validateTemplate(syncer.iframe); err != nil { + return nil, fmt.Errorf("iframe: %v", err) } } @@ -67,18 +84,24 @@ func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, var err error syncer.redirect, err = composeTemplate(syncerConfig.Key, setuidSyncTypeRedirect, hostConfig, *syncerConfig.Redirect) if err != nil { - return nil, err + return nil, fmt.Errorf("redirect: %v", err) + } + if err := validateTemplate(syncer.redirect); err != nil { + return nil, fmt.Errorf("redirect: %v", err) } } return syncer, nil } -var externalHostRegex = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) -var syncerKeyRegex = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) -var syncTypeRegex = regexp.MustCompile(`{{\s*.SyncType\s*}}`) -var userMacroRegex = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) -var redirectRegex = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) +var ( + externalHostRegex = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) + syncerKeyRegex = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) + syncTypeRegex = regexp.MustCompile(`{{\s*.SyncType\s*}}`) + userMacroRegex = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) + redirectRegex = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) + macroRegex = regexp.MustCompile(`{{.*?}}`) +) func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncerEndpoint config.SyncerEndpoint) (*template.Template, error) { redirectTemplate := syncerEndpoint.RedirectURL @@ -91,25 +114,30 @@ func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, sync externalURL = hostConfig.ExternalURL } - redirectURL := externalHostRegex.ReplaceAllString(redirectTemplate, externalURL) - redirectURL = syncerKeyRegex.ReplaceAllString(redirectURL, key) - redirectURL = syncTypeRegex.ReplaceAllString(redirectURL, syncTypeValue) - redirectURL = userMacroRegex.ReplaceAllString(redirectURL, syncerEndpoint.UserMacro) - redirectURL = url.PathEscape(redirectURL) + redirectURL := externalHostRegex.ReplaceAllLiteralString(redirectTemplate, externalURL) + redirectURL = syncerKeyRegex.ReplaceAllLiteralString(redirectURL, key) + redirectURL = syncTypeRegex.ReplaceAllLiteralString(redirectURL, syncTypeValue) + redirectURL = userMacroRegex.ReplaceAllLiteralString(redirectURL, syncerEndpoint.UserMacro) + redirectURL = escapeTemplate(redirectURL) - url := redirectRegex.ReplaceAllString(externalURL, redirectURL) + url := redirectRegex.ReplaceAllString(syncerEndpoint.URL, redirectURL) templateName := strings.ToLower(key) + "_usersync_url" - template, err := template.New(templateName).Parse(url) - if err != nil { - return nil, err - } + return template.New(templateName).Parse(url) +} - if err := validateTemplate(template); err != nil { - return nil, err +func escapeTemplate(x string) string { + escaped := strings.Builder{} + + i := 0 + for _, m := range macroRegex.FindAllStringIndex(x, -1) { + escaped.WriteString(url.QueryEscape(x[i:m[0]])) + escaped.WriteString(x[m[0]:m[1]]) + i = m[1] } + escaped.WriteString(url.QueryEscape(x[i:])) - return template, nil + return escaped.String() } func validateTemplate(template *template.Template) error { @@ -125,7 +153,7 @@ func validateTemplate(template *template.Template) error { } if !validator.IsURL(url) || !validator.IsRequestURL(url) { - return fmt.Errorf("composed url %s is invalid", url) + return fmt.Errorf("composed url \"%s\" is invalid", url) } return nil @@ -136,19 +164,25 @@ func (s standardSyncer) Key() string { } func (s standardSyncer) SupportsType(syncTypes []SyncType) bool { + supported := s.filterSupportedSyncTypes(syncTypes) + return len(supported) > 0 +} + +func (s standardSyncer) filterSupportedSyncTypes(syncTypes []SyncType) []SyncType { + supported := make([]SyncType, 0, len(syncTypes)) for _, syncType := range syncTypes { switch syncType { case SyncTypeIFrame: if s.iframe != nil { - return true + supported = append(supported, SyncTypeIFrame) } case SyncTypeRedirect: if s.redirect != nil { - return true + supported = append(supported, SyncTypeRedirect) } } } - return false + return supported } func (s standardSyncer) GetSync(syncTypes []SyncType, privacyPolicies privacy.Policies) (Sync, error) { @@ -169,9 +203,9 @@ func (s standardSyncer) GetSync(syncTypes []SyncType, privacyPolicies privacy.Po } sync := Sync{ - URL: url, - Type: syncType, - SupportsCORS: s.supportCORS, + URL: url, + Type: syncType, + SupportCORS: s.supportCORS, } return sync, nil } @@ -181,7 +215,13 @@ func (s standardSyncer) chooseSyncType(syncTypes []SyncType) (SyncType, error) { return SyncTypeUnknown, errors.New("no sync types provided") } - for _, syncType := range syncTypes { + supported := s.filterSupportedSyncTypes(syncTypes) + if len(supported) == 0 { + return SyncTypeUnknown, errors.New("no sync types supported") + } + + // prefer default type + for _, syncType := range supported { if syncType == s.defaultSyncType { return syncType, nil } diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go new file mode 100644 index 00000000000..ef57add1e9a --- /dev/null +++ b/usersync/syncer_test.go @@ -0,0 +1,450 @@ +package usersync + +import ( + "testing" + "text/template" + + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/macros" + "github.com/prebid/prebid-server/privacy" + "github.com/prebid/prebid-server/privacy/ccpa" + "github.com/prebid/prebid-server/privacy/gdpr" + "github.com/stretchr/testify/assert" +) + +// NewSyncer + +func TestComposeTemplate(t *testing.T) { + var ( + key = "anyKey" + syncTypeValue = "x" + macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} + ) + + testCases := []struct { + description string + givenHostConfig config.UserSync + givenSyncerEndpoint config.SyncerEndpoint + expectedError string + expectedRendered string + }{ + { + description: "No Composed Macros - Validated Legacy Overrides", + givenHostConfig: config.UserSync{ExternalURL: "externalURL", RedirectURL: "redirectURL"}, + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "hasNoMacros,gdpr={{.GDPR}}", + }, + expectedRendered: "hasNoMacros,gdpr=A", + }, + { + description: "All Composed Macros", + givenHostConfig: config.UserSync{ExternalURL: "externalURL", RedirectURL: "redirectURL"}, + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&f={{.SyncType}}&gdpr={{.GDPR}}&uid={{.UserMacro}}", + ExternalURL: "http://host.com", + UserMacro: "$UID$", + }, + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsetuid%3Fbidder%3DanyKey%26f%3Dx%26gdpr%3DA%26uid%3D%24UID%24", + }, + } + + for _, test := range testCases { + result, err := composeTemplate(key, syncTypeValue, test.givenHostConfig, test.givenSyncerEndpoint) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + resultRendered, err := macros.ResolveMacros(result, macroValues) + if assert.NoError(t, err, test.description+":template_render") { + assert.Equal(t, test.expectedRendered, resultRendered, test.description+":template") + } + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } + } +} + +func TestEscapeTemplate(t *testing.T) { + testCases := []struct { + description string + given string + expected string + }{ + { + description: "Just Macro", + given: "{{.Macro}}", + expected: "{{.Macro}}", + }, + { + description: "Just Text", + given: "/a", + expected: "%2Fa", + }, + { + description: "Start Only", + given: "&a{{.Macro1}}", + expected: "%26a{{.Macro1}}", + }, + { + description: "Middle Only", + given: "{{.Macro1}}&a{{.Macro2}}", + expected: "{{.Macro1}}%26a{{.Macro2}}", + }, + { + description: "End Only", + given: "{{.Macro1}}&a", + expected: "{{.Macro1}}%26a", + }, + { + description: "Start / Middle / End", + given: "&a{{.Macro1}}/b{{.Macro2}}&c", + expected: "%26a{{.Macro1}}%2Fb{{.Macro2}}%26c", + }, + { + description: "Characters In Macros Not Escaped", + given: "{{.Macro&}}", + expected: "{{.Macro&}}", + }, + } + + for _, test := range testCases { + result := escapeTemplate(test.given) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestValidateTemplate(t *testing.T) { + testCases := []struct { + description string + given *template.Template + expectedError string + }{ + { + description: "Contains Unrecognized Macro", + given: template.Must(template.New("test").Parse("invalid:{{.DoesNotExist}}")), + expectedError: "template: test:1:10: executing \"test\" at <.DoesNotExist>: can't evaluate field DoesNotExist in type macros.UserSyncTemplateParams", + }, + { + description: "Not A Url", + given: template.Must(template.New("test").Parse("not-a-url,gdpr:{{.GDPR}},gdprconsent:{{.GDPRConsent}},ccpa:{{.USPrivacy}}")), + expectedError: "composed url \"not-a-url,gdpr:anyGDPR,gdprconsent:anyGDPRConsent,ccpa:anyCCPAConsent\" is invalid", + }, + { + description: "Valid", + given: template.Must(template.New("test").Parse("http://server.com/sync?gdpr={{.GDPR}}&gdprconsent={{.GDPRConsent}}&ccpa={{.USPrivacy}}")), + expectedError: "", + }, + } + + for _, test := range testCases { + err := validateTemplate(test.given) + + if test.expectedError == "" { + assert.NoError(t, err, test.description) + } else { + assert.EqualError(t, err, test.expectedError, test.description) + } + } +} + +func TestSyncerKey(t *testing.T) { + syncer := standardSyncer{key: "foo"} + assert.Equal(t, "foo", syncer.Key()) +} + +func TestSyncerSupportsType(t *testing.T) { + endpointTemplate := template.Must(template.New("test").Parse("iframe")) + + testCases := []struct { + description string + givenSyncTypes []SyncType + givenIFrameTemplate *template.Template + givenRedirectTemplate *template.Template + expected bool + }{ + { + description: "All Available - None", + givenSyncTypes: []SyncType{}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expected: false, + }, + { + description: "All Available - One", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expected: true, + }, + { + description: "All Available - Many", + givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expected: true, + }, + { + description: "One Available - None", + givenSyncTypes: []SyncType{}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: nil, + expected: false, + }, + { + description: "One Available - One - Supported", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: nil, + expected: true, + }, + { + description: "One Available - One - Not Supported", + givenSyncTypes: []SyncType{SyncTypeRedirect}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: nil, + expected: false, + }, + { + description: "One Available - Many", + givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: nil, + expected: true, + }, + } + + for _, test := range testCases { + syncer := standardSyncer{ + iframe: test.givenIFrameTemplate, + redirect: test.givenRedirectTemplate, + } + result := syncer.SupportsType(test.givenSyncTypes) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestSyncerFilterSupportedSyncTypes(t *testing.T) { + endpointTemplate := template.Must(template.New("test").Parse("iframe")) + + testCases := []struct { + description string + givenSyncTypes []SyncType + givenIFrameTemplate *template.Template + givenRedirectTemplate *template.Template + expectedSyncTypes []SyncType + }{ + { + description: "All Available - None", + givenSyncTypes: []SyncType{}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{}, + }, + { + description: "All Available - One", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{SyncTypeIFrame}, + }, + { + description: "All Available - Many", + givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + }, + { + description: "One Available - None", + givenSyncTypes: []SyncType{}, + givenIFrameTemplate: nil, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{}, + }, + { + description: "One Available - One - Not Supported", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenIFrameTemplate: nil, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{}, + }, + { + description: "One Available - One - Supported", + givenSyncTypes: []SyncType{SyncTypeRedirect}, + givenIFrameTemplate: nil, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{SyncTypeRedirect}, + }, + { + description: "One Available - Many", + givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + givenIFrameTemplate: nil, + givenRedirectTemplate: endpointTemplate, + expectedSyncTypes: []SyncType{SyncTypeRedirect}, + }, + } + + for _, test := range testCases { + syncer := standardSyncer{ + iframe: test.givenIFrameTemplate, + redirect: test.givenRedirectTemplate, + } + result := syncer.filterSupportedSyncTypes(test.givenSyncTypes) + assert.ElementsMatch(t, test.expectedSyncTypes, result, test.description) + } +} + +func TestSyncerGetSync(t *testing.T) { + var ( + iframeTemplate = template.Must(template.New("test").Parse("iframe,gdpr:{{.GDPR}},gdprconsent:{{.GDPRConsent}},ccpa:{{.USPrivacy}}")) + redirectTemplate = template.Must(template.New("test").Parse("redirect,gdpr:{{.GDPR}},gdprconsent:{{.GDPRConsent}},ccpa:{{.USPrivacy}}")) + malformedTemplate = template.Must(template.New("test").Parse("malformed,invalid:{{.DoesNotExist}}")) + ) + + testCases := []struct { + description string + givenSyncer standardSyncer + givenSyncTypes []SyncType + givenPrivacyPolicies privacy.Policies + expectedError string + expectedSync Sync + }{ + { + description: "No Sync Types", + givenSyncer: standardSyncer{iframe: iframeTemplate, redirect: redirectTemplate}, + givenSyncTypes: []SyncType{}, + givenPrivacyPolicies: privacy.Policies{GDPR: gdpr.Policy{Signal: "A", Consent: "B"}, CCPA: ccpa.Policy{Consent: "C"}}, + expectedError: "no sync types provided", + }, + { + description: "IFrame", + givenSyncer: standardSyncer{iframe: iframeTemplate, redirect: redirectTemplate}, + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenPrivacyPolicies: privacy.Policies{GDPR: gdpr.Policy{Signal: "A", Consent: "B"}, CCPA: ccpa.Policy{Consent: "C"}}, + expectedSync: Sync{URL: "iframe,gdpr:A,gdprconsent:B,ccpa:C", Type: SyncTypeIFrame, SupportCORS: false}, + }, + { + description: "Redirect", + givenSyncer: standardSyncer{iframe: iframeTemplate, redirect: redirectTemplate}, + givenSyncTypes: []SyncType{SyncTypeRedirect}, + givenPrivacyPolicies: privacy.Policies{GDPR: gdpr.Policy{Signal: "A", Consent: "B"}, CCPA: ccpa.Policy{Consent: "C"}}, + expectedSync: Sync{URL: "redirect,gdpr:A,gdprconsent:B,ccpa:C", Type: SyncTypeRedirect, SupportCORS: false}, + }, + { + description: "Resolve Macros Error", + givenSyncer: standardSyncer{iframe: malformedTemplate}, + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenPrivacyPolicies: privacy.Policies{GDPR: gdpr.Policy{Signal: "A", Consent: "B"}, CCPA: ccpa.Policy{Consent: "C"}}, + expectedError: "template: test:1:20: executing \"test\" at <.DoesNotExist>: can't evaluate field DoesNotExist in type macros.UserSyncTemplateParams", + }, + } + + for _, test := range testCases { + result, err := test.givenSyncer.GetSync(test.givenSyncTypes, test.givenPrivacyPolicies) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedSync, result, test.description+":sync") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } + } +} + +func TestSyncerChooseSyncType(t *testing.T) { + endpointTemplate := template.Must(template.New("test").Parse("iframe")) + + testCases := []struct { + description string + givenSyncTypes []SyncType + givenDefaultSyncType SyncType + givenIFrameTemplate *template.Template + givenRedirectTemplate *template.Template + expectedError string + expectedSyncType SyncType + }{ + { + description: "None Available - Error", + givenSyncTypes: []SyncType{}, + givenDefaultSyncType: SyncTypeRedirect, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedError: "no sync types provided", + }, + { + description: "All Available - Choose Default", + givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, + givenDefaultSyncType: SyncTypeRedirect, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedSyncType: SyncTypeRedirect, + }, + { + description: "Default Not Available - Choose Other One", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenDefaultSyncType: SyncTypeRedirect, + givenIFrameTemplate: endpointTemplate, + givenRedirectTemplate: endpointTemplate, + expectedSyncType: SyncTypeIFrame, + }, + { + description: "None Supported - Error", + givenSyncTypes: []SyncType{SyncTypeIFrame}, + givenDefaultSyncType: SyncTypeRedirect, + givenIFrameTemplate: nil, + givenRedirectTemplate: endpointTemplate, + expectedError: "no sync types supported", + }, + } + + for _, test := range testCases { + syncer := standardSyncer{ + defaultSyncType: test.givenDefaultSyncType, + iframe: test.givenIFrameTemplate, + redirect: test.givenRedirectTemplate, + } + result, err := syncer.chooseSyncType(test.givenSyncTypes) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedSyncType, result, test.description+":sync_type") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } + } +} + +func TestSyncerChooseTemplate(t *testing.T) { + var ( + iframeTemplate = template.Must(template.New("test").Parse("iframe")) + redirectTemplate = template.Must(template.New("test").Parse("redirect")) + ) + + testCases := []struct { + description string + givenSyncType SyncType + expectedTemplate *template.Template + }{ + { + description: "IFrame", + givenSyncType: SyncTypeIFrame, + expectedTemplate: iframeTemplate, + }, + { + description: "Redirect", + givenSyncType: SyncTypeRedirect, + expectedTemplate: redirectTemplate, + }, + { + description: "Invalid", + givenSyncType: SyncType(42), + expectedTemplate: nil, + }, + } + + for _, test := range testCases { + syncer := standardSyncer{iframe: iframeTemplate, redirect: redirectTemplate} + result := syncer.chooseTemplate(test.givenSyncType) + assert.Equal(t, test.expectedTemplate, result, test.description) + } +} From a934a7f931a4daab17ae8b27a43c25ea108ca52e Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Fri, 28 May 2021 10:37:46 -0400 Subject: [PATCH 07/50] Finished TestComposeTemplate --- usersync/syncer.go | 15 +++++------ usersync/syncer_test.go | 55 +++++++++++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/usersync/syncer.go b/usersync/syncer.go index ef0f2c2fdf2..a145fc0d254 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -126,6 +126,7 @@ func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, sync return template.New(templateName).Parse(url) } +// escapeTemplate url encodes a string template leaving the macro tags unaffected. func escapeTemplate(x string) string { escaped := strings.Builder{} @@ -140,14 +141,14 @@ func escapeTemplate(x string) string { return escaped.String() } -func validateTemplate(template *template.Template) error { - testValues := macros.UserSyncTemplateParams{ - GDPR: "anyGDPR", - GDPRConsent: "anyGDPRConsent", - USPrivacy: "anyCCPAConsent", - } +var templateTestValues = macros.UserSyncTemplateParams{ + GDPR: "anyGDPR", + GDPRConsent: "anyGDPRConsent", + USPrivacy: "anyCCPAConsent", +} - url, err := macros.ResolveMacros(template, testValues) +func validateTemplate(template *template.Template) error { + url, err := macros.ResolveMacros(template, templateTestValues) if err != nil { return err } diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index ef57add1e9a..05872b2189a 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -12,13 +12,15 @@ import ( "github.com/stretchr/testify/assert" ) -// NewSyncer +func TestNewSyncer(t *testing.T) { +} func TestComposeTemplate(t *testing.T) { var ( key = "anyKey" syncTypeValue = "x" macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} + hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} ) testCases := []struct { @@ -29,28 +31,56 @@ func TestComposeTemplate(t *testing.T) { expectedRendered string }{ { - description: "No Composed Macros - Validated Legacy Overrides", - givenHostConfig: config.UserSync{ExternalURL: "externalURL", RedirectURL: "redirectURL"}, + description: "No Composed Macros", givenSyncerEndpoint: config.SyncerEndpoint{ - URL: "hasNoMacros,gdpr={{.GDPR}}", + URL: "hasNoComposedMacros,gdpr={{.GDPR}}", }, - expectedRendered: "hasNoMacros,gdpr=A", + expectedRendered: "hasNoComposedMacros,gdpr=A", }, { - description: "All Composed Macros", - givenHostConfig: config.UserSync{ExternalURL: "externalURL", RedirectURL: "redirectURL"}, + description: "All Composed Macros", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&f={{.SyncType}}&gdpr={{.GDPR}}&uid={{.UserMacro}}", - ExternalURL: "http://host.com", + ExternalURL: "http://syncer.com", UserMacro: "$UID$", }, - expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsetuid%3Fbidder%3DanyKey%26f%3Dx%26gdpr%3DA%26uid%3D%24UID%24", + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fsyncer.com%2Fsetuid%3Fbidder%3DanyKey%26f%3Dx%26gdpr%3DA%26uid%3D%24UID%24", + }, + { + description: "Redirect URL + External URL From Host", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + }, + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fhost", + }, + { + description: "Redirect URL From Syncer", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{.ExternalURL}}/syncer", + }, + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsyncer", + }, + { + description: "External URL From Syncer", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + ExternalURL: "http://syncer.com", + }, + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fsyncer.com%2Fhost", + }, + { + description: "Template Parse Error", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "{{malformed}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"malformed\" not defined", }, } for _, test := range testCases { - result, err := composeTemplate(key, syncTypeValue, test.givenHostConfig, test.givenSyncerEndpoint) + result, err := composeTemplate(key, syncTypeValue, hostConfig, test.givenSyncerEndpoint) if test.expectedError == "" { assert.NoError(t, err, test.description+":err") @@ -105,6 +135,11 @@ func TestEscapeTemplate(t *testing.T) { given: "{{.Macro&}}", expected: "{{.Macro&}}", }, + { + description: "Whitespace", + given: " &a {{ .Macro1 }} /b ", + expected: "+%26a+{{ .Macro1 }}+%2Fb+", + }, } for _, test := range testCases { From fe14358a02aaadd357e9aca29411979ff03f93b6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 1 Jun 2021 01:17:12 -0400 Subject: [PATCH 08/50] Finish NewSyncer Tests + Refactor --- endpoints/auction.go | 2 +- endpoints/cookie_sync.go | 2 +- endpoints/setuid.go | 16 ++- metrics/config/todo | 6 +- router/router.go | 7 +- usersync/bidderchooser_test.go | 22 +-- usersync/bidderfilter.go | 6 +- usersync/bidderfilter_test.go | 6 +- usersync/chooser.go | 21 ++- usersync/chooser_test.go | 2 +- usersync/shuffler.go | 2 +- usersync/syncer.go | 158 ++++++++++++++++----- usersync/syncer_test.go | 242 ++++++++++++++++++++++++++++++--- usersync/synctype.go | 41 +++--- usersync/synctype_test.go | 43 ++++-- 15 files changed, 446 insertions(+), 130 deletions(-) diff --git a/endpoints/auction.go b/endpoints/auction.go index e2e1cf5fb59..f16c10abe29 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -495,7 +495,7 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl if err == nil { bidder.UsersyncInfo = &pbs.UsersyncInfo{ URL: sync.URL, - Type: sync.Type.String(), + Type: string(sync.Type), SupportCORS: sync.SupportCORS, } } else { diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index a914ea2d800..9517f78bb44 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -279,7 +279,7 @@ func (c *cookieSyncEndpoint) handleResponse(w http.ResponseWriter, tf usersync.S NoCookie: true, UsersyncInfo: cookieSyncResponseSync{ URL: sync.URL, - Type: sync.Type.String(), + Type: string(sync.Type), SupportCORS: sync.SupportCORS, }, }) diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 461dbf9a0ca..e7b7a19c42e 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -54,7 +54,8 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer query := r.URL.Query() - familyName, err := getFamilyName(query, validKeyLookup) + // get key + verify we have a syncer for the key + syncerKey, err := getFamilyName(query, validKeyLookup) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -64,14 +65,14 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer so.Status = http.StatusBadRequest return } - so.Bidder = familyName + so.Bidder = syncerKey if shouldReturn, status, body := preventSyncsGDPR(query.Get("gdpr"), query.Get("gdpr_consent"), perms); shouldReturn { w.WriteHeader(status) w.Write([]byte(body)) metricsEngine.RecordUserIDSet(metrics.UserLabels{ Action: metrics.RequestActionGDPR, - Bidder: openrtb_ext.BidderName(familyName), + Bidder: openrtb_ext.BidderName(syncerKey), }) so.Status = status return @@ -81,15 +82,15 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer so.UID = uid if uid == "" { - pc.Unsync(familyName) + pc.Unsync(syncerKey) } else { - err = pc.TrySync(familyName, uid) + err = pc.TrySync(syncerKey, uid) } if err == nil { labels := metrics.UserLabels{ Action: metrics.RequestActionSet, - Bidder: openrtb_ext.BidderName(familyName), + Bidder: openrtb_ext.BidderName(syncerKey), } metricsEngine.RecordUserIDSet(labels) so.Success = true @@ -97,6 +98,9 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer setSiteCookie := siteCookieCheck(r.UserAgent()) pc.SetCookieOnResponse(w, setSiteCookie, &cfg, cookieTTL) + + // special return cotnet for image and iframe + // for image, promote the pixel we already have to a common util value }) } diff --git a/metrics/config/todo b/metrics/config/todo index c4286f93724..6e061d86b14 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,7 +1,7 @@ todo - - finish syncer test cases - - load syncer from configs / handle parsing errors + - new syncers construction + tests - add proper samesite behavior - cookie should just know what to do - add new cookiesync metrics to go metrics - /setuid endpoint changes - - migrate all adapter user sync configs \ No newline at end of file + - migrate all adapter user sync configs + - override urls from config in bidder info \ No newline at end of file diff --git a/router/router.go b/router/router.go index dbcff2d974a..033cfc0e054 100644 --- a/router/router.go +++ b/router/router.go @@ -242,8 +242,11 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } - //todo: syncers := usersyncers.NewSyncerMap(cfg) - syncers := make(map[string]usersync.Syncer) + syncers, err := usersync.NewSyncersFromBidderInfos(bidderInfos) + if err != nil { + glog.Fatal(err) // better error message? is this ok to be sepaerate from the validation pass? + } + gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, gvlVendorIDs, generalHttpClient) diff --git a/usersync/bidderchooser_test.go b/usersync/bidderchooser_test.go index da598c6c526..29c2d56b07e 100644 --- a/usersync/bidderchooser_test.go +++ b/usersync/bidderchooser_test.go @@ -54,9 +54,9 @@ func TestBidderChooserChoose(t *testing.T) { for _, test := range testCases { chooser := standardBidderChooser{shuffler: shuffler} - chosen := chooser.choose(test.givenRequested, available, test.givenCooperative) + result := chooser.choose(test.givenRequested, available, test.givenCooperative) - assert.Equal(t, test.expected, chosen, test.description) + assert.Equal(t, test.expected, result, test.description) } } @@ -110,9 +110,9 @@ func TestBidderChooserCooperative(t *testing.T) { for _, test := range testCases { chooser := standardBidderChooser{shuffler: shuffler} - chosen := chooser.chooseCooperative(test.givenRequested, available, test.givenPriorityGroups) + result := chooser.chooseCooperative(test.givenRequested, available, test.givenPriorityGroups) - assert.Equal(t, test.expected, chosen, test.description) + assert.Equal(t, test.expected, result, test.description) } } @@ -145,10 +145,10 @@ func TestBidderChooserShuffledCopy(t *testing.T) { givenCopy := copySlice(test.given) chooser := standardBidderChooser{shuffler: shuffler} - shuffled := chooser.shuffledCopy(test.given) + result := chooser.shuffledCopy(test.given) - assert.Equal(t, givenCopy, test.given, test.description+":input unchanged") - assert.Equal(t, test.expected, shuffled, test.description+":expected") + assert.Equal(t, givenCopy, test.given, test.description+":input") + assert.Equal(t, test.expected, result, test.description+":result") } } @@ -239,14 +239,14 @@ func TestBidderChooserShuffledAppend(t *testing.T) { givenBCopy := copySlice(test.givenB) chooser := standardBidderChooser{shuffler: shuffler} - shuffled := chooser.shuffledAppend(test.givenA, test.givenB) + result := chooser.shuffledAppend(test.givenA, test.givenB) - assert.Equal(t, givenBCopy, test.givenB, test.description+":append input unchanged") - assert.Equal(t, test.expected, shuffled, test.description+":expected") + assert.Equal(t, givenBCopy, test.givenB, test.description+":input") + assert.Equal(t, test.expected, result, test.description+":result") } } -// copySlice clones a slice with proper nil handling. +// copySlice returns a cloned a slice or nil. func copySlice(a []string) []string { var aCopy []string if a != nil { diff --git a/usersync/bidderfilter.go b/usersync/bidderfilter.go index cf3b985fdc0..61a5ec97528 100644 --- a/usersync/bidderfilter.go +++ b/usersync/bidderfilter.go @@ -1,6 +1,6 @@ package usersync -// BidderFilterMode represents the comparison approach of a BidderFilter. +// BidderFilterMode represents the inclusion mode of a BidderFilter. type BidderFilterMode int const ( @@ -15,8 +15,8 @@ type BidderFilter struct { mode BidderFilterMode } -// Allowed returns true if the bidder has permission per the filter settings and returns false if either -// the bidder is denied permission or if the BidderFilter is configured for an unsupported filter mode. +// Allowed returns true if the filter determines the bidder has permission and false if the bidder +// does not have permission or if the BidderFilter is set to an unsupported BidderFilterMode. func (t BidderFilter) Allowed(bidder string) bool { switch t.mode { case BidderFilterModeInclude: diff --git a/usersync/bidderfilter_test.go b/usersync/bidderfilter_test.go index de742bf14b2..afed18c44ae 100644 --- a/usersync/bidderfilter_test.go +++ b/usersync/bidderfilter_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" ) -func TestBidderFilter(t *testing.T) { - bidder := "foo" +func TestNewBidderFilter(t *testing.T) { + bidder := "a" testCases := []struct { description string @@ -78,7 +78,7 @@ func TestBidderFilter(t *testing.T) { } func TestBidderFilterForAll(t *testing.T) { - bidder := "foo" + bidder := "a" testCases := []struct { description string diff --git a/usersync/chooser.go b/usersync/chooser.go index 48c1b897bd8..d371846364a 100644 --- a/usersync/chooser.go +++ b/usersync/chooser.go @@ -1,6 +1,6 @@ package usersync -// Chooser determines which user syncers are eligible for a given user sync request. +// Chooser determines which syncers are eligible for a given request. type Chooser interface { // Choose considers bidders to sync, filters the bidders, and returns the result of the // user sync selection. @@ -21,7 +21,7 @@ func NewChooser(bidderSyncerLookup map[string]Syncer) Chooser { } } -// Request specifies a user sync request from an end user. +// Request specifies a user sync request. type Request struct { Bidders []string Cooperative Cooperative @@ -31,33 +31,33 @@ type Request struct { } // Cooperative specifies the settings for cooperative syncing for a given request, where bidders -// other than used by the publisher are considered for user syncing. +// other than those used by the publisher are considered for syncing. type Cooperative struct { Enabled bool PriorityGroups [][]string } -// Result specifies which bidders were included in the evaluation and which syncers were ultimately chosen. +// Result specifies which bidders were included in the evaluation and which syncers were chosen. type Result struct { BiddersEvaluated []BidderEvaluation Status Status SyncersChosen []SyncerChoice } -// BidderEvaluation specifies the result of a bidder evaluation for a user sync. +// BidderEvaluation specifies which bidders were considered to be synced. type BidderEvaluation struct { Bidder string SyncerKey string Status Status } -// SyncerChoice specifies a syncer chosen for a user sync. +// SyncerChoice specifies a syncer chosen. type SyncerChoice struct { Bidder string Syncer Syncer } -// Status specifies the result of a user sync. +// Status specifies the result of a sync evaluation. type Status int const ( @@ -71,7 +71,7 @@ const ( // or specific bidder syncing. StatusBlockedByGDPR - // StatusBlockedByCCPA specifiers a user's CCPA consent explicitly forbids bidder syncing. + // StatusBlockedByCCPA specifies a user's CCPA consent explicitly forbids bidder syncing. StatusBlockedByCCPA // StatusAlreadySynced specifies a user's cookie has an existing non-expired sync for a specific bidder. @@ -83,12 +83,11 @@ const ( // StatusTypeNotSupported specifies a requested sync type is not supported by a specific bidder. StatusTypeNotSupported - // StatusDuplicate specifies the requested bidders included a duplicate value either explicitly - // or through cooperative syncing. + // StatusDuplicate specifies the bidder is a duplicate or shared a syncer key with another bidder choice. StatusDuplicate ) -// Privacy determines which privacy policies should be enforced for a user sync request. +// Privacy determines which privacy policies will be enforced for a user sync request. type Privacy interface { GDPRAllowsHostCookie() bool GDPRAllowsBidderSync(bidder string) bool diff --git a/usersync/chooser_test.go b/usersync/chooser_test.go index 000bb8b2911..65bd9c70cc1 100644 --- a/usersync/chooser_test.go +++ b/usersync/chooser_test.go @@ -208,7 +208,7 @@ func TestChooserChoose(t *testing.T) { bidders := []string{"anyRequested"} biddersAvailable := []string{"anyAvailable"} for _, test := range testCases { - // set request values which don't need to be specified for each test + // set request values which don't need to be specified for each test case test.givenRequest.Bidders = bidders test.givenRequest.SyncTypeFilter = syncTypeFilter test.givenRequest.Cooperative = cooperativeConfig diff --git a/usersync/shuffler.go b/usersync/shuffler.go index 0185817ec5e..a5bdcaab5ad 100644 --- a/usersync/shuffler.go +++ b/usersync/shuffler.go @@ -7,7 +7,7 @@ type shuffler interface { shuffle(v []string) } -// randomShuffler randombly changes the order of elements in the slice. +// randomShuffler randomly changes the order of elements in the slice. type randomShuffler struct{} func (randomShuffler) shuffle(v []string) { diff --git a/usersync/syncer.go b/usersync/syncer.go index a145fc0d254..27bde1dfab4 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -14,10 +14,17 @@ import ( "github.com/prebid/prebid-server/privacy" ) +var ( + errNoSyncTypesProvided = errors.New("no sync types provided") + errNoSyncTypesSupported = errors.New("no sync types supported") + errDefaultTypeMissingIFrame = errors.New("default is set to iframe but no iframe endpoint is configured") + errDefaultTypeMissingRedirect = errors.New("default is set to redirect but no redirect endpoint is configured") +) + // Syncer represents the user sync configuration for a bidder or a shared set of bidders. type Syncer interface { - // Key is the name of the syncer as stored in the user's cookie. This is not necessarily a - // one-to-one relationship with a bidder. + // Key is the name of the syncer as stored in the user's cookie. This is often, but not + // necessarily, a one-to-one mapping with a bidder. Key() string // SupportsType returns true if the syncer supports at least one of the specified sync types. @@ -28,7 +35,7 @@ type Syncer interface { GetSync(syncTypes []SyncType, privacyPolicies privacy.Policies) (Sync, error) } -// Sync represents a user sync for the user's device to perform. +// Sync represents a user sync to be performed by the user's device. type Sync struct { URL string Type SyncType @@ -48,62 +55,90 @@ const ( setuidSyncTypeRedirect = "i" ) -// NewSyncer creates a new Syncer instance from the provided configuration, or an error if macro -// substition fails or the url specified is invalid. +var errEndpointRequired = errors.New("at least one endpoint (iframe or redirect) is required") +var errDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") + +// NewSyncer creates a new Syncer from the provided configuration, or an error if macro substition +// fails or an endpoint url is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { - return nil, errors.New("at least one iframe or redirect is required") + return nil, errEndpointRequired } - // var defaultSyncType SyncType - // if syncerConfig.Default == "" { - // // error if more than 1 defined, otherwise choose that one - // } else { - // // parse. verify it's defined - // } - syncer := standardSyncer{ key: syncerConfig.Key, supportCORS: syncerConfig.SupportCORS, } - // todo: default sync + if defaultSyncType, err := resolveDefaultSyncType(syncerConfig); err != nil { + return nil, err + } else { + syncer.defaultSyncType = defaultSyncType + } if syncerConfig.IFrame != nil { var err error - syncer.iframe, err = composeTemplate(syncerConfig.Key, setuidSyncTypeIFrame, hostConfig, *syncerConfig.IFrame) + syncer.iframe, err = buildTemplate(syncerConfig.Key, setuidSyncTypeIFrame, hostConfig, *syncerConfig.IFrame) if err != nil { - return nil, fmt.Errorf("iframe: %v", err) + return nil, fmt.Errorf("iframe %v", err) } if err := validateTemplate(syncer.iframe); err != nil { - return nil, fmt.Errorf("iframe: %v", err) + return nil, fmt.Errorf("iframe %v", err) } } if syncerConfig.Redirect != nil { var err error - syncer.redirect, err = composeTemplate(syncerConfig.Key, setuidSyncTypeRedirect, hostConfig, *syncerConfig.Redirect) + syncer.redirect, err = buildTemplate(syncerConfig.Key, setuidSyncTypeRedirect, hostConfig, *syncerConfig.Redirect) if err != nil { - return nil, fmt.Errorf("redirect: %v", err) + return nil, fmt.Errorf("redirect %v", err) } if err := validateTemplate(syncer.redirect); err != nil { - return nil, fmt.Errorf("redirect: %v", err) + return nil, fmt.Errorf("redirect %v", err) } } return syncer, nil } +func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { + if syncerConfig.Default == "" { + if syncerConfig.IFrame != nil && syncerConfig.Redirect != nil { + return SyncTypeUnknown, errDefaultSyncTypeRequired + } else if syncerConfig.IFrame != nil { + return SyncTypeIFrame, nil + } else { + return SyncTypeRedirect, nil + } + } + + if syncType, err := SyncTypeParse(syncerConfig.Default); err == nil { + switch syncType { + case SyncTypeIFrame: + if syncerConfig.IFrame == nil { + return SyncTypeUnknown, errDefaultTypeMissingIFrame + } + case SyncTypeRedirect: + if syncerConfig.Redirect == nil { + return SyncTypeUnknown, errDefaultTypeMissingRedirect + } + } + return syncType, nil + } + + return SyncTypeUnknown, fmt.Errorf("invalid default sync type '%s'", syncerConfig.Default) +} + var ( - externalHostRegex = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) - syncerKeyRegex = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) - syncTypeRegex = regexp.MustCompile(`{{\s*.SyncType\s*}}`) - userMacroRegex = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) - redirectRegex = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) - macroRegex = regexp.MustCompile(`{{.*?}}`) + macroRegexExternalHost = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) + macroRegexSyncerKey = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) + macroRegexSyncType = regexp.MustCompile(`{{\s*.SyncType\s*}}`) + macroRegexUserMacro = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) + macroRegexRedirect = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) + macroRegex = regexp.MustCompile(`{{.*?}}`) ) -func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncerEndpoint config.SyncerEndpoint) (*template.Template, error) { +func buildTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncerEndpoint config.SyncerEndpoint) (*template.Template, error) { redirectTemplate := syncerEndpoint.RedirectURL if redirectTemplate == "" { redirectTemplate = hostConfig.RedirectURL @@ -114,13 +149,13 @@ func composeTemplate(key, syncTypeValue string, hostConfig config.UserSync, sync externalURL = hostConfig.ExternalURL } - redirectURL := externalHostRegex.ReplaceAllLiteralString(redirectTemplate, externalURL) - redirectURL = syncerKeyRegex.ReplaceAllLiteralString(redirectURL, key) - redirectURL = syncTypeRegex.ReplaceAllLiteralString(redirectURL, syncTypeValue) - redirectURL = userMacroRegex.ReplaceAllLiteralString(redirectURL, syncerEndpoint.UserMacro) + redirectURL := macroRegexExternalHost.ReplaceAllLiteralString(redirectTemplate, externalURL) + redirectURL = macroRegexSyncerKey.ReplaceAllLiteralString(redirectURL, key) + redirectURL = macroRegexSyncType.ReplaceAllLiteralString(redirectURL, syncTypeValue) + redirectURL = macroRegexUserMacro.ReplaceAllLiteralString(redirectURL, syncerEndpoint.UserMacro) redirectURL = escapeTemplate(redirectURL) - url := redirectRegex.ReplaceAllString(syncerEndpoint.URL, redirectURL) + url := macroRegexRedirect.ReplaceAllString(syncerEndpoint.URL, redirectURL) templateName := strings.ToLower(key) + "_usersync_url" return template.New(templateName).Parse(url) @@ -154,7 +189,7 @@ func validateTemplate(template *template.Template) error { } if !validator.IsURL(url) || !validator.IsRequestURL(url) { - return fmt.Errorf("composed url \"%s\" is invalid", url) + return fmt.Errorf(`composed url: "%s" is invalid`, url) } return nil @@ -213,12 +248,12 @@ func (s standardSyncer) GetSync(syncTypes []SyncType, privacyPolicies privacy.Po func (s standardSyncer) chooseSyncType(syncTypes []SyncType) (SyncType, error) { if len(syncTypes) == 0 { - return SyncTypeUnknown, errors.New("no sync types provided") + return SyncTypeUnknown, errNoSyncTypesProvided } supported := s.filterSupportedSyncTypes(syncTypes) if len(supported) == 0 { - return SyncTypeUnknown, errors.New("no sync types supported") + return SyncTypeUnknown, errNoSyncTypesSupported } // prefer default type @@ -241,3 +276,56 @@ func (s standardSyncer) chooseTemplate(syncType SyncType) *template.Template { return nil } } + +type bidderInfoWithName struct { + bidderName string + bidderInfo config.BidderInfo +} + +// func NewSyncersFromBidderInfos(hostConfig config.UserSync, bidderInfos config.BidderInfos) (map[string]Syncer, error) { +// bidderInfosBySyncerKey := make(map[string][]bidderInfoWithName) +// for bidderName, bidderInfo := range bidderInfos { +// s := bidderInfo.Syncer +// if s != nil && (s.IFrame != nil || s.Redirect != nil) { +// bidderInfosBySyncerKey[s.Key] = append(bidderInfosBySyncerKey[s.Key], bidderInfoWithName{bidderName, bidderInfo}) +// } +// } + +// // build syncers +// syncersByBidder := map[string]Syncer{} +// var errs []error +// for _, bidderInfos := range bidderInfosBySyncerKey { +// syncerConfig, err := getPrimarySyncerConfig(bidderInfos) +// if err != nil { +// errs = append(errs, err) +// continue +// } + +// syncer, err := NewSyncer(hostConfig, syncerConfig) +// if err != nil { +// errs = append(errs, err) +// continue +// } + +// for _, b := range bidderInfos { +// syncersByBidder[b.bidderName] = syncer +// } +// } + +// if len(errs) > 0 { +// return nil, errortypes.NewAggregateError("msg", errs) +// } + +// return syncersByBidder, nil +// } + +// func getPrimarySyncerConfig(bidderInfos []bidderInfoWithName) (*config.Syncer, error) { +// if len(bidderInfos) == 1 { +// return bidderInfos[0].bidderInfo.Syncer, nil +// } + +// // if multiple, ensure just one has endpoints defined + return it +// // if not just one, return error + +// return nil, nil +// } diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 05872b2189a..1eaa6a111b9 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -13,14 +13,212 @@ import ( ) func TestNewSyncer(t *testing.T) { + var ( + key = "a" + supportCORS = true + hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} + macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} + iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} + redirectConfig = &config.SyncerEndpoint{URL: "https://bidder.com/redirect?redirect={{.RedirectURL}}"} + errParseConfig = &config.SyncerEndpoint{URL: "{{malformed}}"} + errInvalidConfig = &config.SyncerEndpoint{URL: "notAURL:{{.RedirectURL}}"} + ) + + testCases := []struct { + description string + givenDefault string + givenIFrameConfig *config.SyncerEndpoint + givenRedirectConfig *config.SyncerEndpoint + expectedError string + expectedDefault SyncType + expectedIFrame string + expectedRedirect string + }{ + { + description: "No Endpoints", + givenDefault: "", + givenIFrameConfig: nil, + givenRedirectConfig: nil, + expectedDefault: SyncTypeIFrame, + expectedError: "at least one endpoint (iframe or redirect) is required", + }, + { + description: "Resolve Default Sync Type Error ", + givenDefault: "", + givenIFrameConfig: iframeConfig, + givenRedirectConfig: redirectConfig, + expectedError: "default sync type is required when more then one sync endpoint is configured", + }, + { + description: "IFrame & Redirect Endpoints", + givenDefault: "iframe", + givenIFrameConfig: iframeConfig, + givenRedirectConfig: redirectConfig, + expectedDefault: SyncTypeIFrame, + expectedIFrame: "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fhost", + expectedRedirect: "https://bidder.com/redirect?redirect=http%3A%2F%2Fhost.com%2Fhost", + }, + { + description: "IFrame - Parse Error", + givenDefault: "iframe", + givenIFrameConfig: errParseConfig, + givenRedirectConfig: nil, + expectedDefault: SyncTypeIFrame, + expectedError: "iframe template: a_usersync_url:1: function \"malformed\" not defined", + }, + { + description: "IFrame - Validation Error", + givenDefault: "iframe", + givenIFrameConfig: errInvalidConfig, + givenRedirectConfig: nil, + expectedDefault: SyncTypeIFrame, + expectedError: "iframe composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", + }, + { + description: "Redirect - Parse Error", + givenDefault: "redirect", + givenIFrameConfig: nil, + givenRedirectConfig: errParseConfig, + expectedDefault: SyncTypeRedirect, + expectedError: "redirect template: a_usersync_url:1: function \"malformed\" not defined", + }, + { + description: "Redirect - Validation Error", + givenDefault: "redirect", + givenIFrameConfig: nil, + givenRedirectConfig: errInvalidConfig, + expectedDefault: SyncTypeRedirect, + expectedError: "redirect composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", + }, + } + + for _, test := range testCases { + syncerConfig := config.Syncer{ + Key: key, + SupportCORS: supportCORS, + Default: test.givenDefault, + IFrame: test.givenIFrameConfig, + Redirect: test.givenRedirectConfig, + } + + result, err := NewSyncer(hostConfig, syncerConfig) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + if assert.IsType(t, standardSyncer{}, result, test.description+":result_type") { + result := result.(standardSyncer) + assert.Equal(t, key, result.key, test.description+":key") + assert.Equal(t, supportCORS, result.supportCORS, test.description+":cors") + assert.Equal(t, test.expectedDefault, result.defaultSyncType, test.description+":default_sync") + + if test.expectedIFrame == "" { + assert.Nil(t, result.iframe, test.description+":iframe") + } else { + iframeRendered, err := macros.ResolveMacros(result.iframe, macroValues) + if assert.NoError(t, err, test.description+":iframe_render") { + assert.Equal(t, test.expectedIFrame, iframeRendered, test.description+":iframe") + } + } + + if test.expectedRedirect == "" { + assert.Nil(t, result.redirect, test.description+":redirect") + } else { + redirectRendered, err := macros.ResolveMacros(result.redirect, macroValues) + if assert.NoError(t, err, test.description+":redirect_render") { + assert.Equal(t, test.expectedRedirect, redirectRendered, test.description+":redirect") + } + } + } + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result) + } + } } -func TestComposeTemplate(t *testing.T) { +func TestResolveDefaultSyncType(t *testing.T) { + anyEndpoint := &config.SyncerEndpoint{} + + testCases := []struct { + description string + givenConfig config.Syncer + expectedSyncType SyncType + expectedError string + }{ + { + description: "IFrame & Redirect - IFrame Default", + givenConfig: config.Syncer{Default: "iframe", IFrame: anyEndpoint, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeIFrame, + }, + { + description: "IFrame & Redirect - Redirect Default", + givenConfig: config.Syncer{Default: "redirect", IFrame: anyEndpoint, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeRedirect, + }, + { + description: "IFrame & Redirect - No Default", + givenConfig: config.Syncer{Default: "", IFrame: anyEndpoint, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeUnknown, + expectedError: "default sync type is required when more then one sync endpoint is configured", + }, + { + description: "IFrame & Redirect - Invalid Default", + givenConfig: config.Syncer{Default: "invalid", IFrame: anyEndpoint, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeUnknown, + expectedError: "invalid default sync type 'invalid'", + }, + { + description: "IFrame Only - IFrame Default", + givenConfig: config.Syncer{Default: "iframe", IFrame: anyEndpoint, Redirect: nil}, + expectedSyncType: SyncTypeIFrame, + }, + { + description: "IFrame Only - No Default", + givenConfig: config.Syncer{Default: "", IFrame: anyEndpoint, Redirect: nil}, + expectedSyncType: SyncTypeIFrame, + }, + { + description: "IFrame Only - Redirect Default", + givenConfig: config.Syncer{Default: "redirect", IFrame: anyEndpoint, Redirect: nil}, + expectedSyncType: SyncTypeUnknown, + expectedError: "default is set to redirect but no redirect endpoint is configured", + }, + { + description: "Redirect Only - Redirect Default", + givenConfig: config.Syncer{Default: "redirect", IFrame: nil, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeRedirect, + }, + { + description: "IFrame Only - No Default", + givenConfig: config.Syncer{Default: "", IFrame: nil, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeRedirect, + }, + { + description: "IFrame Only - IFrame Default", + givenConfig: config.Syncer{Default: "iframe", IFrame: nil, Redirect: anyEndpoint}, + expectedSyncType: SyncTypeUnknown, + expectedError: "default is set to iframe but no iframe endpoint is configured", + }, + } + + for _, test := range testCases { + result, err := resolveDefaultSyncType(test.givenConfig) + + assert.Equal(t, test.expectedSyncType, result, test.description+":result") + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } + } +} + +func TestBuildTemplate(t *testing.T) { var ( key = "anyKey" syncTypeValue = "x" - macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} + macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} ) testCases := []struct { @@ -63,7 +261,7 @@ func TestComposeTemplate(t *testing.T) { expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsyncer", }, { - description: "External URL From Syncer", + description: "External URL From Host", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", ExternalURL: "http://syncer.com", @@ -80,7 +278,7 @@ func TestComposeTemplate(t *testing.T) { } for _, test := range testCases { - result, err := composeTemplate(key, syncTypeValue, hostConfig, test.givenSyncerEndpoint) + result, err := buildTemplate(key, syncTypeValue, hostConfig, test.givenSyncerEndpoint) if test.expectedError == "" { assert.NoError(t, err, test.description+":err") @@ -101,12 +299,12 @@ func TestEscapeTemplate(t *testing.T) { expected string }{ { - description: "Just Macro", + description: "Macro Only", given: "{{.Macro}}", expected: "{{.Macro}}", }, { - description: "Just Text", + description: "Text Only", given: "/a", expected: "%2Fa", }, @@ -131,12 +329,12 @@ func TestEscapeTemplate(t *testing.T) { expected: "%26a{{.Macro1}}%2Fb{{.Macro2}}%26c", }, { - description: "Characters In Macros Not Escaped", + description: "Characters In Macro Not Escaped", given: "{{.Macro&}}", expected: "{{.Macro&}}", }, { - description: "Whitespace", + description: "Macro Whitespace Insensitive", given: " &a {{ .Macro1 }} /b ", expected: "+%26a+{{ .Macro1 }}+%2Fb+", }, @@ -162,7 +360,7 @@ func TestValidateTemplate(t *testing.T) { { description: "Not A Url", given: template.Must(template.New("test").Parse("not-a-url,gdpr:{{.GDPR}},gdprconsent:{{.GDPRConsent}},ccpa:{{.USPrivacy}}")), - expectedError: "composed url \"not-a-url,gdpr:anyGDPR,gdprconsent:anyGDPRConsent,ccpa:anyCCPAConsent\" is invalid", + expectedError: "composed url: \"not-a-url,gdpr:anyGDPR,gdprconsent:anyGDPRConsent,ccpa:anyCCPAConsent\" is invalid", }, { description: "Valid", @@ -183,8 +381,8 @@ func TestValidateTemplate(t *testing.T) { } func TestSyncerKey(t *testing.T) { - syncer := standardSyncer{key: "foo"} - assert.Equal(t, "foo", syncer.Key()) + syncer := standardSyncer{key: "a"} + assert.Equal(t, "a", syncer.Key()) } func TestSyncerSupportsType(t *testing.T) { @@ -195,56 +393,56 @@ func TestSyncerSupportsType(t *testing.T) { givenSyncTypes []SyncType givenIFrameTemplate *template.Template givenRedirectTemplate *template.Template - expected bool + expectedResult bool }{ { description: "All Available - None", givenSyncTypes: []SyncType{}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: endpointTemplate, - expected: false, + expectedResult: false, }, { description: "All Available - One", givenSyncTypes: []SyncType{SyncTypeIFrame}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: endpointTemplate, - expected: true, + expectedResult: true, }, { description: "All Available - Many", givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: endpointTemplate, - expected: true, + expectedResult: true, }, { description: "One Available - None", givenSyncTypes: []SyncType{}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: nil, - expected: false, + expectedResult: false, }, { description: "One Available - One - Supported", givenSyncTypes: []SyncType{SyncTypeIFrame}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: nil, - expected: true, + expectedResult: true, }, { description: "One Available - One - Not Supported", givenSyncTypes: []SyncType{SyncTypeRedirect}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: nil, - expected: false, + expectedResult: false, }, { description: "One Available - Many", givenSyncTypes: []SyncType{SyncTypeIFrame, SyncTypeRedirect}, givenIFrameTemplate: endpointTemplate, givenRedirectTemplate: nil, - expected: true, + expectedResult: true, }, } @@ -254,7 +452,7 @@ func TestSyncerSupportsType(t *testing.T) { redirect: test.givenRedirectTemplate, } result := syncer.SupportsType(test.givenSyncTypes) - assert.Equal(t, test.expected, result, test.description) + assert.Equal(t, test.expectedResult, result, test.description) } } @@ -366,7 +564,7 @@ func TestSyncerGetSync(t *testing.T) { expectedSync: Sync{URL: "redirect,gdpr:A,gdprconsent:B,ccpa:C", Type: SyncTypeRedirect, SupportCORS: false}, }, { - description: "Resolve Macros Error", + description: "Macro Error", givenSyncer: standardSyncer{iframe: malformedTemplate}, givenSyncTypes: []SyncType{SyncTypeIFrame}, givenPrivacyPolicies: privacy.Policies{GDPR: gdpr.Policy{Signal: "A", Consent: "B"}, CCPA: ccpa.Policy{Consent: "C"}}, @@ -472,7 +670,7 @@ func TestSyncerChooseTemplate(t *testing.T) { }, { description: "Invalid", - givenSyncType: SyncType(42), + givenSyncType: SyncType("invalid"), expectedTemplate: nil, }, } diff --git a/usersync/synctype.go b/usersync/synctype.go index 3a1db225eca..c9e59f360de 100644 --- a/usersync/synctype.go +++ b/usersync/synctype.go @@ -1,21 +1,39 @@ package usersync -// SyncType specifies the protocol used to perform a user sync. -type SyncType int +import ( + "fmt" + "strings" +) + +// SyncType specifies the mechanism used to perform a user sync. +type SyncType string const ( // SyncTypeUnknown specifies the user sync type is invalid or not specified. - SyncTypeUnknown SyncType = -1 + SyncTypeUnknown SyncType = "" - // SyncTypeIFrame specifies the user sync is to be performed within an HTML `iframe` + // SyncTypeIFrame specifies the user sync is to be performed within an HTML iframe // and to expect the server to return a valid HTML page with an embedded script. - SyncTypeIFrame SyncType = 0 + SyncTypeIFrame SyncType = "iframe" // SyncTypeRedirect specifies the user sync is to be performed within an HTML image // and to expect the server to return a 302 redirect. - SyncTypeRedirect SyncType = 1 + SyncTypeRedirect SyncType = "redirect" ) +// SyncTypeParse returns the SyncType parsed from a string, case insensitive. +func SyncTypeParse(v string) (SyncType, error) { + if strings.EqualFold(v, string(SyncTypeIFrame)) { + return SyncTypeIFrame, nil + } + + if strings.EqualFold(v, string(SyncTypeRedirect)) { + return SyncTypeRedirect, nil + } + + return SyncTypeUnknown, fmt.Errorf("invalid sync type `%s`", v) +} + // SyncTypeFilter determines which sync types, if any, the bidder is permitted to use. type SyncTypeFilter struct { IFrame BidderFilter @@ -36,14 +54,3 @@ func (t SyncTypeFilter) ForBidder(bidder string) []SyncType { return syncTypes } - -func (t SyncType) String() string { - switch t { - case SyncTypeIFrame: - return "iframe" - case SyncTypeRedirect: - return "redirect" - default: - return "" - } -} diff --git a/usersync/synctype_test.go b/usersync/synctype_test.go index 8d89c66363f..26161295dd1 100644 --- a/usersync/synctype_test.go +++ b/usersync/synctype_test.go @@ -51,31 +51,48 @@ func TestSyncTypeFilter(t *testing.T) { } } -func TestString(t *testing.T) { +func TestSyncTypeParse(t *testing.T) { testCases := []struct { - description string - given SyncType - expected string + description string + given string + expected SyncType + expectedError string }{ { description: "IFrame", - given: SyncTypeIFrame, - expected: "iframe", + given: "iframe", + expected: SyncTypeIFrame, + }, + { + description: "IFrame - Case Insensitive", + given: "iFrAmE", + expected: SyncTypeIFrame, }, { description: "Redirect", - given: SyncTypeRedirect, - expected: "redirect", + given: "redirect", + expected: SyncTypeRedirect, }, { - description: "Unknown", - given: SyncType(42), - expected: "", + description: "Redirect - Case Insensitive", + given: "ReDiReCt", + expected: SyncTypeRedirect, + }, + { + description: "Invalid", + given: "invalid", + expectedError: "invalid sync type `invalid`", }, } for _, test := range testCases { - result := test.given.String() - assert.Equal(t, test.expected, result) + result, err := SyncTypeParse(test.given) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expected, result, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + } } } From 86a863543726c052e9bc96dd34aeef993d455a01 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 01:00:46 -0400 Subject: [PATCH 09/50] Finished Syncers Builder + First Pass SetUID --- config/adapter.go | 4 + endpoints/cookie_sync_test.go | 7 +- endpoints/events/event.go | 29 ++---- endpoints/setuid.go | 65 ++++++++++--- endpoints/setuid_test.go | 96 ++++++++++--------- metrics/config/todo | 11 ++- router/router.go | 6 +- usersync/chooser_test.go | 4 + usersync/syncer.go | 72 ++++---------- usersync/syncer_test.go | 35 +++++-- usersync/syncersbuilder.go | 88 +++++++++++++++++ usersync/syncersbuilder_test.go | 163 ++++++++++++++++++++++++++++++++ util/httputil/pixel.go | 16 ++++ 13 files changed, 440 insertions(+), 156 deletions(-) create mode 100644 usersync/syncersbuilder.go create mode 100644 usersync/syncersbuilder_test.go create mode 100644 util/httputil/pixel.go diff --git a/config/adapter.go b/config/adapter.go index 845eafc0d49..666fbcddb3d 100644 --- a/config/adapter.go +++ b/config/adapter.go @@ -12,6 +12,10 @@ type Adapter struct { Disabled bool `mapstructure:"disabled"` Endpoint string `mapstructure:"endpoint"` ExtraAdapterInfo string `mapstructure:"extra_info"` + // todo: bidder info overrides + + // needed for backwards compatibility + UserSyncURL string `mapstructure:"usersync_url"` // needed for Rubicon XAPI AdapterXAPI `mapstructure:"xapi"` diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index c28b9a8e213..9ef47bde642 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -24,8 +24,6 @@ import ( "github.com/stretchr/testify/mock" ) -// end to end test? - func TestNewCookieSyncEndpoint(t *testing.T) { var ( syncers = map[string]usersync.Syncer{"a": &MockSyncer{}} @@ -1256,6 +1254,11 @@ func (m *MockSyncer) Key() string { return args.String(0) } +func (m *MockSyncer) DefaultSyncType() usersync.SyncType { + args := m.Called() + return args.Get(0).(usersync.SyncType) +} + func (m *MockSyncer) SupportsType(syncTypes []usersync.SyncType) bool { args := m.Called(syncTypes) return args.Bool(0) diff --git a/endpoints/events/event.go b/endpoints/events/event.go index fe178d8f271..4f033a195a7 100644 --- a/endpoints/events/event.go +++ b/endpoints/events/event.go @@ -4,16 +4,18 @@ import ( "context" "errors" "fmt" + "net/http" + "net/url" + "strconv" + "time" + "github.com/julienschmidt/httprouter" accountService "github.com/prebid/prebid-server/account" "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/stored_requests" - "net/http" - "net/url" - "strconv" - "time" + "github.com/prebid/prebid-server/util/httputil" ) const ( @@ -30,26 +32,11 @@ const ( AnalyticsParameter = "x" ) -var trackingPixelPng = &trackingPixel{ - Content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, - 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, - 0x89, 0x00, 0x00, 0x00, 0x04, 0x73, 0x42, 0x49, 0x54, 0x08, 0x08, 0x08, 0x08, 0x7C, 0x08, 0x64, 0x88, - 0x00, 0x00, 0x00, 0x0D, 0x49, 0x44, 0x41, 0x54, 0x08, 0x99, 0x63, 0x60, 0x60, 0x60, 0x60, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x01, 0x87, 0xA1, 0x4E, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, - 0x42, 0x60, 0x82}, - ContentType: "image/png", -} - -type trackingPixel struct { - Content []byte `json:"content,omitempty"` - ContentType string `json:"content_type,omitempty"` -} - type eventEndpoint struct { Accounts stored_requests.AccountFetcher Analytics analytics.PBSAnalyticsModule Cfg *config.Configuration - TrackingPixel *trackingPixel + TrackingPixel *httputil.OneByOnePixel } func NewEventEndpoint(cfg *config.Configuration, accounts stored_requests.AccountFetcher, analytics analytics.PBSAnalyticsModule) httprouter.Handle { @@ -57,7 +44,7 @@ func NewEventEndpoint(cfg *config.Configuration, accounts stored_requests.Accoun Accounts: accounts, Analytics: analytics, Cfg: cfg, - TrackingPixel: trackingPixelPng, + TrackingPixel: &httputil.OneByOnePixelPNG, } return ee.Handle diff --git a/endpoints/setuid.go b/endpoints/setuid.go index e7b7a19c42e..8467936e1d8 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -16,6 +16,7 @@ import ( "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/httputil" ) const ( @@ -29,11 +30,6 @@ const ( func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour - validKeyLookup := make(map[string]struct{}) - for _, s := range syncers { - validKeyLookup[s.Key()] = struct{}{} - } - return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { so := analytics.SetUIDObject{ Status: http.StatusOK, @@ -55,7 +51,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer query := r.URL.Query() // get key + verify we have a syncer for the key - syncerKey, err := getFamilyName(query, validKeyLookup) + syncerKey, err := getSyncerKey(query, syncers) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -99,24 +95,63 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer setSiteCookie := siteCookieCheck(r.UserAgent()) pc.SetCookieOnResponse(w, setSiteCookie, &cfg, cookieTTL) - // special return cotnet for image and iframe - // for image, promote the pixel we already have to a common util value + responseFormat, err := getResponseFormat(query, syncers[syncerKey]) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + metricsEngine.RecordUserIDSet(metrics.UserLabels{ + Action: metrics.RequestActionErr, + }) + so.Status = http.StatusBadRequest + return + } + + switch responseFormat { + case "i": + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-Type", httputil.OneByOnePixelPNG.ContentType) + //w.Header().Add("Content-Length", strconv.Itoa(len(httputil.TrackingPixelPNG.Content))) - is this automatic? + w.Write(httputil.OneByOnePixelPNG.Content) + case "b": + w.Header().Add("Content-Type", "text/html") + // set content length to 0 - is this automatic? + } }) } -func getFamilyName(query url.Values, validFamilyNameMap map[string]struct{}) (string, error) { - // The family name is bound to the 'bidder' query param. In most cases, these values are the same. - familyName := query.Get("bidder") +func getSyncerKey(query url.Values, syncers map[string]usersync.Syncer) (string, error) { + key := query.Get("bidder") - if familyName == "" { + if key == "" { return "", errors.New(`"bidder" query param is required`) } - if _, ok := validFamilyNameMap[familyName]; !ok { + if _, ok := syncers[key]; !ok { return "", errors.New("The bidder name provided is not supported by Prebid Server") } - return familyName, nil + return key, nil +} + +func getResponseFormat(query url.Values, syncer usersync.Syncer) (string, error) { + format := query.Get("f") + + if format == "" { + switch syncer.DefaultSyncType() { + case usersync.SyncTypeIFrame: + return "b", nil + case usersync.SyncTypeRedirect: + return "i", nil + default: + return "", errors.New("invalid default sync type") + } + } + + if !strings.EqualFold(format, "b") && !strings.EqualFold(format, "i") { + return "", errors.New("invalid value") + } + + return strings.ToLower(format), nil } // siteCookieCheck scans the input User Agent string to check if browser is Chrome and browser version is greater than the minimum version for adding the SameSite cookie attribute @@ -180,5 +215,5 @@ func preventSyncsGDPR(gdprEnabled string, gdprConsent string, perms gdpr.Permiss return false, 0, "" } - return true, http.StatusOK, "The gdpr_consent string prevents cookies from being saved" + return true, http.StatusUnavailableForLegalReasons, "The gdpr_consent string prevents cookies from being saved" } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index db80f6ff97e..ad869dccae4 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -5,7 +5,6 @@ import ( "errors" "net/http" "net/http/httptest" - "net/url" "regexp" "testing" "time" @@ -319,51 +318,52 @@ func TestSiteCookieCheck(t *testing.T) { } func TestGetFamilyName(t *testing.T) { - testCases := []struct { - urlValues url.Values - expectedName string - expectedError string - description string - }{ - { - urlValues: url.Values{"bidder": []string{"valid"}}, - expectedName: "valid", - description: "Should return no error for valid family name", - }, - { - urlValues: url.Values{"bidder": []string{"VALID"}}, - expectedError: "The bidder name provided is not supported by Prebid Server", - description: "Should return error for different case", - }, - { - urlValues: url.Values{"bidder": []string{"invalid"}}, - expectedError: "The bidder name provided is not supported by Prebid Server", - description: "Should return an error for unsupported bidder", - }, - { - urlValues: url.Values{"bidder": []string{}}, - expectedError: `"bidder" query param is required`, - description: "Should return an error for empty bidder name", - }, - { - urlValues: url.Values{}, - expectedError: `"bidder" query param is required`, - description: "Should return an error for missing bidder name", - }, - } - - for _, test := range testCases { - - name, err := getFamilyName(test.urlValues, map[string]struct{}{"valid": {}}) - - assert.Equal(t, test.expectedName, name, test.description) - - if test.expectedError != "" { - assert.EqualError(t, err, test.expectedError, test.description) - } else { - assert.NoError(t, err, test.description) - } - } + // testCases := []struct { + // urlValues url.Values + // expectedName string + // expectedError string + // description string + // }{ + // { + // urlValues: url.Values{"bidder": []string{"valid"}}, + // expectedName: "valid", + // description: "Should return no error for valid family name", + // }, + // { + // urlValues: url.Values{"bidder": []string{"VALID"}}, + // expectedError: "The bidder name provided is not supported by Prebid Server", + // description: "Should return error for different case", + // }, + // { + // urlValues: url.Values{"bidder": []string{"invalid"}}, + // expectedError: "The bidder name provided is not supported by Prebid Server", + // description: "Should return an error for unsupported bidder", + // }, + // { + // urlValues: url.Values{"bidder": []string{}}, + // expectedError: `"bidder" query param is required`, + // description: "Should return an error for empty bidder name", + // }, + // { + // urlValues: url.Values{}, + // expectedError: `"bidder" query param is required`, + // description: "Should return an error for missing bidder name", + // }, + // } + + //for _, test := range testCases { + + // todo + // name, err := getSyncerKey(test.urlValues, map[string]struct{}{"valid": {}}) + + // assert.Equal(t, test.expectedName, name, test.description) + + // if test.expectedError != "" { + // assert.EqualError(t, err, test.expectedError, test.description) + // } else { + // assert.NoError(t, err, test.description) + // } + //} } func assertHasSyncs(t *testing.T, testCase string, resp *httptest.ResponseRecorder, syncs map[string]string) { @@ -463,6 +463,10 @@ func (s fakeSyncer) Key() string { return s.key } +func (s fakeSyncer) DefaultSyncType() usersync.SyncType { + return usersync.SyncTypeIFrame +} + func (s fakeSyncer) SupportsType(syncTypes []usersync.SyncType) bool { return true } diff --git a/metrics/config/todo b/metrics/config/todo index 6e061d86b14..ad0a58776ef 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,7 +1,10 @@ todo - - new syncers construction + tests - - add proper samesite behavior - cookie should just know what to do + - /setuid endpoint tests + + - override urls from config in bidder info - add new cookiesync metrics to go metrics - - /setuid endpoint changes + - add proper samesite behavior - cookie should just know what to do + + [open pr] + - migrate all adapter user sync configs - - override urls from config in bidder info \ No newline at end of file diff --git a/router/router.go b/router/router.go index 033cfc0e054..882bd86f9c7 100644 --- a/router/router.go +++ b/router/router.go @@ -234,6 +234,8 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } + // override bidderInfos user sync with default (iframe? redirect? both? - no, not both. error if both are defined.) + activeBidders := exchange.GetActiveBidders(bidderInfos) disabledBidders := exchange.GetDisabledBiddersErrorMessages(bidderInfos) @@ -242,9 +244,9 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } - syncers, err := usersync.NewSyncersFromBidderInfos(bidderInfos) + syncers, err := usersync.BuildSyncers(cfg.UserSync, bidderInfos) if err != nil { - glog.Fatal(err) // better error message? is this ok to be sepaerate from the validation pass? + glog.Fatal(err) } gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() diff --git a/usersync/chooser_test.go b/usersync/chooser_test.go index 65bd9c70cc1..6274be40f30 100644 --- a/usersync/chooser_test.go +++ b/usersync/chooser_test.go @@ -363,6 +363,10 @@ func (s fakeSyncer) Key() string { return s.key } +func (s fakeSyncer) DefaultSyncType() SyncType { + return SyncTypeIFrame +} + func (s fakeSyncer) SupportsType(syncTypes []SyncType) bool { for _, syncType := range syncTypes { if syncType == SyncTypeIFrame && s.supportsIFrame { diff --git a/usersync/syncer.go b/usersync/syncer.go index 27bde1dfab4..74da8867205 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -27,6 +27,9 @@ type Syncer interface { // necessarily, a one-to-one mapping with a bidder. Key() string + // DefaultSyncType is the default SyncType for this syncer. + DefaultSyncType() SyncType + // SupportsType returns true if the syncer supports at least one of the specified sync types. SupportsType(syncTypes []SyncType) bool @@ -51,16 +54,21 @@ type standardSyncer struct { } const ( - setuidSyncTypeIFrame = "b" - setuidSyncTypeRedirect = "i" + setuidSyncTypeIFrame = "b" // b = blank HTML response + setuidSyncTypeRedirect = "i" // i = image response ) -var errEndpointRequired = errors.New("at least one endpoint (iframe or redirect) is required") +var errEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") +var errKeyRequired = errors.New("key is required") var errDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") // NewSyncer creates a new Syncer from the provided configuration, or an error if macro substition // fails or an endpoint url is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { + if syncerConfig.Key == "" { + return nil, errKeyRequired + } + if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { return nil, errEndpointRequired } @@ -129,6 +137,7 @@ func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { return SyncTypeUnknown, fmt.Errorf("invalid default sync type '%s'", syncerConfig.Default) } +// macro substitution regex var ( macroRegexExternalHost = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) macroRegexSyncerKey = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) @@ -199,6 +208,10 @@ func (s standardSyncer) Key() string { return s.key } +func (s standardSyncer) DefaultSyncType() SyncType { + return s.defaultSyncType +} + func (s standardSyncer) SupportsType(syncTypes []SyncType) bool { supported := s.filterSupportedSyncTypes(syncTypes) return len(supported) > 0 @@ -276,56 +289,3 @@ func (s standardSyncer) chooseTemplate(syncType SyncType) *template.Template { return nil } } - -type bidderInfoWithName struct { - bidderName string - bidderInfo config.BidderInfo -} - -// func NewSyncersFromBidderInfos(hostConfig config.UserSync, bidderInfos config.BidderInfos) (map[string]Syncer, error) { -// bidderInfosBySyncerKey := make(map[string][]bidderInfoWithName) -// for bidderName, bidderInfo := range bidderInfos { -// s := bidderInfo.Syncer -// if s != nil && (s.IFrame != nil || s.Redirect != nil) { -// bidderInfosBySyncerKey[s.Key] = append(bidderInfosBySyncerKey[s.Key], bidderInfoWithName{bidderName, bidderInfo}) -// } -// } - -// // build syncers -// syncersByBidder := map[string]Syncer{} -// var errs []error -// for _, bidderInfos := range bidderInfosBySyncerKey { -// syncerConfig, err := getPrimarySyncerConfig(bidderInfos) -// if err != nil { -// errs = append(errs, err) -// continue -// } - -// syncer, err := NewSyncer(hostConfig, syncerConfig) -// if err != nil { -// errs = append(errs, err) -// continue -// } - -// for _, b := range bidderInfos { -// syncersByBidder[b.bidderName] = syncer -// } -// } - -// if len(errs) > 0 { -// return nil, errortypes.NewAggregateError("msg", errs) -// } - -// return syncersByBidder, nil -// } - -// func getPrimarySyncerConfig(bidderInfos []bidderInfoWithName) (*config.Syncer, error) { -// if len(bidderInfos) == 1 { -// return bidderInfos[0].bidderInfo.Syncer, nil -// } - -// // if multiple, ensure just one has endpoints defined + return it -// // if not just one, return error - -// return nil, nil -// } diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 1eaa6a111b9..452a114d894 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -14,7 +14,6 @@ import ( func TestNewSyncer(t *testing.T) { var ( - key = "a" supportCORS = true hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} @@ -26,6 +25,7 @@ func TestNewSyncer(t *testing.T) { testCases := []struct { description string + givenKey string givenDefault string givenIFrameConfig *config.SyncerEndpoint givenRedirectConfig *config.SyncerEndpoint @@ -35,15 +35,24 @@ func TestNewSyncer(t *testing.T) { expectedRedirect string }{ { - description: "No Endpoints", + description: "Missing Key", + givenKey: "", + givenDefault: "iframe", + givenIFrameConfig: iframeConfig, + givenRedirectConfig: nil, + expectedError: "key is required", + }, + { + description: "Missing Endpoints", + givenKey: "a", givenDefault: "", givenIFrameConfig: nil, givenRedirectConfig: nil, - expectedDefault: SyncTypeIFrame, - expectedError: "at least one endpoint (iframe or redirect) is required", + expectedError: "at least one endpoint (iframe and/or redirect) is required", }, { description: "Resolve Default Sync Type Error ", + givenKey: "a", givenDefault: "", givenIFrameConfig: iframeConfig, givenRedirectConfig: redirectConfig, @@ -51,6 +60,7 @@ func TestNewSyncer(t *testing.T) { }, { description: "IFrame & Redirect Endpoints", + givenKey: "a", givenDefault: "iframe", givenIFrameConfig: iframeConfig, givenRedirectConfig: redirectConfig, @@ -60,41 +70,41 @@ func TestNewSyncer(t *testing.T) { }, { description: "IFrame - Parse Error", + givenKey: "a", givenDefault: "iframe", givenIFrameConfig: errParseConfig, givenRedirectConfig: nil, - expectedDefault: SyncTypeIFrame, expectedError: "iframe template: a_usersync_url:1: function \"malformed\" not defined", }, { description: "IFrame - Validation Error", + givenKey: "a", givenDefault: "iframe", givenIFrameConfig: errInvalidConfig, givenRedirectConfig: nil, - expectedDefault: SyncTypeIFrame, expectedError: "iframe composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", }, { description: "Redirect - Parse Error", + givenKey: "a", givenDefault: "redirect", givenIFrameConfig: nil, givenRedirectConfig: errParseConfig, - expectedDefault: SyncTypeRedirect, expectedError: "redirect template: a_usersync_url:1: function \"malformed\" not defined", }, { description: "Redirect - Validation Error", + givenKey: "a", givenDefault: "redirect", givenIFrameConfig: nil, givenRedirectConfig: errInvalidConfig, - expectedDefault: SyncTypeRedirect, expectedError: "redirect composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", }, } for _, test := range testCases { syncerConfig := config.Syncer{ - Key: key, + Key: test.givenKey, SupportCORS: supportCORS, Default: test.givenDefault, IFrame: test.givenIFrameConfig, @@ -107,7 +117,7 @@ func TestNewSyncer(t *testing.T) { assert.NoError(t, err, test.description+":err") if assert.IsType(t, standardSyncer{}, result, test.description+":result_type") { result := result.(standardSyncer) - assert.Equal(t, key, result.key, test.description+":key") + assert.Equal(t, test.givenKey, result.key, test.description+":key") assert.Equal(t, supportCORS, result.supportCORS, test.description+":cors") assert.Equal(t, test.expectedDefault, result.defaultSyncType, test.description+":default_sync") @@ -385,6 +395,11 @@ func TestSyncerKey(t *testing.T) { assert.Equal(t, "a", syncer.Key()) } +func TestSyncerDefaultSyncType(t *testing.T) { + syncer := standardSyncer{defaultSyncType: SyncTypeRedirect} + assert.Equal(t, SyncTypeRedirect, syncer.DefaultSyncType()) +} + func TestSyncerSupportsType(t *testing.T) { endpointTemplate := template.Must(template.New("test").Parse("iframe")) diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go new file mode 100644 index 00000000000..ebb809b4e04 --- /dev/null +++ b/usersync/syncersbuilder.go @@ -0,0 +1,88 @@ +package usersync + +import ( + "fmt" + "strings" + + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" +) + +type namedSyncerConfig struct { + name string + cfg config.Syncer +} + +func BuildSyncers(hostConfig config.UserSync, bidderInfos config.BidderInfos) (map[string]Syncer, error) { + // map syncer config by bidder + cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) + for bidder, cfg := range bidderInfos { + if cfg.Syncer != nil { + cfgByBidder[bidder] = *cfg.Syncer + } + } + + // map syncer config by key + cfgBySyncerKey := make(map[string][]namedSyncerConfig, len(bidderInfos)) + for bidder, cfg := range cfgByBidder { + if cfg.Key == "" { + cfg.Key = bidder + } + cfgBySyncerKey[cfg.Key] = append(cfgBySyncerKey[cfg.Key], namedSyncerConfig{bidder, cfg}) + } + + // create syncers + errs := []error{} + syncers := make(map[string]Syncer, len(bidderInfos)) + for key, cfgGroup := range cfgBySyncerKey { + primaryCfg, err := chooseSyncerConfig(cfgGroup) + if err != nil { + errs = append(errs, err) + continue + } + + syncer, err := NewSyncer(hostConfig, primaryCfg.cfg) + if err != nil { + errs = append(errs, fmt.Errorf("cannot create syncer for bidder %s with key %s. %v", primaryCfg.name, key, err)) + continue + } + + for _, bidder := range cfgGroup { + syncers[bidder.name] = syncer + } + } + + if len(errs) > 0 { + return nil, errortypes.NewAggregateError("user sync", errs) + } + return syncers, nil +} + +func chooseSyncerConfig(biddersSyncerConfig []namedSyncerConfig) (namedSyncerConfig, error) { + if len(biddersSyncerConfig) == 1 { + return biddersSyncerConfig[0], nil + } + + var bidderNames []string + var bidderNamesWithEndpoints []string + var syncerConfig namedSyncerConfig + for _, bidder := range biddersSyncerConfig { + bidderNames = append(bidderNames, bidder.name) + if bidder.cfg.IFrame != nil || bidder.cfg.Redirect != nil { + bidderNamesWithEndpoints = append(bidderNamesWithEndpoints, bidder.name) + syncerConfig = bidder + } + } + + if len(bidderNamesWithEndpoints) == 0 { + bidders := strings.Join(bidderNames, ", ") + return namedSyncerConfig{}, fmt.Errorf("bidders %s share the same syncer key, but none define endpoints (iframe and/or redirect)", bidders) + } + + if len(bidderNamesWithEndpoints) > 1 { + bidders := strings.Join(bidderNames, ", ") + return namedSyncerConfig{}, fmt.Errorf("bidders %s define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", bidders) + } + + return syncerConfig, nil +} diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go new file mode 100644 index 00000000000..dd9a10a9508 --- /dev/null +++ b/usersync/syncersbuilder_test.go @@ -0,0 +1,163 @@ +package usersync + +import ( + "testing" + + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/privacy" + "github.com/stretchr/testify/assert" +) + +func TestBuildSyncers(t *testing.T) { + var ( + hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"} + iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} + infoKeyAPopulated = config.BidderInfo{Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} + infoKeyAEmpty = config.BidderInfo{Syncer: &config.Syncer{Key: "a"}} + infoKeyAError = config.BidderInfo{Syncer: &config.Syncer{Key: "a", Default: "redirect", IFrame: iframeConfig}} // Error caused by invalid default sync type + infoKeyBPopulated = config.BidderInfo{Syncer: &config.Syncer{Key: "b", IFrame: iframeConfig}} + infoKeyBEmpty = config.BidderInfo{Syncer: &config.Syncer{Key: "b"}} + infoKeyMissingPopulated = config.BidderInfo{Syncer: &config.Syncer{IFrame: iframeConfig}} + ) + + // NOTE: The hostConfig includes the syncer key in the RedirectURL to distinguish between the syncer keys + // in these tests. Look carefully at the end of the expected iframe urls to see the syncer key. + + testCases := []struct { + description string + givenBidderInfos config.BidderInfos + expectedIFramesURLs map[string]string + expectedError string + }{ + { + description: "One", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", + }, + }, + { + description: "One - Missing Key - Defaults To Bidder Name", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyMissingPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fbidder1%2Fhost", + }, + }, + { + description: "One - Syncer Error", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, + expectedError: "user sync (1 error):\n 1: cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + }, + { + description: "Many - Different Syncers", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyBPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", + "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", + }, + }, + { + description: "Many - Same Syncers - One Primary", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAEmpty}, + expectedIFramesURLs: map[string]string{ + "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", + "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", + }, + }, + { + description: "Many - Same Syncers - Many Primaries", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, + expectedError: "user sync (1 error):\n 1: bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", + }, + { + description: "Many - Sync Error - Bidder Correct", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, + expectedError: "user sync (1 error):\n 1: cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", + }, + { + description: "Many - Empty Syncers Ignored", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": {}, "bidder2": infoKeyBPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", + }, + }, + { + description: "Many - Multiple Errors", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, + expectedError: "user sync (2 errors):\n 1: cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n 2: cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", + }, + } + + for _, test := range testCases { + result, err := BuildSyncers(hostConfig, test.givenBidderInfos) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + resultRenderedIFrameURLS := map[string]string{} + for k, v := range result { + iframeRendered, err := v.GetSync([]SyncType{SyncTypeIFrame}, privacy.Policies{}) + if assert.NoError(t, err, test.description+"key:%s,:iframe_render", k) { + resultRenderedIFrameURLS[k] = iframeRendered.URL + } + } + assert.Equal(t, test.expectedIFramesURLs, resultRenderedIFrameURLS, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result, test.description+":result") + } + } +} + +func TestChooseSyncerConfig(t *testing.T) { + var ( + bidderAPopulated = namedSyncerConfig{name: "bidderA", cfg: config.Syncer{Key: "a", IFrame: &config.SyncerEndpoint{URL: "anyURL"}}} + bidderAEmpty = namedSyncerConfig{name: "bidderA", cfg: config.Syncer{}} + bidderBPopulated = namedSyncerConfig{name: "bidderB", cfg: config.Syncer{Key: "a", IFrame: &config.SyncerEndpoint{URL: "anyURL"}}} + bidderBEmpty = namedSyncerConfig{name: "bidderB", cfg: config.Syncer{}} + ) + + testCases := []struct { + description string + given []namedSyncerConfig + expectedConfig namedSyncerConfig + expectedError string + }{ + { + description: "One", + given: []namedSyncerConfig{bidderAPopulated}, + expectedConfig: bidderAPopulated, + }, + { + description: "Many - Same Key - Unique Configs", + given: []namedSyncerConfig{bidderAEmpty, bidderBPopulated}, + expectedConfig: bidderBPopulated, + }, + { + description: "Many - Same Key - Multiple Configs", + given: []namedSyncerConfig{bidderAPopulated, bidderBPopulated}, + expectedError: "bidders bidderA, bidderB define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", + }, + { + description: "Many - Same Key - No Configs", + given: []namedSyncerConfig{bidderAEmpty, bidderBEmpty}, + expectedError: "bidders bidderA, bidderB share the same syncer key, but none define endpoints (iframe and/or redirect)", + }, + { + description: "Many - Same Key - Unique Configs", + given: []namedSyncerConfig{bidderAEmpty, bidderBPopulated}, + expectedConfig: bidderBPopulated, + }, + } + + for _, test := range testCases { + result, err := chooseSyncerConfig(test.given) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedConfig, result, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result, test.description+":result") + } + } +} diff --git a/util/httputil/pixel.go b/util/httputil/pixel.go new file mode 100644 index 00000000000..0489351e1ce --- /dev/null +++ b/util/httputil/pixel.go @@ -0,0 +1,16 @@ +package httputil + +type OneByOnePixel struct { + Content []byte + ContentType string +} + +var OneByOnePixelPNG = OneByOnePixel{ + Content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x04, 0x73, 0x42, 0x49, 0x54, 0x08, 0x08, 0x08, 0x08, 0x7C, 0x08, 0x64, 0x88, + 0x00, 0x00, 0x00, 0x0D, 0x49, 0x44, 0x41, 0x54, 0x08, 0x99, 0x63, 0x60, 0x60, 0x60, 0x60, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x01, 0x87, 0xA1, 0x4E, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, + 0x42, 0x60, 0x82}, + ContentType: "image/png", +} From 258d1ca2e861f8709183d052ec32ce5f276cb99c Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 02:28:43 -0400 Subject: [PATCH 10/50] Add SetUID Tests --- endpoints/events/event.go | 4 +- endpoints/setuid.go | 13 +- endpoints/setuid_test.go | 352 +++++++++++++++++++++++--------------- gdpr/gdpr.go | 6 +- gdpr/impl.go | 4 +- metrics/config/todo | 2 - util/httputil/pixel.go | 4 +- 7 files changed, 228 insertions(+), 157 deletions(-) diff --git a/endpoints/events/event.go b/endpoints/events/event.go index 4f033a195a7..5974ba9f7d8 100644 --- a/endpoints/events/event.go +++ b/endpoints/events/event.go @@ -36,7 +36,7 @@ type eventEndpoint struct { Accounts stored_requests.AccountFetcher Analytics analytics.PBSAnalyticsModule Cfg *config.Configuration - TrackingPixel *httputil.OneByOnePixel + TrackingPixel *httputil.Pixel } func NewEventEndpoint(cfg *config.Configuration, accounts stored_requests.AccountFetcher, analytics analytics.PBSAnalyticsModule) httprouter.Handle { @@ -44,7 +44,7 @@ func NewEventEndpoint(cfg *config.Configuration, accounts stored_requests.Accoun Accounts: accounts, Analytics: analytics, Cfg: cfg, - TrackingPixel: &httputil.OneByOnePixelPNG, + TrackingPixel: &httputil.Pixel1x1PNG, } return ee.Handle diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 8467936e1d8..cd3430022a0 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -50,7 +50,6 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer query := r.URL.Query() - // get key + verify we have a syncer for the key syncerKey, err := getSyncerKey(query, syncers) if err != nil { w.WriteHeader(http.StatusBadRequest) @@ -108,13 +107,14 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer switch responseFormat { case "i": + w.Header().Add("Content-Type", httputil.Pixel1x1PNG.ContentType) + w.Header().Add("Content-Length", strconv.Itoa(len(httputil.Pixel1x1PNG.Content))) w.WriteHeader(http.StatusOK) - w.Header().Add("Content-Type", httputil.OneByOnePixelPNG.ContentType) - //w.Header().Add("Content-Length", strconv.Itoa(len(httputil.TrackingPixelPNG.Content))) - is this automatic? - w.Write(httputil.OneByOnePixelPNG.Content) + w.Write(httputil.Pixel1x1PNG.Content) case "b": w.Header().Add("Content-Type", "text/html") - // set content length to 0 - is this automatic? + w.Header().Add("Content-Length", "0") + w.WriteHeader(http.StatusOK) } }) } @@ -148,9 +148,8 @@ func getResponseFormat(query url.Values, syncer usersync.Syncer) (string, error) } if !strings.EqualFold(format, "b") && !strings.EqualFold(format, "i") { - return "", errors.New("invalid value") + return "", errors.New(`"f" query param is invalid. must be "b" or "i"`) } - return strings.ToLower(format), nil } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index ad869dccae4..89715b65bb9 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "net/http/httptest" + "net/url" "regexp" "testing" "time" @@ -12,12 +13,11 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/metrics" + "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" - "github.com/prebid/prebid-server/openrtb_ext" - analyticsConf "github.com/prebid/prebid-server/analytics/config" metricsConf "github.com/prebid/prebid-server/metrics/config" ) @@ -25,167 +25,210 @@ import ( func TestSetUIDEndpoint(t *testing.T) { testCases := []struct { uri string - validFamilyNames []string + syncers []string existingSyncs map[string]string gdprAllowsHostCookies bool gdprReturnsError bool + gdprMalformed bool expectedSyncs map[string]string - expectedRespMessage string - expectedResponseCode int + expectedBody string + expectedStatusCode int + expectedHeaders map[string]string description string }{ { uri: "/setuid?bidder=pubmatic&uid=123", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: nil, gdprAllowsHostCookies: true, expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedResponseCode: http.StatusOK, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, description: "Set uid for valid bidder", }, { uri: "/setuid?bidder=unsupported-bidder&uid=123", - validFamilyNames: []string{}, + syncers: []string{}, existingSyncs: nil, gdprAllowsHostCookies: true, expectedSyncs: nil, - expectedResponseCode: http.StatusBadRequest, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "The bidder name provided is not supported by Prebid Server", description: "Don't set uid for an unsupported bidder", }, { uri: "/setuid?bidder=&uid=123", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: nil, gdprAllowsHostCookies: true, expectedSyncs: nil, - expectedResponseCode: http.StatusBadRequest, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, description: "Don't set uid for an empty bidder", }, { uri: "/setuid?bidder=unsupported-bidder&uid=123", - validFamilyNames: []string{}, + syncers: []string{}, existingSyncs: map[string]string{"pubmatic": "1234"}, gdprAllowsHostCookies: true, expectedSyncs: nil, - expectedResponseCode: http.StatusBadRequest, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "The bidder name provided is not supported by Prebid Server", description: "No need to set existing syncs back in response for a request " + "to set uid for an unsupported bidder", }, { uri: "/setuid?bidder=&uid=123", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: map[string]string{"pubmatic": "1234"}, gdprAllowsHostCookies: true, expectedSyncs: nil, - expectedResponseCode: http.StatusBadRequest, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, description: "No need to set existing syncs back in response for a request " + "to set uid for an empty bidder", }, { uri: "/setuid?bidder=pubmatic", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: map[string]string{"pubmatic": "1234"}, gdprAllowsHostCookies: true, expectedSyncs: map[string]string{}, - expectedResponseCode: http.StatusOK, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, description: "Unset uid for a bidder if the request contains an empty uid for that bidder", }, { uri: "/setuid?bidder=pubmatic&uid=123", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: map[string]string{"rubicon": "def"}, gdprAllowsHostCookies: true, expectedSyncs: map[string]string{"pubmatic": "123", "rubicon": "def"}, - expectedResponseCode: http.StatusOK, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, description: "Add the uid for the requested bidder to the list of existing syncs", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=0", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, existingSyncs: nil, gdprAllowsHostCookies: true, expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedResponseCode: http.StatusOK, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, description: "Don't care about GDPR consent if GDPR is set to 0", }, - { - uri: "/setuid?bidder=pubmatic&uid=123", - validFamilyNames: []string{"pubmatic"}, - existingSyncs: nil, - expectedSyncs: nil, - expectedResponseCode: http.StatusOK, - expectedRespMessage: "The gdpr_consent string prevents cookies from being saved", - description: "Return err message if the GDPR consent doesn't allow syncs for the given bidder", - }, { uri: "/setuid?uid=123", - validFamilyNames: []string{"appnexus"}, + syncers: []string{"appnexus"}, existingSyncs: nil, expectedSyncs: nil, gdprAllowsHostCookies: true, - expectedResponseCode: http.StatusBadRequest, - expectedRespMessage: `"bidder" query param is required`, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, description: "Return an error if the bidder param is missing from the request", }, { uri: "/setuid?bidder=appnexus&uid=123&gdpr=2", - validFamilyNames: []string{"appnexus"}, + syncers: []string{"appnexus"}, existingSyncs: nil, expectedSyncs: nil, gdprAllowsHostCookies: true, - expectedResponseCode: http.StatusBadRequest, - expectedRespMessage: "the gdpr query param must be either 0 or 1. You gave 2", + expectedStatusCode: http.StatusBadRequest, + expectedBody: "the gdpr query param must be either 0 or 1. You gave 2", description: "Return an error if GDPR is set to anything else other that 0 or 1", }, { uri: "/setuid?bidder=appnexus&uid=123&gdpr=1", - validFamilyNames: []string{"appnexus"}, + syncers: []string{"appnexus"}, existingSyncs: nil, expectedSyncs: nil, gdprAllowsHostCookies: true, - expectedResponseCode: http.StatusBadRequest, - expectedRespMessage: "gdpr_consent is required when gdpr=1", + expectedStatusCode: http.StatusBadRequest, + expectedBody: "gdpr_consent is required when gdpr=1", description: "Return an error if GDPR is set to 1 but GDPR consent string is missing", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - validFamilyNames: []string{"pubmatic"}, - existingSyncs: nil, - expectedSyncs: nil, - gdprReturnsError: true, - expectedResponseCode: http.StatusBadRequest, - expectedRespMessage: "No global vendor list was available to interpret this consent string. " + + syncers: []string{"pubmatic"}, + existingSyncs: nil, + expectedSyncs: nil, + gdprReturnsError: true, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "No global vendor list was available to interpret this consent string. " + "If this is a new, valid version, it should become available soon.", description: "Return an error if the GDPR string is either malformed or using a newer version that isn't yet supported", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - validFamilyNames: []string{"pubmatic"}, - existingSyncs: nil, - expectedSyncs: nil, - expectedResponseCode: http.StatusOK, - expectedRespMessage: "The gdpr_consent string prevents cookies from being saved", - description: "Shouldn't set uid for a bidder if it is not allowed by the GDPR consent string", + syncers: []string{"pubmatic"}, + existingSyncs: nil, + expectedSyncs: nil, + expectedStatusCode: http.StatusUnavailableForLegalReasons, + expectedBody: "The gdpr_consent string prevents cookies from being saved", + description: "Shouldn't set uid for a bidder if it is not allowed by the GDPR consent string", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, existingSyncs: nil, expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedResponseCode: http.StatusOK, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, description: "Should set uid for a bidder that is allowed by the GDPR consent string", }, + { + uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + + "malformed", + syncers: []string{"pubmatic"}, + gdprAllowsHostCookies: true, + gdprMalformed: true, + existingSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "gdpr_consent was invalid. malformed consent string malformed: some error", + description: "Should return an error if GDPR consent string is malformed", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=b", + syncers: []string{"pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Set uid for valid bidder with iframe format", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=i", + syncers: []string{"pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "image/png", "Content-Length": "86"}, + description: "Set uid for valid bidder with redirect format", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=x", + syncers: []string{"pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"f" query param is invalid. must be "b" or "i"`, + description: "Set uid for valid bidder with invalid format", + }, } metrics := &metricsConf.DummyMetricsEngine{} for _, test := range testCases { response := doRequest(makeRequest(test.uri, test.existingSyncs), metrics, - test.validFamilyNames, test.gdprAllowsHostCookies, test.gdprReturnsError) - assert.Equal(t, test.expectedResponseCode, response.Code, "Test Case: %s. /setuid returned unexpected error code", test.description) + test.syncers, test.gdprAllowsHostCookies, test.gdprReturnsError, test.gdprMalformed) + assert.Equal(t, test.expectedStatusCode, response.Code, "Test Case: %s. /setuid returned unexpected error code", test.description) if test.expectedSyncs != nil { assertHasSyncs(t, test.description, response, test.expectedSyncs) @@ -193,9 +236,21 @@ func TestSetUIDEndpoint(t *testing.T) { assert.Equal(t, "", response.Header().Get("Set-Cookie"), "Test Case: %s. /setuid returned unexpected cookie", test.description) } - if test.expectedRespMessage != "" { - assert.Equal(t, test.expectedRespMessage, response.Body.String(), "Test Case: %s. /setuid returned unexpected message") + if test.expectedBody != "" { + assert.Equal(t, test.expectedBody, response.Body.String(), "Test Case: %s. /setuid returned unexpected message", test.description) } + + // compare header values, except for the cookies + responseHeaders := map[string]string{} + for k, v := range response.Result().Header { + if k != "Set-Cookie" { + responseHeaders[k] = v[0] + } + } + if test.expectedHeaders == nil { + test.expectedHeaders = map[string]string{} + } + assert.Equal(t, test.expectedHeaders, responseHeaders, test.description+":headers") } } @@ -206,7 +261,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { testCases := []struct { uri string cookies []*usersync.Cookie - validFamilyNames []string + syncers []string gdprAllowsHostCookies bool expectedMetricAction metrics.RequestAction expectedMetricBidder openrtb_ext.BidderName @@ -216,7 +271,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{}, - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionSet, expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), @@ -226,7 +281,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=", cookies: []*usersync.Cookie{}, - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionSet, expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), @@ -236,7 +291,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{cookieWithOptOut}, - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionOptOut, expectedResponseCode: 401, @@ -245,7 +300,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{}, - validFamilyNames: []string{}, + syncers: []string{}, gdprAllowsHostCookies: true, expectedMetricAction: metrics.RequestActionErr, expectedResponseCode: 400, @@ -254,7 +309,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1", cookies: []*usersync.Cookie{}, - validFamilyNames: []string{"pubmatic"}, + syncers: []string{"pubmatic"}, gdprAllowsHostCookies: false, expectedMetricAction: metrics.RequestActionGDPR, expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), @@ -275,7 +330,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { for _, v := range test.cookies { addCookie(req, v) } - response := doRequest(req, metricsEngine, test.validFamilyNames, test.gdprAllowsHostCookies, false) + response := doRequest(req, metricsEngine, test.syncers, test.gdprAllowsHostCookies, false, false) assert.Equal(t, test.expectedResponseCode, response.Code, test.description) metricsEngine.AssertExpectations(t) @@ -287,9 +342,9 @@ func TestOptedOut(t *testing.T) { cookie := usersync.NewCookie() cookie.SetOptOut(true) addCookie(request, cookie) - validFamilyNames := []string{"pubmatic"} + syncers := []string{"pubmatic"} metrics := &metricsConf.DummyMetricsEngine{} - response := doRequest(request, metrics, validFamilyNames, true, false) + response := doRequest(request, metrics, syncers, true, false, false) assert.Equal(t, http.StatusUnauthorized, response.Code) } @@ -317,53 +372,75 @@ func TestSiteCookieCheck(t *testing.T) { } } -func TestGetFamilyName(t *testing.T) { - // testCases := []struct { - // urlValues url.Values - // expectedName string - // expectedError string - // description string - // }{ - // { - // urlValues: url.Values{"bidder": []string{"valid"}}, - // expectedName: "valid", - // description: "Should return no error for valid family name", - // }, - // { - // urlValues: url.Values{"bidder": []string{"VALID"}}, - // expectedError: "The bidder name provided is not supported by Prebid Server", - // description: "Should return error for different case", - // }, - // { - // urlValues: url.Values{"bidder": []string{"invalid"}}, - // expectedError: "The bidder name provided is not supported by Prebid Server", - // description: "Should return an error for unsupported bidder", - // }, - // { - // urlValues: url.Values{"bidder": []string{}}, - // expectedError: `"bidder" query param is required`, - // description: "Should return an error for empty bidder name", - // }, - // { - // urlValues: url.Values{}, - // expectedError: `"bidder" query param is required`, - // description: "Should return an error for missing bidder name", - // }, - // } - - //for _, test := range testCases { - - // todo - // name, err := getSyncerKey(test.urlValues, map[string]struct{}{"valid": {}}) - - // assert.Equal(t, test.expectedName, name, test.description) - - // if test.expectedError != "" { - // assert.EqualError(t, err, test.expectedError, test.description) - // } else { - // assert.NoError(t, err, test.description) - // } - //} +func TestGetResponseFormat(t *testing.T) { + testCases := []struct { + urlValues url.Values + syncer usersync.Syncer + expectedFormat string + expectedError string + description string + }{ + { + urlValues: url.Values{}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeIFrame}, + expectedFormat: "b", + description: "parameter not provided, use default sync type iframe", + }, + { + urlValues: url.Values{}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeRedirect}, + expectedFormat: "i", + description: "parameter not provided, use default sync type redirect", + }, + { + urlValues: url.Values{}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncType("invalid")}, + expectedError: "invalid default sync type", + description: "parameter not provided, use default sync type invalid", + }, + { + urlValues: url.Values{"f": []string{"b"}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeRedirect}, + expectedFormat: "b", + description: "parameter given as `b`, default sync type is opposite", + }, + { + urlValues: url.Values{"f": []string{"B"}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeRedirect}, + expectedFormat: "b", + description: "parameter given as `b`, default sync type is opposite - case insensitive", + }, + { + urlValues: url.Values{"f": []string{"i"}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeIFrame}, + expectedFormat: "i", + description: "parameter given as `b`, default sync type is opposite", + }, + { + urlValues: url.Values{"f": []string{"I"}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeIFrame}, + expectedFormat: "i", + description: "parameter given as `b`, default sync type is opposite - case insensitive", + }, + { + urlValues: url.Values{"f": []string{"x"}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeIFrame}, + expectedError: `"f" query param is invalid. must be "b" or "i"`, + description: "parameter given invalid", + }, + } + + for _, test := range testCases { + result, err := getResponseFormat(test.urlValues, test.syncer) + + if test.expectedError == "" { + assert.NoError(t, err, test.description+":err") + assert.Equal(t, test.expectedFormat, result, test.description+":result") + } else { + assert.EqualError(t, err, test.expectedError, test.description+":err") + assert.Empty(t, result, test.description+":result") + } + } } func assertHasSyncs(t *testing.T, testCase string, resp *httptest.ResponseRecorder, syncs map[string]string) { @@ -383,28 +460,29 @@ func makeRequest(uri string, existingSyncs map[string]string) *http.Request { request := httptest.NewRequest("GET", uri, nil) if len(existingSyncs) > 0 { pbsCookie := usersync.NewCookie() - for family, value := range existingSyncs { - pbsCookie.TrySync(family, value) + for key, value := range existingSyncs { + pbsCookie.TrySync(key, value) } addCookie(request, pbsCookie) } return request } -func doRequest(req *http.Request, metrics metrics.MetricsEngine, validFamilyNames []string, gdprAllowsHostCookies bool, gdprReturnsError bool) *httptest.ResponseRecorder { +func doRequest(req *http.Request, metrics metrics.MetricsEngine, syncers []string, gdprAllowsHostCookies, gdprReturnsError, gdprReturnsMalformedError bool) *httptest.ResponseRecorder { cfg := config.Configuration{} perms := &mockPermsSetUID{ - allowHost: gdprAllowsHostCookies, - errorHost: gdprReturnsError, - allowPI: true, + allowHost: gdprAllowsHostCookies, + errorHost: gdprReturnsError, + errorMalformed: gdprReturnsMalformedError, + allowPI: true, } analytics := analyticsConf.NewPBSAnalytics(&cfg.Analytics) - syncers := make(map[string]usersync.Syncer) - for _, name := range validFamilyNames { - syncers[name] = newFakeSyncer(name) + syncersMap := make(map[string]usersync.Syncer) + for _, key := range syncers { + syncersMap[key] = fakeSyncer{key: key, defaultSyncType: usersync.SyncTypeIFrame} } - endpoint := NewSetUIDEndpoint(cfg.HostCookie, syncers, perms, analytics, metrics) + endpoint := NewSetUIDEndpoint(cfg.HostCookie, syncersMap, perms, analytics, metrics) response := httptest.NewRecorder() endpoint(response, req, nil) return response @@ -427,17 +505,20 @@ func parseCookieString(t *testing.T, response *httptest.ResponseRecorder) *users } type mockPermsSetUID struct { - allowHost bool - errorHost bool - allowPI bool + allowHost bool + errorHost bool + errorMalformed bool + allowPI bool } func (g *mockPermsSetUID) HostCookiesAllowed(ctx context.Context, gdprSignal gdpr.Signal, consent string) (bool, error) { - var err error + if g.errorMalformed { + return g.allowHost, &gdpr.ErrorMalformedConsent{Consent: consent, Cause: errors.New("some error")} + } if g.errorHost { - err = errors.New("something went wrong") + return g.allowHost, errors.New("something went wrong") } - return g.allowHost, err + return g.allowHost, nil } func (g *mockPermsSetUID) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, gdprSignal gdpr.Signal, consent string) (bool, error) { @@ -448,30 +529,23 @@ func (g *mockPermsSetUID) PersonalInfoAllowed(ctx context.Context, bidder openrt return g.allowPI, g.allowPI, g.allowPI, nil } -func newFakeSyncer(key string) usersync.Syncer { - return fakeSyncer{ - key: key, - } -} - type fakeSyncer struct { - key string + key string + defaultSyncType usersync.SyncType } -// FamilyNames implements the Usersyncer interface. func (s fakeSyncer) Key() string { return s.key } func (s fakeSyncer) DefaultSyncType() usersync.SyncType { - return usersync.SyncTypeIFrame + return s.defaultSyncType } func (s fakeSyncer) SupportsType(syncTypes []usersync.SyncType) bool { return true } -// GetUsersyncInfo implements the Usersyncer interface with a no-op. func (s fakeSyncer) GetSync(syncTypes []usersync.SyncType, privacyPolicies privacy.Policies) (usersync.Sync, error) { return usersync.Sync{}, nil } diff --git a/gdpr/gdpr.go b/gdpr/gdpr.go index bb947392e9e..ea4524200a5 100644 --- a/gdpr/gdpr.go +++ b/gdpr/gdpr.go @@ -58,10 +58,10 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ // An ErrorMalformedConsent will be returned by the Permissions interface if // the consent string argument was the reason for the failure. type ErrorMalformedConsent struct { - consent string - cause error + Consent string + Cause error } func (e *ErrorMalformedConsent) Error() string { - return "malformed consent string " + e.consent + ": " + e.cause.Error() + return "malformed consent string " + e.Consent + ": " + e.Cause.Error() } diff --git a/gdpr/impl.go b/gdpr/impl.go index 095b88cd7aa..f3cff7df42c 100644 --- a/gdpr/impl.go +++ b/gdpr/impl.go @@ -195,8 +195,8 @@ func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, cons parsedConsent, err = vendorconsent.ParseString(consent) if err != nil { err = &ErrorMalformedConsent{ - consent: consent, - cause: err, + Consent: consent, + Cause: err, } return } diff --git a/metrics/config/todo b/metrics/config/todo index ad0a58776ef..2d83fa8336c 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,6 +1,4 @@ todo - - /setuid endpoint tests - - override urls from config in bidder info - add new cookiesync metrics to go metrics - add proper samesite behavior - cookie should just know what to do diff --git a/util/httputil/pixel.go b/util/httputil/pixel.go index 0489351e1ce..0beefdb1924 100644 --- a/util/httputil/pixel.go +++ b/util/httputil/pixel.go @@ -1,11 +1,11 @@ package httputil -type OneByOnePixel struct { +type Pixel struct { Content []byte ContentType string } -var OneByOnePixelPNG = OneByOnePixel{ +var Pixel1x1PNG = Pixel{ Content: []byte{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, 0x89, 0x00, 0x00, 0x00, 0x04, 0x73, 0x42, 0x49, 0x54, 0x08, 0x08, 0x08, 0x08, 0x7C, 0x08, 0x64, 0x88, From f71c5d5cfba6c9093825b7bdcb78f8930ce6e363 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 03:51:08 -0400 Subject: [PATCH 11/50] Add Proper Syncer Key Lookup In Exchange Logic --- config/config_test.go | 23 -------- endpoints/openrtb2/auction_benchmark_test.go | 2 + exchange/exchange.go | 16 ++++-- exchange/exchange_test.go | 29 +++++++---- exchange/utils.go | 11 ++-- exchange/utils_test.go | 21 ++++++-- metrics/config/todo | 1 + router/router.go | 2 +- usersync/cookie.go | 42 +++++---------- usersync/cookie_test.go | 16 ++---- usersync/syncersbuilder_test.go | 55 +++++++++++++------- 11 files changed, 110 insertions(+), 108 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 8fae7f725b4..f6f026c6ee8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -278,20 +278,6 @@ adapters: usersync_url: https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r= `) -var invalidUserSyncURLConfig = []byte(` -adapters: - appnexus: - endpoint: http://ib.adnxs.com/some/endpoint - audienceNetwork: - endpoint: http://facebook.com/pbs - usersync_url: http://facebook.com/ortb/prebid-s2s - platform_id: abcdefgh1234 - brightroll: - usersync_url: http//test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&url=%s - adkerneladn: - usersync_url: http:\\tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r= -`) - var oldStoredRequestsConfig = []byte(` stored_requests: filesystem: true @@ -507,15 +493,6 @@ func TestInvalidAdapterEndpointConfig(t *testing.T) { assert.Error(t, err, "invalid endpoint in config should return an error") } -func TestInvalidAdapterUserSyncURLConfig(t *testing.T) { - v := viper.New() - SetupViper(v, "") - v.SetConfigType("yaml") - v.ReadConfig(bytes.NewBuffer(invalidUserSyncURLConfig)) - _, err := New(v) - assert.Error(t, err, "invalid user_sync URL in config should return an error") -} - func TestNegativeRequestSize(t *testing.T) { cfg := newDefaultConfig(t) cfg.MaxRequestSize = -1 diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index e55ffd11093..7fb4cbad7fd 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/prebid/prebid-server/currency" + "github.com/prebid/prebid-server/usersync" analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" @@ -75,6 +76,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { adapters, nil, &config.Configuration{}, + map[string]usersync.Syncer{}, newTestMetrics(), infos, gdpr.AlwaysAllow{}, diff --git a/exchange/exchange.go b/exchange/exchange.go index bf30850e4b6..df2fb905e97 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -15,6 +15,7 @@ import ( "time" "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/usersync" "github.com/golang/glog" "github.com/mxmCherry/openrtb" @@ -44,14 +45,14 @@ type Exchange interface { // IdFetcher can find the user's ID for a specific Bidder. type IdFetcher interface { - // GetId returns the ID for the bidder. The boolean will be true if the ID exists, and false otherwise. - GetId(bidder openrtb_ext.BidderName) (string, bool) + GetUID(key string) (uid string, exists bool, notExpired bool) HasAnyLiveSyncs() bool } type exchange struct { adapterMap map[openrtb_ext.BidderName]adaptedBidder bidderInfo config.BidderInfos + bidderToSyncerKey map[string]string me metrics.MetricsEngine cache prebid_cache_client.Client cacheTime time.Duration @@ -61,6 +62,7 @@ type exchange struct { UsersyncIfAmbiguous bool privacyConfig config.Privacy categoriesFetcher stored_requests.CategoryFetcher + // should add a map of bidder to syncer key, then can eliminate the family names hack } // Container to pass out response ext data from the GetAllBids goroutines back into the main thread @@ -79,10 +81,16 @@ type bidResponseWrapper struct { bidder openrtb_ext.BidderName } -func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher) Exchange { +func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, syncers map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher) Exchange { + bidderToSyncerKey := map[string]string{} + for bidder, syncer := range syncers { + bidderToSyncerKey[bidder] = syncer.Key() + } + return &exchange{ adapterMap: adapters, bidderInfo: infos, + bidderToSyncerKey: bidderToSyncerKey, cache: cache, cacheTime: time.Duration(cfg.CacheURL.ExpectedTimeMillis) * time.Millisecond, categoriesFetcher: categoriesFetcher, @@ -157,7 +165,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog * usersyncIfAmbiguous := e.parseUsersyncIfAmbiguous(r.BidRequest) // Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder - bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, e.gDPR, usersyncIfAmbiguous, e.privacyConfig) + bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, e.bidderToSyncerKey, e.gDPR, usersyncIfAmbiguous, e.privacyConfig) e.me.RecordRequestPrivacy(privacyLabels) diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index be2ca0dfd1e..b878f5a80d6 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" @@ -65,7 +66,7 @@ func TestNewExchange(t *testing.T) { } currencyConverter := currency.NewRateConverter(&http.Client{}, "", time.Duration(0)) - e := NewExchange(adapters, nil, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) + e := NewExchange(adapters, nil, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) for _, bidderName := range knownAdapters { if _, ok := e.adapterMap[bidderName]; !ok { t.Errorf("NewExchange produced an Exchange without bidder %s", bidderName) @@ -113,7 +114,7 @@ func TestCharacterEscape(t *testing.T) { } currencyConverter := currency.NewRateConverter(&http.Client{}, "", time.Duration(0)) - e := NewExchange(adapters, nil, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) + e := NewExchange(adapters, nil, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) /* 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs */ //liveAdapters []openrtb_ext.BidderName, @@ -747,7 +748,7 @@ func TestGetBidCacheInfoEndToEnd(t *testing.T) { } currencyConverter := currency.NewRateConverter(&http.Client{}, "", time.Duration(0)) pbc := pbc.NewClient(&http.Client{}, &cfg.CacheURL, &cfg.ExtCacheURL, testEngine) - e := NewExchange(adapters, pbc, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) + e := NewExchange(adapters, pbc, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) /* 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs */ liveAdapters := []openrtb_ext.BidderName{bidderName} @@ -1100,7 +1101,7 @@ func TestBidResponseCurrency(t *testing.T) { } currencyConverter := currency.NewRateConverter(&http.Client{}, "", time.Duration(0)) - e := NewExchange(adapters, nil, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) + e := NewExchange(adapters, nil, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) liveAdapters := make([]openrtb_ext.BidderName, 1) liveAdapters[0] = "appnexus" @@ -1286,7 +1287,7 @@ func TestRaceIntegration(t *testing.T) { } debugLog := DebugLog{} - ex := NewExchange(adapters, &wellBehavedCache{}, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, &nilCategoryFetcher{}).(*exchange) + ex := NewExchange(adapters, &wellBehavedCache{}, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, &nilCategoryFetcher{}).(*exchange) _, err = ex.HoldAuction(context.Background(), auctionRequest, &debugLog) if err != nil { t.Errorf("HoldAuction returned unexpected error: %v", err) @@ -1380,7 +1381,7 @@ func TestPanicRecovery(t *testing.T) { } currencyConverter := currency.NewRateConverter(&http.Client{}, "", time.Duration(0)) - e := NewExchange(adapters, nil, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) + e := NewExchange(adapters, nil, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, nilCategoryFetcher{}).(*exchange) chBids := make(chan *bidResponseWrapper, 1) panicker := func(bidderRequest BidderRequest, conversions currency.Conversions) { @@ -1476,7 +1477,7 @@ func TestPanicRecoveryHighLevel(t *testing.T) { t.Errorf("Failed to create a category Fetcher: %v", error) } - e := NewExchange(adapters, &mockCache{}, cfg, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, categoriesFetcher).(*exchange) + e := NewExchange(adapters, &mockCache{}, cfg, map[string]usersync.Syncer{}, &metricsConf.DummyMetricsEngine{}, biddersInfo, gdpr.AlwaysAllow{}, currencyConverter, categoriesFetcher).(*exchange) e.adapterMap[openrtb_ext.BidderBeachfront] = panicingAdapter{} e.adapterMap[openrtb_ext.BidderAppnexus] = panicingAdapter{} @@ -1762,6 +1763,11 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string] t.Fatalf("Failed to create a category Fetcher: %v", error) } + bidderToSyncerKey := map[string]string{} + for _, bidderName := range openrtb_ext.CoreBidderNames() { + bidderToSyncerKey[string(bidderName)] = string(bidderName) + } + return &exchange{ adapterMap: bidderAdapters, me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames(), []string{}), @@ -1773,6 +1779,7 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string] privacyConfig: privacyConfig, categoriesFetcher: categoriesFetcher, bidderInfo: bidderInfos, + bidderToSyncerKey: bidderToSyncerKey, externalURL: "http://localhost", } } @@ -2840,8 +2847,8 @@ type bidderBid struct { type mockIdFetcher map[string]string -func (f mockIdFetcher) GetId(bidder openrtb_ext.BidderName) (id string, ok bool) { - id, ok = f[string(bidder)] +func (f mockIdFetcher) GetUID(key string) (uid string, exists bool, notExpired bool) { + uid, exists = f[string(key)] return } @@ -3015,8 +3022,8 @@ func (c *wellBehavedCache) PutJson(ctx context.Context, values []pbc.Cacheable) type emptyUsersync struct{} -func (e *emptyUsersync) GetId(bidder openrtb_ext.BidderName) (string, bool) { - return "", false +func (e *emptyUsersync) GetUID(key string) (uid string, exists bool, notExpired bool) { + return "", false, false } func (e *emptyUsersync) HasAnyLiveSyncs() bool { diff --git a/exchange/utils.go b/exchange/utils.go index 3d3f08a0ae8..5355d82e7ac 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -55,6 +55,7 @@ func BidderToPrebidSChains(req *openrtb_ext.ExtRequest) (map[string]*openrtb_ext func cleanOpenRTBRequests(ctx context.Context, req AuctionRequest, requestExt *openrtb_ext.ExtRequest, + bidderToSyncerKey map[string]string, gDPR gdpr.Permissions, usersyncIfAmbiguous bool, privacyConfig config.Privacy) (bidderRequests []BidderRequest, privacyLabels metrics.PrivacyLabels, errs []error) { @@ -70,7 +71,7 @@ func cleanOpenRTBRequests(ctx context.Context, return } - bidderRequests, errs = getAuctionBidderRequests(req, requestExt, impsByBidder, aliases) + bidderRequests, errs = getAuctionBidderRequests(req, requestExt, bidderToSyncerKey, impsByBidder, aliases) if len(bidderRequests) == 0 { return @@ -181,6 +182,7 @@ func extractLMT(orig *openrtb.BidRequest, privacyConfig config.Privacy) privacy. func getAuctionBidderRequests(req AuctionRequest, requestExt *openrtb_ext.ExtRequest, + bidderToSyncerKey map[string]string, impsByBidder map[string][]openrtb.Imp, aliases map[string]string) ([]BidderRequest, []error) { @@ -232,7 +234,8 @@ func getAuctionBidderRequests(req AuctionRequest, }, } - if hadSync := prepareUser(&reqCopy, bidder, coreBidder, explicitBuyerUIDs, req.UserSyncs); !hadSync && req.BidRequest.App == nil { + syncerKey := bidderToSyncerKey[string(coreBidder)] + if hadSync := prepareUser(&reqCopy, bidder, syncerKey, explicitBuyerUIDs, req.UserSyncs); !hadSync && req.BidRequest.App == nil { bidderRequest.BidderLabels.CookieFlag = metrics.CookieFlagNo } else { bidderRequest.BidderLabels.CookieFlag = metrics.CookieFlagYes @@ -428,8 +431,8 @@ func isSpecialField(bidder string) bool { // // In this function, "givenBidder" may or may not be an alias. "coreBidder" must *not* be an alias. // It returns true if a Cookie User Sync existed, and false otherwise. -func prepareUser(req *openrtb.BidRequest, givenBidder string, coreBidder openrtb_ext.BidderName, explicitBuyerUIDs map[string]string, usersyncs IdFetcher) bool { - cookieId, hadCookie := usersyncs.GetId(coreBidder) // TODO: this will need to have a lookup from bidder name to cookie key +func prepareUser(req *openrtb.BidRequest, givenBidder, syncerKey string, explicitBuyerUIDs map[string]string, usersyncs IdFetcher) bool { + cookieId, hadCookie, _ := usersyncs.GetUID(syncerKey) if id, ok := explicitBuyerUIDs[givenBidder]; ok { req.User = copyWithBuyerUID(req.User, id) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 1945aee1b98..3d6b2d322d6 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -442,6 +442,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { applyCOPPA: false, consentedVendors: map[string]bool{"appnexus": true, "brightroll": true}, }, + // identity with user syncs } privacyConfig := config.Privacy{ @@ -454,7 +455,8 @@ func TestCleanOpenRTBRequests(t *testing.T) { } for _, test := range testCases { - bidderRequests, _, err := cleanOpenRTBRequests(context.Background(), test.req, nil, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) + bidderToSyncerKey := map[string]string{} + bidderRequests, _, err := cleanOpenRTBRequests(context.Background(), test.req, nil, bidderToSyncerKey, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) if test.hasError { assert.NotNil(t, err, "Error shouldn't be nil") } else { @@ -605,10 +607,12 @@ func TestCleanOpenRTBRequestsCCPA(t *testing.T) { Account: accountConfig, } + bidderToSyncerKey := map[string]string{} bidderRequests, privacyLabels, errs := cleanOpenRTBRequests( context.Background(), auctionReq, nil, + bidderToSyncerKey, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) @@ -669,7 +673,9 @@ func TestCleanOpenRTBRequestsCCPAErrors(t *testing.T) { Enforce: true, }, } - _, _, errs := cleanOpenRTBRequests(context.Background(), auctionReq, &reqExtStruct, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) + + bidderToSyncerKey := map[string]string{} + _, _, errs := cleanOpenRTBRequests(context.Background(), auctionReq, &reqExtStruct, bidderToSyncerKey, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) assert.ElementsMatch(t, []error{test.expectError}, errs, test.description) } @@ -709,7 +715,8 @@ func TestCleanOpenRTBRequestsCOPPA(t *testing.T) { UserSyncs: &emptyUsersync{}, } - bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(context.Background(), auctionReq, nil, &permissionsMock{personalInfoAllowed: true}, true, config.Privacy{}) + bidderToSyncerKey := map[string]string{} + bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(context.Background(), auctionReq, nil, bidderToSyncerKey, &permissionsMock{personalInfoAllowed: true}, true, config.Privacy{}) result := bidderRequests[0] assert.Nil(t, errs) @@ -816,7 +823,8 @@ func TestCleanOpenRTBRequestsSChain(t *testing.T) { UserSyncs: &emptyUsersync{}, } - bidderRequests, _, errs := cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, &permissionsMock{}, true, config.Privacy{}) + bidderToSyncerKey := map[string]string{} + bidderRequests, _, errs := cleanOpenRTBRequests(context.Background(), auctionReq, extRequest, bidderToSyncerKey, &permissionsMock{}, true, config.Privacy{}) if test.hasError == true { assert.NotNil(t, errs) assert.Len(t, bidderRequests, 0) @@ -1397,7 +1405,8 @@ func TestCleanOpenRTBRequestsLMT(t *testing.T) { }, } - results, privacyLabels, errs := cleanOpenRTBRequests(context.Background(), auctionReq, nil, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) + bidderToSyncerKey := map[string]string{} + results, privacyLabels, errs := cleanOpenRTBRequests(context.Background(), auctionReq, nil, bidderToSyncerKey, &permissionsMock{personalInfoAllowed: true}, true, privacyConfig) result := results[0] assert.Nil(t, errs) @@ -1608,10 +1617,12 @@ func TestCleanOpenRTBRequestsGDPR(t *testing.T) { Account: accountConfig, } + bidderToSyncerKey := map[string]string{} results, privacyLabels, errs := cleanOpenRTBRequests( context.Background(), auctionReq, nil, + bidderToSyncerKey, &permissionsMock{personalInfoAllowed: !test.gdprScrub, personalInfoAllowedError: test.permissionsError}, test.userSyncIfAmbiguous, privacyConfig) diff --git a/metrics/config/todo b/metrics/config/todo index 2d83fa8336c..7e7dfbd7cac 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,6 +1,7 @@ todo - override urls from config in bidder info - add new cookiesync metrics to go metrics + - rename setuid metrics to match new cookiesync metric names - add proper samesite behavior - cookie should just know what to do [open pr] diff --git a/router/router.go b/router/router.go index 882bd86f9c7..44148ae8acf 100644 --- a/router/router.go +++ b/router/router.go @@ -261,7 +261,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatalf("%v", errs) } - theExchange := exchange.NewExchange(adapters, cacheClient, cfg, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor, categoriesFetcher) + theExchange := exchange.NewExchange(adapters, cacheClient, cfg, syncers, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor, categoriesFetcher) openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, fetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders) if err != nil { diff --git a/usersync/cookie.go b/usersync/cookie.go index 243b61dc8c2..6b16c8157d3 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -20,12 +20,6 @@ const uidCookieName = "uids" // separate from the cookie ttl. const uidTTL = 14 * 24 * time.Hour -// bidderToFamilyNames maps the BidderName to Adapter.Name() for the early adapters. -// If a mapping isn't listed here, then we assume that the two are the same. -var bidderToFamilyNames = map[openrtb_ext.BidderName]string{ - openrtb_ext.BidderAppnexus: "adnxs", -} - // Cookie is the cookie used in Prebid Server. // // To get an instance of this from a request, use ParseCookieFromRequest. @@ -124,15 +118,15 @@ func (cookie *Cookie) ToHTTPCookie(ttl time.Duration) *http.Cookie { } } -// GetUID Gets this user's ID for the given family. +// GetUID Gets this user's ID for the given syncer key. // The first returned value is the user's ID. // The second returned value is true if we had a value stored, and false if we didn't. // The third returned value is true if that value is "active", and false if it's expired. // // If no value was stored, then the "isActive" return value will be false. -func (cookie *Cookie) GetUID(familyName string) (string, bool, bool) { +func (cookie *Cookie) GetUID(key string) (string, bool, bool) { if cookie != nil { - if uid, ok := cookie.uids[familyName]; ok { + if uid, ok := cookie.uids[key]; ok { return uid.UID, true, time.Now().Before(uid.Expires) } } @@ -151,16 +145,6 @@ func (cookie *Cookie) GetUIDs() map[string]string { return uids } -// GetId wraps GetUID, letting callers fetch the ID given an OpenRTB BidderName. -func (cookie *Cookie) GetId(bidderName openrtb_ext.BidderName) (id string, exists bool) { - if familyName, ok := bidderToFamilyNames[bidderName]; ok { - id, exists, _ = cookie.GetUID(familyName) - } else { - id, exists, _ = cookie.GetUID(string(bidderName)) - } - return -} - // SetCookieOnResponse is a shortcut for "ToHTTPCookie(); cookie.setDomain(domain); setCookie(w, cookie)" func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie bool, cfg *config.HostCookie, ttl time.Duration) { httpCookie := cookie.ToHTTPCookie(ttl) @@ -196,14 +180,14 @@ func (cookie *Cookie) SetCookieOnResponse(w http.ResponseWriter, setSiteCookie b w.Header().Add("Set-Cookie", httpCookie.String()) } -// Unsync removes the user's ID for the given family from this cookie. -func (cookie *Cookie) Unsync(familyName string) { - delete(cookie.uids, familyName) +// Unsync removes the user's ID for the given syncer key from this cookie. +func (cookie *Cookie) Unsync(key string) { + delete(cookie.uids, key) } -// HasLiveSync returns true if we have an active UID for the given family, and false otherwise. -func (cookie *Cookie) HasLiveSync(familyName string) bool { - _, _, isLive := cookie.GetUID(familyName) +// HasLiveSync returns true if we have an active UID for the given syncer key, and false otherwise. +func (cookie *Cookie) HasLiveSync(key string) bool { + _, _, isLive := cookie.GetUID(key) return isLive } @@ -220,19 +204,19 @@ func (cookie *Cookie) HasAnyLiveSyncs() bool { return false } -// TrySync tries to set the UID for some family name. It returns an error if the set didn't happen. -func (cookie *Cookie) TrySync(familyName string, uid string) error { +// TrySync tries to set the UID for some syncer key. It returns an error if the set didn't happen. +func (cookie *Cookie) TrySync(key string, uid string) error { if !cookie.AllowSyncs() { return errors.New("The user has opted out of prebid server cookie syncs.") } // At the moment, Facebook calls /setuid with a UID of 0 if the user isn't logged into Facebook. // They shouldn't be sending us a sentinel value... but since they are, we're refusing to save that ID. - if familyName == string(openrtb_ext.BidderAudienceNetwork) && uid == "0" { + if key == string(openrtb_ext.BidderAudienceNetwork) && uid == "0" { return errors.New("audienceNetwork uses a UID of 0 as \"not yet recognized\".") } - cookie.uids[familyName] = uidWithExpiry{ + cookie.uids[key] = uidWithExpiry{ UID: uid, Expires: time.Now().Add(uidTTL), } diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index cc464827fc5..f3944a389c3 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/openrtb_ext" "github.com/stretchr/testify/assert" ) @@ -48,7 +47,7 @@ func TestCookieWithData(t *testing.T) { func TestBidderNameGets(t *testing.T) { cookie := newSampleCookie() - id, exists := cookie.GetId(openrtb_ext.BidderAppnexus) + id, exists, _ := cookie.GetUID("adnxs") if !exists { t.Errorf("Cookie missing expected Appnexus ID") } @@ -56,7 +55,7 @@ func TestBidderNameGets(t *testing.T) { t.Errorf("Bad appnexus id. Expected %s, got %s", "123", id) } - id, exists = cookie.GetId(openrtb_ext.BidderRubicon) + id, exists, _ = cookie.GetUID("rubicon") if !exists { t.Errorf("Cookie missing expected Rubicon ID") } @@ -313,15 +312,6 @@ func TestNilCookie(t *testing.T) { if isLive { t.Error("nil cookies shouldn't report live UID mappings.") } - - uid, hadUID = nilCookie.GetId("anything") - - if uid != "" { - t.Error("nil cookies should return empty strings for the UID.") - } - if hadUID { - t.Error("nil cookies shouldn't claim to have a UID mapping.") - } } func TestGetUIDs(t *testing.T) { @@ -378,7 +368,7 @@ func TestTrimCookiesClosestExpirationDates(t *testing.T) { processedCookie := writeThenRead(cookieToSend, testCases[i].maxCookieSize) actualKeys := make([]string, 0, 7) - for key, _ := range processedCookie.uids { + for key := range processedCookie.uids { actualKeys = append(actualKeys, key) } diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index dd9a10a9508..e5deb25000f 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -1,6 +1,7 @@ package usersync import ( + "strings" "testing" "github.com/prebid/prebid-server/config" @@ -24,10 +25,11 @@ func TestBuildSyncers(t *testing.T) { // in these tests. Look carefully at the end of the expected iframe urls to see the syncer key. testCases := []struct { - description string - givenBidderInfos config.BidderInfos - expectedIFramesURLs map[string]string - expectedError string + description string + givenBidderInfos config.BidderInfos + expectedIFramesURLs map[string]string + expectedErrorHeader string + expectedErrorSegments []string }{ { description: "One", @@ -44,9 +46,12 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "One - Syncer Error", - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, - expectedError: "user sync (1 error):\n 1: cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "One - Syncer Error", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + }, }, { description: "Many - Different Syncers", @@ -65,14 +70,20 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Same Syncers - Many Primaries", - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, - expectedError: "user sync (1 error):\n 1: bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", + description: "Many - Same Syncers - Many Primaries", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", + }, }, { - description: "Many - Sync Error - Bidder Correct", - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, - expectedError: "user sync (1 error):\n 1: cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "Many - Sync Error - Bidder Correct", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", + }, }, { description: "Many - Empty Syncers Ignored", @@ -82,16 +93,20 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Multiple Errors", - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, - expectedError: "user sync (2 errors):\n 1: cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n 2: cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", + description: "Many - Multiple Errors", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, + expectedErrorHeader: "user sync (2 errors)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + "cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", + }, }, } for _, test := range testCases { result, err := BuildSyncers(hostConfig, test.givenBidderInfos) - if test.expectedError == "" { + if test.expectedErrorHeader == "" { assert.NoError(t, err, test.description+":err") resultRenderedIFrameURLS := map[string]string{} for k, v := range result { @@ -102,7 +117,11 @@ func TestBuildSyncers(t *testing.T) { } assert.Equal(t, test.expectedIFramesURLs, resultRenderedIFrameURLS, test.description+":result") } else { - assert.EqualError(t, err, test.expectedError, test.description+":err") + errMessage := err.Error() + assert.True(t, strings.HasPrefix(errMessage, test.expectedErrorHeader), test.description+":err") + for _, s := range test.expectedErrorSegments { + assert.Contains(t, errMessage, s, test.description+":err") + } assert.Empty(t, result, test.description+":result") } } From 684d997d9fc47747494c01c76d4d8de1f6814e93 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 11:54:17 -0400 Subject: [PATCH 12/50] Initial Support For Config Overrides --- config/bidderinfo.go | 91 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 4813534d77f..3f86c07cce8 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -44,27 +44,63 @@ type DebugInfo struct { Allow bool `yaml:"allow"` } -// Syncer specifies the user sync settings for a bidder. +// Syncer specifies the user sync settings for a bidder. This struct is shared by the account config, +// so it needs to have both yaml and mapstructure mappings. type Syncer struct { // Key is used as the record key for the user sync cookie. We recommend using the bidder name // as the key for consistency, but that is not enforced as a requirement. Each bidder must // have a unique key. - Key string `yaml:"key"` + Key string `yaml:"key" mapstructure:"key"` // Default identifies which endpoint is preferred if both are allowed by the publisher. This is // only required if there is more than one endpoint configured for the bidder. Valid values are // `iframe` and `redirect`. - Default string `yaml:"default"` + Default string `yaml:"default" mapstructure:"default"` // IFrame configures an iframe endpoint for user syncing. - IFrame *SyncerEndpoint `yaml:"iframe"` + IFrame *SyncerEndpoint `yaml:"iframe" mapstructure:"iframe"` // Redirect configures an redirect endpoint for user syncing. This is also known as an image // endpoint in the Prebid.js project. - Redirect *SyncerEndpoint `yaml:"redirect"` + Redirect *SyncerEndpoint `yaml:"redirect" mapstructure:"redirect"` // SupportCORS identifies if CORS is supported for user syncing. - SupportCORS bool + SupportCORS *bool `yaml:"supportcors" mapstructure:"supportcors"` +} + +func (s *Syncer) ApplyTo(v *Syncer) *Syncer { + if s.Empty() { + return v + } + + if v == nil { + v = &Syncer{} + } + + if s.Key != "" { + v.Key = s.Key + } + + if s.Default != "" { + v.Key = s.Default + } + + v.IFrame = s.IFrame.ApplyTo(v.IFrame) + + v.Redirect = s.IFrame.ApplyTo(v.Redirect) + + if s.SupportCORS != nil { + v.SupportCORS = s.SupportCORS + } + + return v +} + +func (s *Syncer) Empty() bool { + if s == nil { + return true + } + return s.Key == "" && s.Default == "" && s.IFrame.Empty() && s.Redirect.Empty() && s.SupportCORS == nil } // SyncerEndpoint specifies the configuration of the URL returned by the /cookie_sync endpoint @@ -97,7 +133,7 @@ type SyncerEndpoint struct { // {{.GDPR}} - This will be replaced with the "gdpr" property sent to /cookie_sync. // {{.Consent}} - This will be replaced with the "consent" property sent to /cookie_sync. // {{.USPrivacy}} - This will be replaced with the "us_privacy" property sent to /cookie_sync. - URL string + URL string `yaml:"url" mapstructure:"url"` // RedirectURL is an endpoint on the host server the user will be redirected to when a user sync // request has been completed by the bidder server. The following macros are resolved at application @@ -111,15 +147,50 @@ type SyncerEndpoint struct { // // The endpoint on the host server is usually Prebid Server's /setuid endpoint. The default value is: // `{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&f={{.SyncType}}&uid={{.UserMacro}}` - RedirectURL string + RedirectURL string `yaml:"redirectUrl" mapstructure:"redirect_url"` // ExternalURL is available as a macro to the RedirectURL template. If not specified, the host configuration // value is used. - ExternalURL string + ExternalURL string `yaml:"externalUrl" mapstructure:"external_url"` // UserMacro is available as a macro to the RedirectURL template. This value is specific to the bidder server // and has no default. - UserMacro string + UserMacro string `yaml:"userMacro" mapstructure:"user_macro"` +} + +func (s *SyncerEndpoint) ApplyTo(v *SyncerEndpoint) *SyncerEndpoint { + if s.Empty() { + return v + } + + if v == nil { + v = &SyncerEndpoint{} + } + + if s.URL != "" { + v.URL = s.URL + } + + if s.RedirectURL != "" { + v.RedirectURL = s.RedirectURL + } + + if s.ExternalURL != "" { + v.ExternalURL = s.ExternalURL + } + + if s.UserMacro != "" { + v.UserMacro = s.UserMacro + } + + return v +} + +func (s *SyncerEndpoint) Empty() bool { + if s == nil { + return true + } + return s.URL == "" && s.RedirectURL == "" && s.ExternalURL == "" && s.UserMacro == "" } // LoadBidderInfoFromDisk parses all static/bidder-info/{bidder}.yaml files from the file system. From c3217a7e2cf3713c430a5701d07d54a0460d9ef2 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 11:54:40 -0400 Subject: [PATCH 13/50] Stuff I Missed From The Last Commit --- config/adapter.go | 8 ++++---- metrics/config/todo | 3 +-- router/router.go | 7 ++++++- usersync/syncer.go | 6 +++--- usersync/syncer_test.go | 2 +- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/config/adapter.go b/config/adapter.go index 666fbcddb3d..eae47981a70 100644 --- a/config/adapter.go +++ b/config/adapter.go @@ -9,10 +9,10 @@ import ( ) type Adapter struct { - Disabled bool `mapstructure:"disabled"` - Endpoint string `mapstructure:"endpoint"` - ExtraAdapterInfo string `mapstructure:"extra_info"` - // todo: bidder info overrides + Disabled bool `mapstructure:"disabled"` + Endpoint string `mapstructure:"endpoint"` + ExtraAdapterInfo string `mapstructure:"extra_info"` + Syncer *Syncer `mapstructure:"usersync"` // needed for backwards compatibility UserSyncURL string `mapstructure:"usersync_url"` diff --git a/metrics/config/todo b/metrics/config/todo index 7e7dfbd7cac..587bdcc7d65 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,8 +1,7 @@ todo - - override urls from config in bidder info + - override bidder info tests - add new cookiesync metrics to go metrics - rename setuid metrics to match new cookiesync metric names - - add proper samesite behavior - cookie should just know what to do [open pr] diff --git a/router/router.go b/router/router.go index 44148ae8acf..c8522200ff4 100644 --- a/router/router.go +++ b/router/router.go @@ -234,7 +234,12 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } - // override bidderInfos user sync with default (iframe? redirect? both? - no, not both. error if both are defined.) + // apply adapter overrides to bidder info + for bidderName, bidderInfo := range bidderInfos { + if adapterCfg, exists := cfg.Adapters[bidderName]; exists { + bidderInfo.Syncer = adapterCfg.Syncer.ApplyTo(bidderInfo.Syncer) + } + } activeBidders := exchange.GetActiveBidders(bidderInfos) disabledBidders := exchange.GetDisabledBiddersErrorMessages(bidderInfos) diff --git a/usersync/syncer.go b/usersync/syncer.go index 74da8867205..a2734273e81 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -75,7 +75,7 @@ func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, syncer := standardSyncer{ key: syncerConfig.Key, - supportCORS: syncerConfig.SupportCORS, + supportCORS: syncerConfig.SupportCORS != nil && *syncerConfig.SupportCORS, } if defaultSyncType, err := resolveDefaultSyncType(syncerConfig); err != nil { @@ -158,10 +158,10 @@ func buildTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncer externalURL = hostConfig.ExternalURL } - redirectURL := macroRegexExternalHost.ReplaceAllLiteralString(redirectTemplate, externalURL) - redirectURL = macroRegexSyncerKey.ReplaceAllLiteralString(redirectURL, key) + redirectURL := macroRegexSyncerKey.ReplaceAllLiteralString(redirectTemplate, key) redirectURL = macroRegexSyncType.ReplaceAllLiteralString(redirectURL, syncTypeValue) redirectURL = macroRegexUserMacro.ReplaceAllLiteralString(redirectURL, syncerEndpoint.UserMacro) + redirectURL = macroRegexExternalHost.ReplaceAllLiteralString(redirectURL, externalURL) redirectURL = escapeTemplate(redirectURL) url := macroRegexRedirect.ReplaceAllString(syncerEndpoint.URL, redirectURL) diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 452a114d894..c43e78a09c6 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -105,7 +105,7 @@ func TestNewSyncer(t *testing.T) { for _, test := range testCases { syncerConfig := config.Syncer{ Key: test.givenKey, - SupportCORS: supportCORS, + SupportCORS: &supportCORS, Default: test.givenDefault, IFrame: test.givenIFrameConfig, Redirect: test.givenRedirectConfig, From 021d57d7a310127b73ef82dd9bdcbbdc1ffd0b97 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Fri, 4 Jun 2021 15:37:18 -0400 Subject: [PATCH 14/50] Fixed Metric Tests + Added Config Bindings --- config/bidderinfo.go | 2 +- config/config.go | 37 +++++++++---- endpoints/openrtb2/amp_auction_test.go | 39 ++++++------- endpoints/openrtb2/auction_benchmark_test.go | 14 +++-- endpoints/openrtb2/auction_test.go | 58 ++++++++++---------- endpoints/openrtb2/video_auction_test.go | 16 ++++-- metrics/config/metrics.go | 2 +- metrics/config/metrics_test.go | 3 +- metrics/config/todo | 10 +++- metrics/go_metrics.go | 46 ++++++++++------ metrics/go_metrics_test.go | 30 +++++----- server/listener_test.go | 2 +- 12 files changed, 146 insertions(+), 113 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 3f86c07cce8..5cded10c1ef 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -65,7 +65,7 @@ type Syncer struct { Redirect *SyncerEndpoint `yaml:"redirect" mapstructure:"redirect"` // SupportCORS identifies if CORS is supported for user syncing. - SupportCORS *bool `yaml:"supportcors" mapstructure:"supportcors"` + SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` } func (s *Syncer) ApplyTo(v *Syncer) *Syncer { diff --git a/config/config.go b/config/config.go index a62a1ab469d..a765b9ca95c 100644 --- a/config/config.go +++ b/config/config.go @@ -902,18 +902,33 @@ func migrateConfig(v *viper.Viper) { } } +var nilBoolean *bool + func setBidderDefaults(v *viper.Viper, bidder string) { - adapterCfgPrefix := "adapters." - v.SetDefault(adapterCfgPrefix+bidder+".endpoint", "") - v.SetDefault(adapterCfgPrefix+bidder+".usersync_url", "") - v.SetDefault(adapterCfgPrefix+bidder+".platform_id", "") - v.SetDefault(adapterCfgPrefix+bidder+".app_secret", "") - v.SetDefault(adapterCfgPrefix+bidder+".xapi.username", "") - v.SetDefault(adapterCfgPrefix+bidder+".xapi.password", "") - v.SetDefault(adapterCfgPrefix+bidder+".xapi.tracker", "") - v.SetDefault(adapterCfgPrefix+bidder+".disabled", false) - v.SetDefault(adapterCfgPrefix+bidder+".partner_id", "") - v.SetDefault(adapterCfgPrefix+bidder+".extra_info", "") + adapterCfgPrefix := "adapters." + bidder + v.SetDefault(adapterCfgPrefix+".endpoint", "") + v.SetDefault(adapterCfgPrefix+".usersync_url", "") + v.SetDefault(adapterCfgPrefix+".platform_id", "") + v.SetDefault(adapterCfgPrefix+".app_secret", "") + v.SetDefault(adapterCfgPrefix+".xapi.username", "") + v.SetDefault(adapterCfgPrefix+".xapi.password", "") + v.SetDefault(adapterCfgPrefix+".xapi.tracker", "") + v.SetDefault(adapterCfgPrefix+".disabled", false) + v.SetDefault(adapterCfgPrefix+".partner_id", "") + v.SetDefault(adapterCfgPrefix+".extra_info", "") + + // user sync + v.SetDefault(adapterCfgPrefix+".usersync.key", "") + v.SetDefault(adapterCfgPrefix+".usersync.default", "") + v.SetDefault(adapterCfgPrefix+".usersync.iframe.url", "") + v.SetDefault(adapterCfgPrefix+".usersync.iframe.redirect_url", "") + v.SetDefault(adapterCfgPrefix+".usersync.iframe.external_url", "") + v.SetDefault(adapterCfgPrefix+".usersync.iframe.user_macro", "") + v.SetDefault(adapterCfgPrefix+".usersync.redirect.url", "") + v.SetDefault(adapterCfgPrefix+".usersync.redirect.redirect_url", "") + v.SetDefault(adapterCfgPrefix+".usersync.redirect.external_url", "") + v.SetDefault(adapterCfgPrefix+".usersync.redirect.user_macro", "") + v.SetDefault(adapterCfgPrefix+".usersync.redirect.support_cors", nilBoolean) } func isValidCookieSize(maxCookieSize int) error { diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go index 77e76613c29..3b7dd031abe 100644 --- a/endpoints/openrtb2/amp_auction_test.go +++ b/endpoints/openrtb2/amp_auction_test.go @@ -12,15 +12,14 @@ import ( "testing" "github.com/prebid/prebid-server/analytics" - "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" - - "github.com/mxmCherry/openrtb" analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/exchange" - "github.com/prebid/prebid-server/metrics" + metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" - gometrics "github.com/rcrowley/go-metrics" + "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + + "github.com/mxmCherry/openrtb" "github.com/stretchr/testify/assert" ) @@ -39,15 +38,13 @@ func TestGoodAmpRequests(t *testing.T) { "9": json.RawMessage(validRequest(t, "user.json")), } - // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. - // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. endpoint, _ := NewAmpEndpoint( &mockAmpExchange{}, newParamsValidator(t), &mockAmpStoredReqFetcher{goodRequests}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -99,7 +96,7 @@ func TestAMPPageInfo(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -203,7 +200,7 @@ func TestGDPRConsent(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -355,7 +352,7 @@ func TestCCPAConsent(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -465,7 +462,7 @@ func TestConsentWarnings(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -557,7 +554,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -608,7 +605,7 @@ func TestAMPSiteExt(t *testing.T) { &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), nil, nil, @@ -644,7 +641,7 @@ func TestAmpBadRequests(t *testing.T) { &mockAmpStoredReqFetcher{badRequests}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -674,7 +671,7 @@ func TestAmpDebug(t *testing.T) { &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -746,7 +743,7 @@ func TestQueryParamOverrides(t *testing.T) { &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -898,7 +895,7 @@ func (s formatOverrideSpec) execute(t *testing.T) { &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -1277,7 +1274,7 @@ func TestBuildAmpObject(t *testing.T) { mockAmpFetcher, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, logger, map[string]string{}, []byte{}, @@ -1296,7 +1293,3 @@ func TestBuildAmpObject(t *testing.T) { assert.Equalf(t, test.expectedAmpObject.Origin, actualAmpObject.Origin, "Amp Object Origin field doesn't match expected: %s\n", test.description) } } - -func newTestMetrics() *metrics.Metrics { - return metrics.NewMetrics(gometrics.NewRegistry(), openrtb_ext.CoreBidderNames(), config.DisabledMetrics{}) -} diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index 7fb4cbad7fd..ecc0554f45a 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -7,15 +7,15 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/currency" - "github.com/prebid/prebid-server/usersync" - analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/currency" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/gdpr" + metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/usersync" ) // dummyServer returns the header bidding test ad. This response was scraped from a real appnexus server response. @@ -67,7 +67,9 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { return } - adapters, adaptersErr := exchange.BuildAdapters(server.Client(), &config.Configuration{}, infos, newTestMetrics()) + nilMetrics := &metricsConfig.DummyMetricsEngine{} + + adapters, adaptersErr := exchange.BuildAdapters(server.Client(), &config.Configuration{}, infos, nilMetrics) if adaptersErr != nil { b.Fatal("unable to build adapters") } @@ -77,7 +79,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { nil, &config.Configuration{}, map[string]usersync.Syncer{}, - newTestMetrics(), + nilMetrics, infos, gdpr.AlwaysAllow{}, currency.NewRateConverter(&http.Client{}, "", time.Duration(0)), @@ -90,7 +92,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + nilMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 19c58f05ae0..e55da570a77 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -17,19 +17,20 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/stored_requests" - - "github.com/buger/jsonparser" - jsonpatch "github.com/evanphx/json-patch" - "github.com/mxmCherry/openrtb" analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/metrics" + metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/util/iputil" + + "github.com/buger/jsonparser" + jsonpatch "github.com/evanphx/json-patch" + "github.com/mxmCherry/openrtb" "github.com/stretchr/testify/assert" ) @@ -406,15 +407,14 @@ func TestExplicitUserId(t *testing.T) { Name: cookieName, Value: mockId, }) - // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. - // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. + endpoint, _ := NewEndpoint( ex, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -453,7 +453,7 @@ func doRequest(t *testing.T, test testCase) (int, string) { BlacklistedAcctMap: test.Config.getBlackListedAccountMap(), AccountRequired: test.Config.AccountRequired, }, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, []byte(test.Config.AliasJSON), @@ -516,7 +516,7 @@ func doBadAliasRequest(t *testing.T, filename string, expectMsg string) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, aliasJSON, @@ -566,7 +566,7 @@ func TestNilExchange(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap()) @@ -586,7 +586,7 @@ func TestNilValidator(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -607,7 +607,7 @@ func TestExchangeError(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -729,7 +729,7 @@ func TestImplicitIPsEndToEnd(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -923,7 +923,7 @@ func TestImplicitDNTEndToEnd(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -986,7 +986,7 @@ func TestStoredRequests(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1025,7 +1025,7 @@ func TestOversizedRequest(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(len(reqBody) - 1)}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1060,7 +1060,7 @@ func TestRequestSizeEdgeCase(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(len(reqBody))}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1093,7 +1093,7 @@ func TestNoEncoding(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -1168,7 +1168,7 @@ func TestContentType(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -1381,7 +1381,7 @@ func TestValidateImpExt(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(8096)}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{"disabledbidder": "The bidder 'disabledbidder' has been disabled."}, false, @@ -1427,7 +1427,7 @@ func TestCurrencyTrunc(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1471,7 +1471,7 @@ func TestCCPAInvalid(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1521,7 +1521,7 @@ func TestNoSaleInvalid(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1572,7 +1572,7 @@ func TestValidateSourceTID(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1613,7 +1613,7 @@ func TestSChainInvalid(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1832,7 +1832,7 @@ func TestEidPermissionsInvalid(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -2087,7 +2087,7 @@ func TestIOS14EndToEnd(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, @@ -2115,7 +2115,7 @@ func TestAuctionWarnings(t *testing.T) { empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(len(reqBody))}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index a70d45ac3b8..8fb21a37de5 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -12,15 +12,18 @@ import ( "strings" "testing" - "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/analytics" analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/metrics" + metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + + "github.com/mxmCherry/openrtb" + gometrics "github.com/rcrowley/go-metrics" "github.com/stretchr/testify/assert" ) @@ -1199,7 +1202,9 @@ func TestFormatTargetingKeyLongKey(t *testing.T) { func mockDepsWithMetrics(t *testing.T, ex *mockExchangeVideo) (*endpointDeps, *metrics.Metrics, *mockAnalyticsModule) { mockModule := &mockAnalyticsModule{} - metrics := newTestMetrics() + + metrics := metrics.NewMetrics(gometrics.NewRegistry(), openrtb_ext.CoreBidderNames(), config.DisabledMetrics{}, nil) + deps := &endpointDeps{ ex, newParamsValidator(t), @@ -1217,7 +1222,6 @@ func mockDepsWithMetrics(t *testing.T, ex *mockExchangeVideo) (*endpointDeps, *m nil, hardcodedResponseIPValidator{response: true}, } - return deps, metrics, mockModule } @@ -1250,7 +1254,7 @@ func mockDeps(t *testing.T, ex *mockExchangeVideo) *endpointDeps { &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1272,7 +1276,7 @@ func mockDepsAppendBidderNames(t *testing.T, ex *mockExchangeAppendBidderNames) &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, @@ -1294,7 +1298,7 @@ func mockDepsNoBids(t *testing.T, ex *mockExchangeVideoNoBids) *endpointDeps { &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, - newTestMetrics(), + &metricsConfig.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, false, diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index e68a9e1e7c1..776640d42e6 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -22,7 +22,7 @@ func NewMetricsEngine(cfg *config.Configuration, adapterList []openrtb_ext.Bidde if cfg.Metrics.Influxdb.Host != "" { // Currently use go-metrics as the metrics piece for influx - returnEngine.GoMetrics = metrics.NewMetrics(gometrics.NewPrefixedRegistry("prebidserver."), adapterList, cfg.Metrics.Disabled) + returnEngine.GoMetrics = metrics.NewMetrics(gometrics.NewPrefixedRegistry("prebidserver."), adapterList, cfg.Metrics.Disabled, syncerKeys) engineList = append(engineList, returnEngine.GoMetrics) // Set up the Influx logger go influxdb.InfluxDB( diff --git a/metrics/config/metrics_test.go b/metrics/config/metrics_test.go index 62fc7992415..e8b336ff8c7 100644 --- a/metrics/config/metrics_test.go +++ b/metrics/config/metrics_test.go @@ -7,6 +7,7 @@ import ( mainConfig "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/metrics" "github.com/prebid/prebid-server/openrtb_ext" + gometrics "github.com/rcrowley/go-metrics" ) @@ -39,7 +40,7 @@ func TestMultiMetricsEngine(t *testing.T) { cfg := mainConfig.Configuration{} cfg.Metrics.Influxdb.Host = "localhost" adapterList := openrtb_ext.CoreBidderNames() - goEngine := metrics.NewMetrics(gometrics.NewPrefixedRegistry("prebidserver."), adapterList, mainConfig.DisabledMetrics{}) + goEngine := metrics.NewMetrics(gometrics.NewPrefixedRegistry("prebidserver."), adapterList, mainConfig.DisabledMetrics{}, nil) engineList := make(MultiMetricsEngine, 2) engineList[0] = goEngine engineList[1] = &DummyMetricsEngine{} diff --git a/metrics/config/todo b/metrics/config/todo index 587bdcc7d65..d8445811f30 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -1,7 +1,13 @@ todo - - override bidder info tests - - add new cookiesync metrics to go metrics - rename setuid metrics to match new cookiesync metric names + - GoMetrics + - prometheus + + - add metric tests + - GoMetrics + - prometheus + + - override bidder info tests [open pr] diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index 2621369bd9e..ad77c081dd2 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -38,8 +38,8 @@ type Metrics struct { RequestStatuses map[RequestType]map[RequestStatus]metrics.Meter AmpNoCookieMeter metrics.Meter CookieSyncMeter metrics.Meter - CookieSyncGen map[openrtb_ext.BidderName]metrics.Meter - CookieSyncGDPRPrevent map[openrtb_ext.BidderName]metrics.Meter + CookieSyncStatusMeter map[CookieSyncStatus]metrics.Meter + SyncerRequestsMeter map[string]map[SyncerStatus]metrics.Meter userSyncOptout metrics.Meter userSyncBadRequest metrics.Meter userSyncSet map[openrtb_ext.BidderName]metrics.Meter @@ -66,7 +66,6 @@ type Metrics struct { // Don't export accountMetrics because we need helper functions here to insure its properly populated dynamically accountMetrics map[string]*accountMetrics accountMetricsRWMutex sync.RWMutex - userSyncRwMutex sync.RWMutex exchanges []openrtb_ext.BidderName // Will hold boolean values to help us disable metric collection if needed @@ -139,8 +138,8 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderNa AccountCacheMeter: make(map[CacheResult]metrics.Meter), AmpNoCookieMeter: blankMeter, CookieSyncMeter: blankMeter, - CookieSyncGen: make(map[openrtb_ext.BidderName]metrics.Meter), - CookieSyncGDPRPrevent: make(map[openrtb_ext.BidderName]metrics.Meter), + CookieSyncStatusMeter: make(map[CookieSyncStatus]metrics.Meter), + SyncerRequestsMeter: make(map[string]map[SyncerStatus]metrics.Meter), userSyncOptout: blankMeter, userSyncBadRequest: blankMeter, userSyncSet: make(map[openrtb_ext.BidderName]metrics.Meter), @@ -212,7 +211,7 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderNa // metrics object to contain only the metrics we are interested in. This would allow for debug // mode metrics. The code would allways try to record the metrics, but effectively noop if we are // using a blank meter/timer. -func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, disableAccountMetrics config.DisabledMetrics) *Metrics { +func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, disableAccountMetrics config.DisabledMetrics, syncerKeys []string) *Metrics { newMetrics := NewBlankMetrics(registry, exchanges, disableAccountMetrics) newMetrics.ConnectionCounter = metrics.GetOrRegisterCounter("active_connections", registry) newMetrics.ConnectionAcceptErrorMeter = metrics.GetOrRegisterMeter("connection_accept_errors", registry) @@ -245,16 +244,28 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d } newMetrics.AmpNoCookieMeter = metrics.GetOrRegisterMeter("amp_no_cookie_requests", registry) - newMetrics.CookieSyncMeter = metrics.GetOrRegisterMeter("cookie_sync_requests", registry) + + newMetrics.CookieSyncMeter = metrics.GetOrRegisterMeter("cookie_sync_requests", registry) // total count + + for _, s := range CookieSyncStatuses() { + newMetrics.CookieSyncStatusMeter[s] = metrics.GetOrRegisterMeter(fmt.Sprintf("cookie_sync_requests.%s", s), registry) + } + + for _, syncerKey := range syncerKeys { + newMetrics.SyncerRequestsMeter[syncerKey] = make(map[SyncerStatus]metrics.Meter) + for _, status := range SyncerStatuses() { + newMetrics.SyncerRequestsMeter[syncerKey][status] = metrics.GetOrRegisterMeter(fmt.Sprintf("syncer_requests.%s.%s", syncerKey, status), registry) + } + } + newMetrics.userSyncBadRequest = metrics.GetOrRegisterMeter("usersync.bad_requests", registry) newMetrics.userSyncOptout = metrics.GetOrRegisterMeter("usersync.opt_outs", registry) for _, a := range exchanges { - newMetrics.CookieSyncGen[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("cookie_sync.%s.gen", string(a)), registry) - newMetrics.CookieSyncGDPRPrevent[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("cookie_sync.%s.gdpr_prevent", string(a)), registry) newMetrics.userSyncSet[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("usersync.%s.sets", string(a)), registry) newMetrics.userSyncGDPRPrevent[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("usersync.%s.gdpr_prevent", string(a)), registry) registerAdapterMetrics(registry, "adapter", string(a), newMetrics.AdapterMetrics[a]) } + for typ, statusMap := range newMetrics.RequestStatuses { for stat := range statusMap { statusMap[stat] = metrics.GetOrRegisterMeter("requests."+string(stat)+"."+string(typ), registry) @@ -574,7 +585,6 @@ func (me *Metrics) RecordAdapterBidReceived(labels AdapterLabels, bidType openrt } else { glog.Errorf("bid/adm metrics map entry does not exist for type %s. This is a bug, and should be reported.", bidType) } - return } // RecordAdapterPrice implements a part of the MetricsEngine interface. Generates a histogram of winning bid prices @@ -609,16 +619,20 @@ func (me *Metrics) RecordAdapterTime(labels AdapterLabels, length time.Duration) // RecordCookieSync implements a part of the MetricsEngine interface. Records a cookie sync request func (me *Metrics) RecordCookieSync(status CookieSyncStatus) { - // TODO: add influx db metrics me.CookieSyncMeter.Mark(1) + + if meter, exists := me.CookieSyncStatusMeter[status]; exists { + meter.Mark(1) + } } // RecordSyncerRequest implements a part of the MetricsEngine interface. Records a cookie sync syncer request and status func (me *Metrics) RecordSyncerRequest(key string, status SyncerStatus) { - // me.CookieSyncGen[adapter].Mark(1) - // if blockedByPrivacy { - // me.CookieSyncGDPRPrevent[adapter].Mark(1) - // } + if keyMeter, exists := me.SyncerRequestsMeter[key]; exists { + if statusMeter, exists := keyMeter[status]; exists { + statusMeter.Mark(1) + } + } } // RecordUserIDSet implements a part of the MetricsEngine interface. Records a cookie setuid request @@ -676,7 +690,6 @@ func (me *Metrics) RecordTimeoutNotice(success bool) { } else { me.TimeoutNotificationFailure.Mark(1) } - return } func (me *Metrics) RecordRequestPrivacy(privacy PrivacyLabels) { @@ -702,7 +715,6 @@ func (me *Metrics) RecordRequestPrivacy(privacy PrivacyLabels) { if privacy.LMTEnforced { me.PrivacyLMTRequest.Mark(1) } - return } func doMark(bidder openrtb_ext.BidderName, meters map[openrtb_ext.BidderName]metrics.Meter) { diff --git a/metrics/go_metrics_test.go b/metrics/go_metrics_test.go index 2d0b9097b11..f14f7dc92ac 100644 --- a/metrics/go_metrics_test.go +++ b/metrics/go_metrics_test.go @@ -10,9 +10,11 @@ import ( "github.com/stretchr/testify/assert" ) +// todo: add tests for new sync metrics + func TestNewMetrics(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, nil) ensureContains(t, registry, "app_requests", m.AppRequestMeter) ensureContains(t, registry, "no_cookie_requests", m.NoCookieMeter) @@ -21,8 +23,6 @@ func TestNewMetrics(t *testing.T) { ensureContainsAdapterMetrics(t, registry, "adapter.appnexus", m.AdapterMetrics["appnexus"]) ensureContainsAdapterMetrics(t, registry, "adapter.rubicon", m.AdapterMetrics["rubicon"]) ensureContains(t, registry, "cookie_sync_requests", m.CookieSyncMeter) - ensureContains(t, registry, "cookie_sync.appnexus.gen", m.CookieSyncGen["appnexus"]) - ensureContains(t, registry, "cookie_sync.appnexus.gdpr_prevent", m.CookieSyncGDPRPrevent["appnexus"]) ensureContains(t, registry, "usersync.appnexus.gdpr_prevent", m.userSyncGDPRPrevent["appnexus"]) ensureContains(t, registry, "usersync.rubicon.gdpr_prevent", m.userSyncGDPRPrevent["rubicon"]) ensureContains(t, registry, "usersync.unknown.gdpr_prevent", m.userSyncGDPRPrevent["unknown"]) @@ -67,7 +67,7 @@ func TestNewMetrics(t *testing.T) { func TestRecordBidType(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{}, nil) m.RecordAdapterBidReceived(AdapterLabels{ Adapter: openrtb_ext.BidderAppnexus, @@ -84,7 +84,7 @@ func TestRecordBidType(t *testing.T) { func TestRecordGDPRRejection(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{}, nil) m.RecordUserIDSet(UserLabels{ Action: RequestActionGDPR, Bidder: openrtb_ext.BidderAppnexus, @@ -165,7 +165,7 @@ func TestRecordBidTypeDisabledConfig(t *testing.T) { for _, test := range testCases { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, test.DisabledMetrics) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, test.DisabledMetrics, nil) m.RecordAdapterBidReceived(AdapterLabels{ Adapter: openrtb_ext.BidderAppnexus, @@ -201,7 +201,7 @@ func TestRecordDNSTime(t *testing.T) { } for _, test := range testCases { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordDNSTime(test.inDnsLookupDuration) @@ -228,7 +228,7 @@ func TestRecordTLSHandshakeTime(t *testing.T) { } for _, test := range testCases { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordTLSHandshakeTime(test.tLSHandshakeDuration) @@ -333,7 +333,7 @@ func TestRecordAdapterConnections(t *testing.T) { for i, test := range testCases { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AdapterConnectionMetrics: test.in.connMetricsDisabled}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AdapterConnectionMetrics: test.in.connMetricsDisabled}, nil) m.RecordAdapterConnections(test.in.adapterName, test.in.connWasReused, test.in.connWait) @@ -345,14 +345,14 @@ func TestRecordAdapterConnections(t *testing.T) { func TestNewMetricsWithDisabledConfig(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) assert.True(t, m.MetricsDisabled.AccountAdapterDetails, "Accound adapter metrics should be disabled") } func TestRecordPrebidCacheRequestTimeWithSuccess(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordPrebidCacheRequestTime(true, 42) @@ -362,7 +362,7 @@ func TestRecordPrebidCacheRequestTimeWithSuccess(t *testing.T) { func TestRecordPrebidCacheRequestTimeWithNotSuccess(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordPrebidCacheRequestTime(false, 42) @@ -430,7 +430,7 @@ func TestRecordStoredDataFetchTime(t *testing.T) { for _, tt := range tests { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordStoredDataFetchTime(StoredDataLabels{ DataType: tt.dataType, DataFetchType: tt.fetchType, @@ -504,7 +504,7 @@ func TestRecordStoredDataError(t *testing.T) { for _, tt := range tests { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) m.RecordStoredDataError(StoredDataLabels{ DataType: tt.dataType, Error: tt.errorType, @@ -517,7 +517,7 @@ func TestRecordStoredDataError(t *testing.T) { func TestRecordRequestPrivacy(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}) + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{AccountAdapterDetails: true}, nil) // CCPA m.RecordRequestPrivacy(PrivacyLabels{ diff --git a/server/listener_test.go b/server/listener_test.go index f91dbddbc54..750d0ed8dda 100644 --- a/server/listener_test.go +++ b/server/listener_test.go @@ -25,7 +25,7 @@ func TestCloseErrorMetrics(t *testing.T) { func doTest(t *testing.T, allowAccept bool, allowClose bool) { reg := gometrics.NewRegistry() - me := metrics.NewMetrics(reg, nil, config.DisabledMetrics{}) + me := metrics.NewMetrics(reg, nil, config.DisabledMetrics{}, nil) var listener net.Listener = &mockListener{ listenSuccess: allowAccept, From d2339cf1ba88ea33a2376c30ea2e8ec9d0a7e020 Mon Sep 17 00:00:00 2001 From: prebid-jenkins Date: Sat, 5 Jun 2021 00:11:09 -0400 Subject: [PATCH 15/50] Biddeer Info Tweaks --- config/bidderinfo.go | 8 +-- config/bidderinfo_test.go | 70 ++++++++----------------- config/test/bidder-info/someBidder.yaml | 14 +++++ router/router.go | 2 +- 4 files changed, 42 insertions(+), 52 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 5cded10c1ef..f7656285575 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -68,7 +68,7 @@ type Syncer struct { SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` } -func (s *Syncer) ApplyTo(v *Syncer) *Syncer { +func (s *Syncer) Override(v *Syncer) *Syncer { if s.Empty() { return v } @@ -85,9 +85,9 @@ func (s *Syncer) ApplyTo(v *Syncer) *Syncer { v.Key = s.Default } - v.IFrame = s.IFrame.ApplyTo(v.IFrame) + v.IFrame = s.IFrame.Override(v.IFrame) - v.Redirect = s.IFrame.ApplyTo(v.Redirect) + v.Redirect = s.IFrame.Override(v.Redirect) if s.SupportCORS != nil { v.SupportCORS = s.SupportCORS @@ -158,7 +158,7 @@ type SyncerEndpoint struct { UserMacro string `yaml:"userMacro" mapstructure:"user_macro"` } -func (s *SyncerEndpoint) ApplyTo(v *SyncerEndpoint) *SyncerEndpoint { +func (s *SyncerEndpoint) Override(v *SyncerEndpoint) *SyncerEndpoint { if s.Empty() { return v } diff --git a/config/bidderinfo_test.go b/config/bidderinfo_test.go index 62a0b853abd..a3644b06a9d 100644 --- a/config/bidderinfo_test.go +++ b/config/bidderinfo_test.go @@ -10,24 +10,15 @@ import ( ) const testInfoFilesPath = "./test/bidder-info" -const testYAML = ` +const testSimpleYAML = ` maintainer: email: "some-email@domain.com" gvlVendorID: 42 -capabilities: - app: - mediaTypes: - - banner - - native - site: - mediaTypes: - - banner - - video - - native ` func TestLoadBidderInfoFromDisk(t *testing.T) { bidder := "someBidder" + trueValue := true adapterConfigs := make(map[string]Adapter) adapterConfigs[strings.ToLower(bidder)] = Adapter{} @@ -52,6 +43,23 @@ func TestLoadBidderInfoFromDisk(t *testing.T) { MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeVideo, openrtb_ext.BidTypeNative}, }, }, + Syncer: &Syncer{ + Key: "foo", + Default: "iframe", + IFrame: &SyncerEndpoint{ + URL: "https://foo.com/sync?mode=iframe&r={{.RedirectURL}}", + RedirectURL: "{{.ExternalURL}}/setuid/iframe", + ExternalURL: "https://iframe.host", + UserMacro: "%UID", + }, + Redirect: &SyncerEndpoint{ + URL: "https://foo.com/sync?mode=redirect&r={{.RedirectURL}}", + RedirectURL: "{{.ExternalURL}}/setuid/redirect", + ExternalURL: "https://redirect.host", + UserMacro: "#UID", + }, + SupportCORS: &trueValue, + }, }, } assert.Equal(t, expected, infos) @@ -71,7 +79,7 @@ func TestLoadBidderInfo(t *testing.T) { { description: "Enabled", givenConfigs: map[string]Adapter{strings.ToLower(bidder): {}}, - givenContent: testYAML, + givenContent: testSimpleYAML, expectedInfo: map[string]BidderInfo{ bidder: { Enabled: true, @@ -79,21 +87,13 @@ func TestLoadBidderInfo(t *testing.T) { Email: "some-email@domain.com", }, GVLVendorID: 42, - Capabilities: &CapabilitiesInfo{ - App: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeNative}, - }, - Site: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeVideo, openrtb_ext.BidTypeNative}, - }, - }, }, }, }, { description: "Disabled - Bidder Not Configured", givenConfigs: map[string]Adapter{}, - givenContent: testYAML, + givenContent: testSimpleYAML, expectedInfo: map[string]BidderInfo{ bidder: { Enabled: false, @@ -101,21 +101,13 @@ func TestLoadBidderInfo(t *testing.T) { Email: "some-email@domain.com", }, GVLVendorID: 42, - Capabilities: &CapabilitiesInfo{ - App: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeNative}, - }, - Site: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeVideo, openrtb_ext.BidTypeNative}, - }, - }, }, }, }, { description: "Disabled - Bidder Wrong Case", givenConfigs: map[string]Adapter{bidder: {}}, - givenContent: testYAML, + givenContent: testSimpleYAML, expectedInfo: map[string]BidderInfo{ bidder: { Enabled: false, @@ -123,21 +115,13 @@ func TestLoadBidderInfo(t *testing.T) { Email: "some-email@domain.com", }, GVLVendorID: 42, - Capabilities: &CapabilitiesInfo{ - App: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeNative}, - }, - Site: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeVideo, openrtb_ext.BidTypeNative}, - }, - }, }, }, }, { description: "Disabled - Explicitly Configured", givenConfigs: map[string]Adapter{strings.ToLower(bidder): {Disabled: false}}, - givenContent: testYAML, + givenContent: testSimpleYAML, expectedInfo: map[string]BidderInfo{ bidder: { Enabled: true, @@ -145,14 +129,6 @@ func TestLoadBidderInfo(t *testing.T) { Email: "some-email@domain.com", }, GVLVendorID: 42, - Capabilities: &CapabilitiesInfo{ - App: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeNative}, - }, - Site: &PlatformInfo{ - MediaTypes: []openrtb_ext.BidType{openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeVideo, openrtb_ext.BidTypeNative}, - }, - }, }, }, }, diff --git a/config/test/bidder-info/someBidder.yaml b/config/test/bidder-info/someBidder.yaml index 232b73d0aac..0a1b3059cd4 100644 --- a/config/test/bidder-info/someBidder.yaml +++ b/config/test/bidder-info/someBidder.yaml @@ -11,3 +11,17 @@ capabilities: - banner - video - native +userSync: + key: "foo" + default: "iframe" + iframe: + url: "https://foo.com/sync?mode=iframe&r={{.RedirectURL}}" + redirectUrl: "{{.ExternalURL}}/setuid/iframe" + externalUrl: "https://iframe.host" + userMacro: "%UID" + redirect: + url: "https://foo.com/sync?mode=redirect&r={{.RedirectURL}}" + redirectUrl: "{{.ExternalURL}}/setuid/redirect" + externalUrl: "https://redirect.host" + userMacro: "#UID" + supportCors: true \ No newline at end of file diff --git a/router/router.go b/router/router.go index c8522200ff4..eefffbfce00 100644 --- a/router/router.go +++ b/router/router.go @@ -237,7 +237,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R // apply adapter overrides to bidder info for bidderName, bidderInfo := range bidderInfos { if adapterCfg, exists := cfg.Adapters[bidderName]; exists { - bidderInfo.Syncer = adapterCfg.Syncer.ApplyTo(bidderInfo.Syncer) + bidderInfo.Syncer = adapterCfg.Syncer.Override(bidderInfo.Syncer) } } From fbc0709c8962a85c2fb43d79c365b4f6bfe833c6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sat, 5 Jun 2021 01:33:48 -0400 Subject: [PATCH 16/50] Self Code Review Pass --- analytics/filesystem/file_module_test.go | 3 +-- config/bidderinfo.go | 2 +- config/config.go | 4 +-- endpoints/cookie_sync.go | 32 ++++++++++------------- exchange/exchange.go | 1 - exchange/exchange_test.go | 7 +++-- exchange/utils_test.go | 1 - metrics/config/todo | 4 +-- router/router.go | 33 +++++++++++++++--------- usersync/cookie.go | 3 --- usersync/syncersbuilder.go | 5 +++- 11 files changed, 47 insertions(+), 48 deletions(-) diff --git a/analytics/filesystem/file_module_test.go b/analytics/filesystem/file_module_test.go index 2bd1a58b9b4..5d0cf0172ba 100644 --- a/analytics/filesystem/file_module_test.go +++ b/analytics/filesystem/file_module_test.go @@ -6,10 +6,9 @@ import ( "strings" "testing" - "github.com/prebid/prebid-server/config" - "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/config" ) const TEST_DIR string = "testFiles" diff --git a/config/bidderinfo.go b/config/bidderinfo.go index f7656285575..6749bf95d81 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -64,7 +64,7 @@ type Syncer struct { // endpoint in the Prebid.js project. Redirect *SyncerEndpoint `yaml:"redirect" mapstructure:"redirect"` - // SupportCORS identifies if CORS is supported for user syncing. + // SupportCORS identifies if CORS is supported for the user syncing endpoints. SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` } diff --git a/config/config.go b/config/config.go index a765b9ca95c..1b8ea425f4f 100644 --- a/config/config.go +++ b/config/config.go @@ -31,7 +31,7 @@ type Configuration struct { CacheURL Cache `mapstructure:"cache"` ExtCacheURL ExternalCache `mapstructure:"external_cache"` RecaptchaSecret string `mapstructure:"recaptcha_secret"` - HostCookie HostCookie `mapstructure:"host_cookie"` // to do - move to usersync + HostCookie HostCookie `mapstructure:"host_cookie"` Metrics Metrics `mapstructure:"metrics"` DataCache DataCache `mapstructure:"datacache"` StoredRequests StoredRequests `mapstructure:"stored_requests"` @@ -928,7 +928,7 @@ func setBidderDefaults(v *viper.Viper, bidder string) { v.SetDefault(adapterCfgPrefix+".usersync.redirect.redirect_url", "") v.SetDefault(adapterCfgPrefix+".usersync.redirect.external_url", "") v.SetDefault(adapterCfgPrefix+".usersync.redirect.user_macro", "") - v.SetDefault(adapterCfgPrefix+".usersync.redirect.support_cors", nilBoolean) + //v.SetDefault(adapterCfgPrefix+".usersync.support_cors", nilBoolean) } func isValidCookieSize(maxCookieSize int) error { diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 9517f78bb44..c4390976df9 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -29,12 +29,6 @@ var ( errCookieSyncInvalidBiddersType = errors.New("invalid bidders type. must either be a string '*' or a string array of bidders") ) -var emptyUserSyncRequest = usersync.Request{} -var emptyPrivacyPolicies = privacy.Policies{} -var emptySyncTypeFilter = usersync.SyncTypeFilter{} -var emptySyncBidderFilter = usersync.BidderFilter{} -var emptySyncersChosen = []usersync.SyncerChoice{} - var cookieSyncBidderFilterAllowAll = usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude) func NewCookieSyncEndpoint( @@ -91,7 +85,7 @@ func (c *cookieSyncEndpoint) Handle(w http.ResponseWriter, r *http.Request, _ ht c.handleError(w, errCookieSyncOptOut, http.StatusUnauthorized) case usersync.StatusBlockedByGDPR: c.metrics.RecordCookieSync(metrics.CookieSyncGDPRHostCookieBlocked) - c.handleResponse(w, request.SyncTypeFilter, cookie, privacyPolicies, emptySyncersChosen) + c.handleResponse(w, request.SyncTypeFilter, cookie, privacyPolicies, nil) case usersync.StatusOK: c.metrics.RecordCookieSync(metrics.CookieSyncOK) c.writeBidderMetrics(result.BiddersEvaluated) @@ -103,26 +97,26 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, pr defer r.Body.Close() body, err := ioutil.ReadAll(r.Body) if err != nil { - return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncBody + return usersync.Request{}, privacy.Policies{}, errCookieSyncBody } request := cookieSyncRequest{} if err := json.Unmarshal(body, &request); err != nil { - return emptyUserSyncRequest, emptyPrivacyPolicies, fmt.Errorf("JSON parsing failed: %s", err.Error()) + return usersync.Request{}, privacy.Policies{}, fmt.Errorf("JSON parsing failed: %s", err.Error()) } gdprSignal, err := gdpr.SignalParse(request.GDPR) if err != nil { - return emptyUserSyncRequest, emptyPrivacyPolicies, err + return usersync.Request{}, privacy.Policies{}, err } if request.GDPRConsent == "" { if gdprSignal == gdpr.SignalYes { - return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncGDPRConsentMissing + return usersync.Request{}, privacy.Policies{}, errCookieSyncGDPRConsentMissing } if gdprSignal == gdpr.SignalAmbiguous && gdpr.SignalNormalize(gdprSignal, c.privacyConfig.gdprConfig) == gdpr.SignalYes { - return emptyUserSyncRequest, emptyPrivacyPolicies, errCookieSyncGDPRConsentMissingSignalAmbiguous + return usersync.Request{}, privacy.Policies{}, errCookieSyncGDPRConsentMissingSignalAmbiguous } } @@ -149,7 +143,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, pr syncTypeFilter, err := parseTypeFilter(request.FilterSettings) if err != nil { - return emptyUserSyncRequest, emptyPrivacyPolicies, err + return usersync.Request{}, privacy.Policies{}, err } rx := usersync.Request{ @@ -180,13 +174,13 @@ func parseTypeFilter(request *cookieSyncRequestFilterSettings) (usersync.SyncTyp if filter, err := parseBidderFilter(request.IFrame); err == nil { syncTypeFilter.IFrame = filter } else { - return emptySyncTypeFilter, fmt.Errorf("error parsing filtersettings.iframe: %v", err) + return usersync.SyncTypeFilter{}, fmt.Errorf("error parsing filtersettings.iframe: %v", err) } if filter, err := parseBidderFilter(request.Redirect); err == nil { syncTypeFilter.Redirect = filter } else { - return emptySyncTypeFilter, fmt.Errorf("error parsing filtersettings.image: %v", err) + return usersync.SyncTypeFilter{}, fmt.Errorf("error parsing filtersettings.image: %v", err) } } @@ -205,7 +199,7 @@ func parseBidderFilter(filter *cookieSyncRequestFilter) (usersync.BidderFilter, case "exclude": mode = usersync.BidderFilterModeExclude default: - return emptySyncBidderFilter, fmt.Errorf("invalid filter value '%s'. must be either 'include' or 'exclude'", filter.Mode) + return usersync.BidderFilter{}, fmt.Errorf("invalid filter value '%s'. must be either 'include' or 'exclude'", filter.Mode) } switch v := filter.Bidders.(type) { @@ -213,19 +207,19 @@ func parseBidderFilter(filter *cookieSyncRequestFilter) (usersync.BidderFilter, if v == "*" { return usersync.NewBidderFilterForAll(mode), nil } - return emptySyncBidderFilter, fmt.Errorf("invalid bidders value `%s`. must either be '*' or a string array", v) + return usersync.BidderFilter{}, fmt.Errorf("invalid bidders value `%s`. must either be '*' or a string array", v) case []interface{}: bidders := make([]string, len(v)) for i, x := range v { if bidder, ok := x.(string); ok { bidders[i] = bidder } else { - return emptySyncBidderFilter, errCookieSyncInvalidBiddersType + return usersync.BidderFilter{}, errCookieSyncInvalidBiddersType } } return usersync.NewBidderFilter(bidders, mode), nil default: - return emptySyncBidderFilter, errCookieSyncInvalidBiddersType + return usersync.BidderFilter{}, errCookieSyncInvalidBiddersType } } diff --git a/exchange/exchange.go b/exchange/exchange.go index df2fb905e97..15d84af399b 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -62,7 +62,6 @@ type exchange struct { UsersyncIfAmbiguous bool privacyConfig config.Privacy categoriesFetcher stored_requests.CategoryFetcher - // should add a map of bidder to syncer key, then can eliminate the family names hack } // Container to pass out response ext data from the GetAllBids goroutines back into the main thread diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index b878f5a80d6..dee3b3df142 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -15,12 +15,10 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/errortypes" - "github.com/prebid/prebid-server/usersync" - "github.com/prebid/prebid-server/adapters" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/currency" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/metrics" metricsConf "github.com/prebid/prebid-server/metrics/config" @@ -29,6 +27,7 @@ import ( pbc "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/file_fetcher" + "github.com/prebid/prebid-server/usersync" "github.com/buger/jsonparser" "github.com/mxmCherry/openrtb" @@ -1770,7 +1769,7 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string] return &exchange{ adapterMap: bidderAdapters, - me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames(), []string{}), + me: metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.CoreBidderNames(), nil), cache: &wellBehavedCache{}, cacheTime: 0, gDPR: gdpr.AlwaysFail{}, diff --git a/exchange/utils_test.go b/exchange/utils_test.go index 3d6b2d322d6..2102390fe5a 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -442,7 +442,6 @@ func TestCleanOpenRTBRequests(t *testing.T) { applyCOPPA: false, consentedVendors: map[string]bool{"appnexus": true, "brightroll": true}, }, - // identity with user syncs } privacyConfig := config.Privacy{ diff --git a/metrics/config/todo b/metrics/config/todo index d8445811f30..8908b5f4041 100644 --- a/metrics/config/todo +++ b/metrics/config/todo @@ -9,6 +9,6 @@ todo - override bidder info tests - [open pr] - - migrate all adapter user sync configs + + - remove todo file \ No newline at end of file diff --git a/router/router.go b/router/router.go index eefffbfce00..68b58c9ad80 100644 --- a/router/router.go +++ b/router/router.go @@ -212,8 +212,28 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R legacyBidderList := openrtb_ext.CoreBidderNames() legacyBidderList = append(legacyBidderList, openrtb_ext.BidderName("districtm")) + p, _ := filepath.Abs(infoDirectory) + bidderInfos, err := config.LoadBidderInfoFromDisk(p, cfg.Adapters, openrtb_ext.BuildBidderStringSlice()) + if err != nil { + glog.Fatal(err) + } + + syncers, err := usersync.BuildSyncers(cfg.UserSync, bidderInfos) + if err != nil { + glog.Fatal(err) + } + + syncerKeys := make([]string, 0, len(syncers)) + syncerKeysHashSet := map[string]struct{}{} + for _, syncer := range syncers { + syncerKeysHashSet[syncer.Key()] = struct{}{} + } + for syncerKey := range syncerKeysHashSet { + syncerKeys = append(syncerKeys, syncerKey) + } + // Metrics engine - r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList, nil) // todo: write up syncers + r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList, syncerKeys) db, shutdown, fetcher, ampFetcher, accounts, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) // todo(zachbadgett): better shutdown r.Shutdown = shutdown @@ -228,12 +248,6 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatalf("Failed to create the bidder params validator. %v", err) } - p, _ := filepath.Abs(infoDirectory) - bidderInfos, err := config.LoadBidderInfoFromDisk(p, cfg.Adapters, openrtb_ext.BuildBidderStringSlice()) - if err != nil { - glog.Fatal(err) - } - // apply adapter overrides to bidder info for bidderName, bidderInfo := range bidderInfos { if adapterCfg, exists := cfg.Adapters[bidderName]; exists { @@ -249,11 +263,6 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatal(err) } - syncers, err := usersync.BuildSyncers(cfg.UserSync, bidderInfos) - if err != nil { - glog.Fatal(err) - } - gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() gdprPerms := gdpr.NewPermissions(context.Background(), cfg.GDPR, gvlVendorIDs, generalHttpClient) diff --git a/usersync/cookie.go b/usersync/cookie.go index 6b16c8157d3..5732e6c4c31 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -12,8 +12,6 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) -// todo: add "useSameSite" into the cookie itself, read from the request via user agent headers - const uidCookieName = "uids" // uidTTL is the default amount of time a uid stored within a cookie is considered valid. This is @@ -39,7 +37,6 @@ type uidWithExpiry struct { Expires time.Time `json:"expires"` } -// todo: add samesite if cookie is already same site or detect by browser version // ParseCookieFromRequest parses the UserSyncMap from an HTTP Request. func ParseCookieFromRequest(r *http.Request, cookie *config.HostCookie) *Cookie { if cookie.OptOutCookie.Name != "" { diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index ebb809b4e04..579fdb4bf13 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -2,6 +2,7 @@ package usersync import ( "fmt" + "sort" "strings" "github.com/prebid/prebid-server/config" @@ -75,12 +76,14 @@ func chooseSyncerConfig(biddersSyncerConfig []namedSyncerConfig) (namedSyncerCon } if len(bidderNamesWithEndpoints) == 0 { + sort.Strings(bidderNames) bidders := strings.Join(bidderNames, ", ") return namedSyncerConfig{}, fmt.Errorf("bidders %s share the same syncer key, but none define endpoints (iframe and/or redirect)", bidders) } if len(bidderNamesWithEndpoints) > 1 { - bidders := strings.Join(bidderNames, ", ") + sort.Strings(bidderNamesWithEndpoints) + bidders := strings.Join(bidderNamesWithEndpoints, ", ") return namedSyncerConfig{}, fmt.Errorf("bidders %s define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", bidders) } From 5d6d6bfba4a1f4c98b63e157cabbbb4dad40007b Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sat, 5 Jun 2021 02:34:56 -0400 Subject: [PATCH 17/50] Remove Go 1.16 Specific Type --- endpoints/cookie_sync_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index d51fc29a923..7d411749d5c 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "strings" "testing" - "testing/iotest" "time" "github.com/prebid/prebid-server/analytics" @@ -642,7 +641,7 @@ func TestCookieSyncParseRequest(t *testing.T) { }, { description: "HTTP Read Error", - givenBody: iotest.ErrReader(errors.New("anyError")), + givenBody: ErrReader(errors.New("anyError")), givenGDPRConfig: config.GDPR{Enabled: true, UsersyncIfAmbiguous: true}, givenCCPAEnabled: true, expectedError: "Failed to read request body", @@ -1315,3 +1314,17 @@ func (m *MockGDPRPerms) AuctionActivitiesAllowed(ctx context.Context, bidder ope args := m.Called(ctx, bidder, PublisherID, gdprSignal, consent, weakVendorEnforcement) return args.Bool(0), args.Bool(1), args.Bool(2), args.Error(3) } + +// ErrReader returns an io.Reader that returns 0, err from all Read calls. This is added in +// Go 1.16. Copied here for now until we switch over. +func ErrReader(err error) io.Reader { + return &errReader{err: err} +} + +type errReader struct { + err error +} + +func (r *errReader) Read(p []byte) (int, error) { + return 0, r.err +} From 23052a3d1da9f6e15311590822560d9a6137f80f Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sat, 12 Jun 2021 23:26:14 -0400 Subject: [PATCH 18/50] Don't Create Syncer If Disabled --- config/bidderinfo.go | 2 +- usersync/syncersbuilder.go | 2 +- usersync/syncersbuilder_test.go | 20 ++++++++++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 6749bf95d81..d3511723297 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -65,7 +65,7 @@ type Syncer struct { Redirect *SyncerEndpoint `yaml:"redirect" mapstructure:"redirect"` // SupportCORS identifies if CORS is supported for the user syncing endpoints. - SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` + SupportCORS *bool `yaml:"supportCors" mapstructure:"supportcors"` } func (s *Syncer) Override(v *Syncer) *Syncer { diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index 579fdb4bf13..33c1027f5b8 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -18,7 +18,7 @@ func BuildSyncers(hostConfig config.UserSync, bidderInfos config.BidderInfos) (m // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { - if cfg.Syncer != nil { + if cfg.Enabled && cfg.Syncer != nil { cfgByBidder[bidder] = *cfg.Syncer } } diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index e5deb25000f..838e0bbe4ac 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -13,12 +13,13 @@ func TestBuildSyncers(t *testing.T) { var ( hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"} iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} - infoKeyAPopulated = config.BidderInfo{Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} - infoKeyAEmpty = config.BidderInfo{Syncer: &config.Syncer{Key: "a"}} - infoKeyAError = config.BidderInfo{Syncer: &config.Syncer{Key: "a", Default: "redirect", IFrame: iframeConfig}} // Error caused by invalid default sync type - infoKeyBPopulated = config.BidderInfo{Syncer: &config.Syncer{Key: "b", IFrame: iframeConfig}} - infoKeyBEmpty = config.BidderInfo{Syncer: &config.Syncer{Key: "b"}} - infoKeyMissingPopulated = config.BidderInfo{Syncer: &config.Syncer{IFrame: iframeConfig}} + infoKeyAPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} + infoKeyADisabled = config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} + infoKeyAEmpty = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a"}} + infoKeyAError = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a", Default: "redirect", IFrame: iframeConfig}} // Error caused by invalid default sync type + infoKeyBPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "b", IFrame: iframeConfig}} + infoKeyBEmpty = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "b"}} + infoKeyMissingPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{IFrame: iframeConfig}} ) // NOTE: The hostConfig includes the syncer key in the RedirectURL to distinguish between the syncer keys @@ -92,6 +93,13 @@ func TestBuildSyncers(t *testing.T) { "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", }, }, + { + description: "Many - Disabled Syncers Ignored", + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyADisabled, "bidder2": infoKeyBPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", + }, + }, { description: "Many - Multiple Errors", givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, From 70fb2f8529b0db2c4ef15a8fa594a426cf47d861 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sun, 13 Jun 2021 00:56:44 -0400 Subject: [PATCH 19/50] Finish Config Integration --- config/bidderinfo.go | 77 ++++++++++--------- config/bidderinfo_test.go | 155 ++++++++++++++++++++++++++++++++++++++ config/config.go | 25 +++--- 3 files changed, 207 insertions(+), 50 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index d3511723297..29be6583155 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -65,42 +65,48 @@ type Syncer struct { Redirect *SyncerEndpoint `yaml:"redirect" mapstructure:"redirect"` // SupportCORS identifies if CORS is supported for the user syncing endpoints. - SupportCORS *bool `yaml:"supportCors" mapstructure:"supportcors"` + SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` } -func (s *Syncer) Override(v *Syncer) *Syncer { - if s.Empty() { - return v +func (s *Syncer) Override(original *Syncer) *Syncer { + if s == nil && original == nil { + return nil } - if v == nil { - v = &Syncer{} + var copy Syncer + if original != nil { + copy = *original + } + + if s == nil { + return © } if s.Key != "" { - v.Key = s.Key + copy.Key = s.Key } if s.Default != "" { - v.Key = s.Default + copy.Default = s.Default } - v.IFrame = s.IFrame.Override(v.IFrame) + if original == nil { + copy.IFrame = s.IFrame.Override(nil) + } else { + copy.IFrame = s.IFrame.Override(original.IFrame) + } - v.Redirect = s.IFrame.Override(v.Redirect) + if original == nil { + copy.Redirect = s.Redirect.Override(nil) + } else { + copy.Redirect = s.Redirect.Override(original.Redirect) + } if s.SupportCORS != nil { - v.SupportCORS = s.SupportCORS + copy.SupportCORS = s.SupportCORS } - return v -} - -func (s *Syncer) Empty() bool { - if s == nil { - return true - } - return s.Key == "" && s.Default == "" && s.IFrame.Empty() && s.Redirect.Empty() && s.SupportCORS == nil + return © } // SyncerEndpoint specifies the configuration of the URL returned by the /cookie_sync endpoint @@ -158,39 +164,38 @@ type SyncerEndpoint struct { UserMacro string `yaml:"userMacro" mapstructure:"user_macro"` } -func (s *SyncerEndpoint) Override(v *SyncerEndpoint) *SyncerEndpoint { - if s.Empty() { - return v +// Override returns a new SyncerEndpoint with original values overridden by non empty values. +func (s *SyncerEndpoint) Override(original *SyncerEndpoint) *SyncerEndpoint { + if s == nil && original == nil { + return nil + } + + var copy SyncerEndpoint + if original != nil { + copy = *original } - if v == nil { - v = &SyncerEndpoint{} + if s == nil { + return © } if s.URL != "" { - v.URL = s.URL + copy.URL = s.URL } if s.RedirectURL != "" { - v.RedirectURL = s.RedirectURL + copy.RedirectURL = s.RedirectURL } if s.ExternalURL != "" { - v.ExternalURL = s.ExternalURL + copy.ExternalURL = s.ExternalURL } if s.UserMacro != "" { - v.UserMacro = s.UserMacro + copy.UserMacro = s.UserMacro } - return v -} - -func (s *SyncerEndpoint) Empty() bool { - if s == nil { - return true - } - return s.URL == "" && s.RedirectURL == "" && s.ExternalURL == "" && s.UserMacro == "" + return © } // LoadBidderInfoFromDisk parses all static/bidder-info/{bidder}.yaml files from the file system. diff --git a/config/bidderinfo_test.go b/config/bidderinfo_test.go index a3644b06a9d..4772574b345 100644 --- a/config/bidderinfo_test.go +++ b/config/bidderinfo_test.go @@ -160,6 +160,161 @@ func TestLoadBidderInfo(t *testing.T) { } } +func TestSyncerOverride(t *testing.T) { + var ( + trueValue = true + falseValue = false + ) + + testCases := []struct { + description string + givenOriginal *Syncer + givenOverride *Syncer + expected *Syncer + }{ + { + description: "Nil", + givenOriginal: nil, + givenOverride: nil, + expected: nil, + }, + { + description: "Original Nil", + givenOriginal: nil, + givenOverride: &Syncer{Key: "anyKey"}, + expected: &Syncer{Key: "anyKey"}, + }, + { + description: "Original Empty", + givenOriginal: &Syncer{}, + givenOverride: &Syncer{Key: "anyKey"}, + expected: &Syncer{Key: "anyKey"}, + }, + { + description: "Override Nil", + givenOriginal: &Syncer{Key: "anyKey"}, + givenOverride: nil, + expected: &Syncer{Key: "anyKey"}, + }, + { + description: "Override Empty", + givenOriginal: &Syncer{Key: "anyKey"}, + givenOverride: &Syncer{}, + expected: &Syncer{Key: "anyKey"}, + }, + { + description: "Override Key", + givenOriginal: &Syncer{Key: "original"}, + givenOverride: &Syncer{Key: "override"}, + expected: &Syncer{Key: "override"}, + }, + { + description: "Override Default", + givenOriginal: &Syncer{Default: "original"}, + givenOverride: &Syncer{Default: "override"}, + expected: &Syncer{Default: "override"}, + }, + { + description: "Override IFrame", + givenOriginal: &Syncer{IFrame: &SyncerEndpoint{URL: "original"}}, + givenOverride: &Syncer{IFrame: &SyncerEndpoint{URL: "override"}}, + expected: &Syncer{IFrame: &SyncerEndpoint{URL: "override"}}, + }, + { + description: "Override Redirect", + givenOriginal: &Syncer{Redirect: &SyncerEndpoint{URL: "original"}}, + givenOverride: &Syncer{Redirect: &SyncerEndpoint{URL: "override"}}, + expected: &Syncer{Redirect: &SyncerEndpoint{URL: "override"}}, + }, + { + description: "Override SupportCORS", + givenOriginal: &Syncer{SupportCORS: &trueValue}, + givenOverride: &Syncer{SupportCORS: &falseValue}, + expected: &Syncer{SupportCORS: &falseValue}, + }, + } + + for _, test := range testCases { + result := test.givenOverride.Override(test.givenOriginal) + assert.Equal(t, test.expected, result, test.description) + } +} + +func TestSyncerEndpointOverride(t *testing.T) { + testCases := []struct { + description string + givenOriginal *SyncerEndpoint + givenOverride *SyncerEndpoint + expected *SyncerEndpoint + }{ + { + description: "Nil", + givenOriginal: nil, + givenOverride: nil, + expected: nil, + }, + { + description: "Original Nil", + givenOriginal: nil, + givenOverride: &SyncerEndpoint{URL: "anyURL"}, + expected: &SyncerEndpoint{URL: "anyURL"}, + }, + { + description: "Original Empty", + givenOriginal: &SyncerEndpoint{}, + givenOverride: &SyncerEndpoint{URL: "anyURL"}, + expected: &SyncerEndpoint{URL: "anyURL"}, + }, + { + description: "Override Nil", + givenOriginal: &SyncerEndpoint{URL: "anyURL"}, + givenOverride: nil, + expected: &SyncerEndpoint{URL: "anyURL"}, + }, + { + description: "Override Empty", + givenOriginal: &SyncerEndpoint{URL: "anyURL"}, + givenOverride: &SyncerEndpoint{}, + expected: &SyncerEndpoint{URL: "anyURL"}, + }, + { + description: "Override URL", + givenOriginal: &SyncerEndpoint{URL: "original"}, + givenOverride: &SyncerEndpoint{URL: "override"}, + expected: &SyncerEndpoint{URL: "override"}, + }, + { + description: "Override RedirectURL", + givenOriginal: &SyncerEndpoint{RedirectURL: "original"}, + givenOverride: &SyncerEndpoint{RedirectURL: "override"}, + expected: &SyncerEndpoint{RedirectURL: "override"}, + }, + { + description: "Override ExternalURL", + givenOriginal: &SyncerEndpoint{ExternalURL: "original"}, + givenOverride: &SyncerEndpoint{ExternalURL: "override"}, + expected: &SyncerEndpoint{ExternalURL: "override"}, + }, + { + description: "Override UserMacro", + givenOriginal: &SyncerEndpoint{UserMacro: "original"}, + givenOverride: &SyncerEndpoint{UserMacro: "override"}, + expected: &SyncerEndpoint{UserMacro: "override"}, + }, + { + description: "Override", + givenOriginal: &SyncerEndpoint{URL: "originalURL", RedirectURL: "originalRedirectURL", ExternalURL: "originalExternalURL", UserMacro: "originalUserMacro"}, + givenOverride: &SyncerEndpoint{URL: "overideURL", RedirectURL: "overideRedirectURL", ExternalURL: "overideExternalURL", UserMacro: "overideUserMacro"}, + expected: &SyncerEndpoint{URL: "overideURL", RedirectURL: "overideRedirectURL", ExternalURL: "overideExternalURL", UserMacro: "overideUserMacro"}, + }, + } + + for _, test := range testCases { + result := test.givenOverride.Override(test.givenOriginal) + assert.Equal(t, test.expected, result, test.description) + } +} + type fakeInfoReader struct { content string err error diff --git a/config/config.go b/config/config.go index b1d0bece7d2..b148749c860 100644 --- a/config/config.go +++ b/config/config.go @@ -919,8 +919,6 @@ func migrateConfig(v *viper.Viper) { } } -var nilBoolean *bool - func setBidderDefaults(v *viper.Viper, bidder string) { adapterCfgPrefix := "adapters." + bidder v.SetDefault(adapterCfgPrefix+".endpoint", "") @@ -934,18 +932,17 @@ func setBidderDefaults(v *viper.Viper, bidder string) { v.SetDefault(adapterCfgPrefix+".partner_id", "") v.SetDefault(adapterCfgPrefix+".extra_info", "") - // user sync - v.SetDefault(adapterCfgPrefix+".usersync.key", "") - v.SetDefault(adapterCfgPrefix+".usersync.default", "") - v.SetDefault(adapterCfgPrefix+".usersync.iframe.url", "") - v.SetDefault(adapterCfgPrefix+".usersync.iframe.redirect_url", "") - v.SetDefault(adapterCfgPrefix+".usersync.iframe.external_url", "") - v.SetDefault(adapterCfgPrefix+".usersync.iframe.user_macro", "") - v.SetDefault(adapterCfgPrefix+".usersync.redirect.url", "") - v.SetDefault(adapterCfgPrefix+".usersync.redirect.redirect_url", "") - v.SetDefault(adapterCfgPrefix+".usersync.redirect.external_url", "") - v.SetDefault(adapterCfgPrefix+".usersync.redirect.user_macro", "") - //v.SetDefault(adapterCfgPrefix+".usersync.support_cors", nilBoolean) + v.BindEnv(adapterCfgPrefix + ".usersync.key") + v.BindEnv(adapterCfgPrefix + ".usersync.default") + v.BindEnv(adapterCfgPrefix + ".usersync.iframe.url") + v.BindEnv(adapterCfgPrefix + ".usersync.iframe.redirect_url") + v.BindEnv(adapterCfgPrefix + ".usersync.iframe.external_url") + v.BindEnv(adapterCfgPrefix + ".usersync.iframe.user_macro") + v.BindEnv(adapterCfgPrefix + ".usersync.redirect.url") + v.BindEnv(adapterCfgPrefix + ".usersync.redirect.redirect_url") + v.BindEnv(adapterCfgPrefix + ".usersync.redirect.external_url") + v.BindEnv(adapterCfgPrefix + ".usersync.redirect.user_macro") + v.BindEnv(adapterCfgPrefix + ".usersync.support_cors") } func isValidCookieSize(maxCookieSize int) error { From bc31e6ca3ede7ac8acf88a8cfa10615a11419e16 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sun, 13 Jun 2021 01:14:48 -0400 Subject: [PATCH 20/50] Fix Merge Conflicts --- adapters/algorix/algorix.go | 4 ++-- adapters/bmtm/usersync.go | 12 ---------- adapters/bmtm/usersync_test.go | 29 ------------------------ adapters/e_volution/usersync.go | 12 ---------- adapters/e_volution/usersync_test.go | 33 ---------------------------- endpoints/cookie_sync_test.go | 16 -------------- 6 files changed, 2 insertions(+), 104 deletions(-) delete mode 100644 adapters/bmtm/usersync.go delete mode 100644 adapters/bmtm/usersync_test.go delete mode 100644 adapters/e_volution/usersync.go delete mode 100644 adapters/e_volution/usersync_test.go diff --git a/adapters/algorix/algorix.go b/adapters/algorix/algorix.go index 7c0a6787c7b..31d013490cf 100644 --- a/adapters/algorix/algorix.go +++ b/adapters/algorix/algorix.go @@ -16,7 +16,7 @@ import ( ) type adapter struct { - EndpointTemplate template.Template + EndpointTemplate *template.Template } // Builder builds a new instance of the AlgoriX adapter for the given bidder with the given config. @@ -26,7 +26,7 @@ func Builder(bidderName openrtb_ext.BidderName, config config.Adapter) (adapters return nil, fmt.Errorf("unable to parse endpoint url template: %v", err) } bidder := &adapter{ - EndpointTemplate: *endpoint, + EndpointTemplate: endpoint, } return bidder, nil } diff --git a/adapters/bmtm/usersync.go b/adapters/bmtm/usersync.go deleted file mode 100644 index e89dec3b3a7..00000000000 --- a/adapters/bmtm/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package bmtm - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewBmtmSyncer(template *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("bmtm", template, adapters.SyncTypeRedirect) -} diff --git a/adapters/bmtm/usersync_test.go b/adapters/bmtm/usersync_test.go deleted file mode 100644 index e22ec822d63..00000000000 --- a/adapters/bmtm/usersync_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package bmtm - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSyncer(t *testing.T) { - syncURL := "https://synctest/?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&url=localhost%2Fsetuid%3Fbidder%3Dbmtm%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3DUUID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewBmtmSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "consent-string", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://synctest/?gdpr=1&gdpr_consent=consent-string&url=localhost%2Fsetuid%3Fbidder%3Dbmtm%26gdpr%3D1%26gdpr_consent%3Dconsent-string%26uid%3DUUID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) -} diff --git a/adapters/e_volution/usersync.go b/adapters/e_volution/usersync.go deleted file mode 100644 index f22784d018b..00000000000 --- a/adapters/e_volution/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package evolution - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewEvolutionSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("e_volution", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/e_volution/usersync_test.go b/adapters/e_volution/usersync_test.go deleted file mode 100644 index d7a3eba5f0a..00000000000 --- a/adapters/e_volution/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package evolution - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestNewEvolutionSyncer(t *testing.T) { - syncURL := "https://sync.test.com/pbserver?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewEvolutionSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "allGdpr", - }, - CCPA: ccpa.Policy{ - Consent: "1---", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://sync.test.com/pbserver?gdpr=0&gdpr_consent=allGdpr&ccpa=1---", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 557ae93c35c..e5d9785b085 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -10,11 +10,7 @@ import ( "testing" "time" - "github.com/prebid/prebid-server/adapters/appnexus" - "github.com/prebid/prebid-server/adapters/audienceNetwork" - "github.com/prebid/prebid-server/adapters/pubmatic" "github.com/prebid/prebid-server/analytics" - analyticsConf "github.com/prebid/prebid-server/analytics/config" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/metrics" @@ -24,8 +20,6 @@ import ( gdprPrivacy "github.com/prebid/prebid-server/privacy/gdpr" "github.com/prebid/prebid-server/usersync" - "github.com/buger/jsonparser" - "github.com/julienschmidt/httprouter" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -855,16 +849,6 @@ func TestCookieSyncHandleError(t *testing.T) { }) } -func TestGDPRPreventsBidders(t *testing.T) { - rr := doPost(`{"gdpr":1,"bidders":["appnexus", "pubmatic"],"gdpr_consent":"BOONs2HOONs2HABABBENAGgAAAAPrABACGA"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{ - openrtb_ext.BidderPubmatic: pubmatic.NewPubmaticSyncer(template.Must(template.New("sync").Parse("someurl.com"))), - }) - assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8") - assert.Equal(t, http.StatusOK, rr.Code) - assert.ElementsMatch(t, []string{"pubmatic"}, parseSyncs(t, rr.Body.Bytes())) - assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes())) -} - func TestCookieSyncWriteBidderMetrics(t *testing.T) { testCases := []struct { description string From 68a27109fe0d8c4f50f8cf3864454d596ef39dfc Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Sun, 13 Jun 2021 01:54:23 -0400 Subject: [PATCH 21/50] Implement Legacy UserSyncURL Overrides --- main.go | 2 +- router/router.go | 55 ++++++++++++++++++++++++++++++++--------- router/router_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index ebeabf29df8..ccee83f8444 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,7 @@ func main() { err = serve(Rev, cfg) if err != nil { - glog.Errorf("prebid-server failed: %v", err) + glog.Fatalf("prebid-server failed: %v", err) } } diff --git a/router/router.go b/router/router.go index 5cd34e600a8..a8e8d7d94e5 100644 --- a/router/router.go +++ b/router/router.go @@ -213,12 +213,16 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R p, _ := filepath.Abs(infoDirectory) bidderInfos, err := config.LoadBidderInfoFromDisk(p, cfg.Adapters, openrtb_ext.BuildBidderStringSlice()) if err != nil { - glog.Fatal(err) + return nil, err + } + + if err := applyBidderInfoConfigOverrides(bidderInfos, cfg.Adapters); err != nil { + return nil, err } syncers, err := usersync.BuildSyncers(cfg.UserSync, bidderInfos) if err != nil { - glog.Fatal(err) + return nil, err } syncerKeys := make([]string, 0, len(syncers)) @@ -246,19 +250,12 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R glog.Fatalf("Failed to create the bidder params validator. %v", err) } - // apply adapter overrides to bidder info - for bidderName, bidderInfo := range bidderInfos { - if adapterCfg, exists := cfg.Adapters[bidderName]; exists { - bidderInfo.Syncer = adapterCfg.Syncer.Override(bidderInfo.Syncer) - } - } - activeBidders := exchange.GetActiveBidders(bidderInfos) disabledBidders := exchange.GetDisabledBiddersErrorMessages(bidderInfos) defaultAliases, defReqJSON := readDefaultRequest(cfg.DefReqConfig) if err := validateDefaultAliases(defaultAliases); err != nil { - glog.Fatal(err) + return nil, err } gvlVendorIDs := bidderInfos.ToGVLVendorIDMap() @@ -270,7 +267,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R adapters, adaptersErrs := exchange.BuildAdapters(generalHttpClient, cfg, bidderInfos, r.MetricsEngine) if len(adaptersErrs) > 0 { errs := errortypes.NewAggregateError("Failed to initialize adapters", adaptersErrs) - glog.Fatalf("%v", errs) + return nil, errs } theExchange := exchange.NewExchange(adapters, cacheClient, cfg, syncers, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor, categoriesFetcher) @@ -333,6 +330,42 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return r, nil } +func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg map[string]config.Adapter) error { + for bidderName, bidderInfo := range bidderInfos { + if adapterCfg, exists := adaptersCfg[bidderName]; exists { + bidderInfo.Syncer = adapterCfg.Syncer.Override(bidderInfo.Syncer) + + // comptability with legacy settings (if possible unambiguously) + if adapterCfg.UserSyncURL != "" { + if bidderInfo.Syncer == nil { + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync", bidderName) + } + + endpointsCount := 0 + if bidderInfo.Syncer.IFrame != nil { + bidderInfo.Syncer.IFrame.URL = adapterCfg.UserSyncURL + endpointsCount++ + } + if bidderInfo.Syncer.Redirect != nil { + bidderInfo.Syncer.Redirect.URL = adapterCfg.UserSyncURL + endpointsCount++ + } + + if endpointsCount == 0 { + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define user sync endpoints", bidderName) + } + + if endpointsCount > 1 { + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder defines multiple user sync endpoints", bidderName) + } + } + + bidderInfos[bidderName] = bidderInfo + } + } + return nil +} + // Fixes #648 // // These CORS options pose a security risk... but it's a calculated one. diff --git a/router/router_test.go b/router/router_test.go index 65fe299e309..5865efddf41 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -74,6 +74,63 @@ func TestExchangeMap(t *testing.T) { } } +func TestApplyBidderInfoConfigOverrides(t *testing.T) { + var testCases = []struct { + description string + givenBidderInfos config.BidderInfos + givenAdaptersCfg map[string]config.Adapter + expectedError string + expectedBidderInfos config.BidderInfos + }{ + { + description: "Syncer Override", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Key: "original"}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {Syncer: &config.Syncer{Key: "override"}}}, + expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Key: "override"}}}, + }, + { + description: "UserSyncURL Override IFrame", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "original"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "override"}}}}, + }, + { + description: "UserSyncURL Override Redirect", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Redirect: &config.SyncerEndpoint{URL: "original"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Redirect: &config.SyncerEndpoint{URL: "override"}}}}, + }, + { + description: "UserSyncURL Override Syncer Not Defined", + givenBidderInfos: config.BidderInfos{"a": {}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync", + }, + { + description: "UserSyncURL Override Syncer Endpoints Not Defined", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define user sync endpoints", + }, + { + description: "UserSyncURL Override Ambiguous", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "originalIFrame"}, Redirect: &config.SyncerEndpoint{URL: "originalRedirect"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder defines multiple user sync endpoints", + }, + } + + for _, test := range testCases { + resultErr := applyBidderInfoConfigOverrides(test.givenBidderInfos, test.givenAdaptersCfg) + if test.expectedError == "" { + assert.NoError(t, resultErr, test.description+":err") + assert.Equal(t, test.expectedBidderInfos, test.givenBidderInfos, test.description+":result") + } else { + assert.EqualError(t, resultErr, test.expectedError, test.description+":err") + } + } +} + // Prevents #648 func TestCORSSupport(t *testing.T) { const origin = "https://publisher-domain.com" From 02cc6a7701754bf0945193092385739b55342c0c Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 14 Jun 2021 16:57:42 -0400 Subject: [PATCH 22/50] Set UserSync Endpoints --- static/bidder-info/33across.yaml | 4 ++++ static/bidder-info/acuityads.yaml | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/static/bidder-info/33across.yaml b/static/bidder-info/33across.yaml index 044cfe140b2..1ee5366f969 100644 --- a/static/bidder-info/33across.yaml +++ b/static/bidder-info/33across.yaml @@ -6,3 +6,7 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + url: "https://ic.tynt.com/r/d?m=xch&rt=html&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&ru={{.RedirectURL}}&id=zzz000000000002zzz" + userMacro: "33XUSERID33X" \ No newline at end of file diff --git a/static/bidder-info/acuityads.yaml b/static/bidder-info/acuityads.yaml index 9539e36b91e..c806294d644 100644 --- a/static/bidder-info/acuityads.yaml +++ b/static/bidder-info/acuityads.yaml @@ -12,4 +12,7 @@ capabilities: - banner - video - native - +userSync: + redirect: + url: "https://cs.admanmedia.com/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" \ No newline at end of file From 6f4950b5716198435415938cdc98dafd1d50e200 Mon Sep 17 00:00:00 2001 From: Veronika Solovei Date: Tue, 15 Jun 2021 22:40:04 -0700 Subject: [PATCH 23/50] Bidder user sync configs --- static/bidder-info/adf.yaml | 4 ++++ static/bidder-info/adform.yaml | 4 ++++ static/bidder-info/adkernel.yaml | 4 ++++ static/bidder-info/adkernelAdn.yaml | 4 ++++ static/bidder-info/adman.yaml | 6 +++++- static/bidder-info/admixer.yaml | 6 +++++- static/bidder-info/adpone.yaml | 4 ++++ static/bidder-info/advangelists.yaml | 6 +++++- static/bidder-info/adyoulike.yaml | 4 ++++ static/bidder-info/aja.yaml | 4 ++++ static/bidder-info/amx.yaml | 6 +++++- static/bidder-info/appnexus.yaml | 5 +++++ static/bidder-info/avocet.yaml | 4 ++++ static/bidder-info/beachfront.yaml | 4 ++++ static/bidder-info/beintoo.yaml | 5 +++++ static/bidder-info/between.yaml | 6 +++++- static/bidder-info/brightroll.yaml | 4 ++++ static/bidder-info/colossus.yaml | 4 ++++ static/bidder-info/connectad.yaml | 4 ++++ static/bidder-info/consumable.yaml | 4 ++++ static/bidder-info/conversant.yaml | 4 ++++ static/bidder-info/cpmstar.yaml | 4 ++++ static/bidder-info/datablocks.yaml | 4 ++++ static/bidder-info/deepintent.yaml | 4 ++++ static/bidder-info/e_volution.yaml | 6 +++++- static/bidder-info/emx_digital.yaml | 6 +++++- static/bidder-info/engagebdr.yaml | 5 +++++ static/bidder-info/eplanning.yaml | 4 ++++ static/bidder-info/gamoshi.yaml | 4 ++++ static/bidder-info/grid.yaml | 6 +++++- static/bidder-info/gumgum.yaml | 6 +++++- static/bidder-info/improvedigital.yaml | 4 ++++ static/bidder-info/ix.yaml | 4 ++++ static/bidder-info/jixie.yaml | 4 ++++ static/bidder-info/krushmedia.yaml | 6 +++++- static/bidder-info/lockerdome.yaml | 4 ++++ static/bidder-info/logicad.yaml | 4 ++++ static/bidder-info/lunamedia.yaml | 6 +++++- static/bidder-info/marsmedia.yaml | 4 ++++ static/bidder-info/mgid.yaml | 4 ++++ static/bidder-info/nanointeractive.yaml | 4 ++++ static/bidder-info/ninthdecimal.yaml | 6 +++++- static/bidder-info/nobid.yaml | 4 ++++ static/bidder-info/onetag.yaml | 6 +++++- static/bidder-info/openx.yaml | 4 ++++ static/bidder-info/outbrain.yaml | 4 ++++ static/bidder-info/pubmatic.yaml | 4 ++++ static/bidder-info/pulsepoint.yaml | 4 ++++ static/bidder-info/rhythmone.yaml | 4 ++++ static/bidder-info/sharethrough.yaml | 4 ++++ static/bidder-info/smartadserver.yaml | 4 ++++ static/bidder-info/smartrtb.yaml | 4 ++++ static/bidder-info/smartyads.yaml | 4 ++++ static/bidder-info/somoaudience.yaml | 4 ++++ static/bidder-info/sonobi.yaml | 4 ++++ static/bidder-info/sovrn.yaml | 4 ++++ static/bidder-info/synacormedia.yaml | 4 ++++ static/bidder-info/tappx.yaml | 6 +++++- static/bidder-info/telaria.yaml | 4 ++++ static/bidder-info/triplelift.yaml | 4 ++++ static/bidder-info/triplelift_native.yaml | 5 +++++ static/bidder-info/trustx.yaml | 4 ++++ static/bidder-info/ucfunnel.yaml | 4 ++++ static/bidder-info/unruly.yaml | 4 ++++ static/bidder-info/valueimpression.yaml | 4 ++++ static/bidder-info/visx.yaml | 4 ++++ static/bidder-info/yieldlab.yaml | 4 ++++ static/bidder-info/yieldmo.yaml | 4 ++++ static/bidder-info/yieldone.yaml | 4 ++++ static/bidder-info/zeroclickfraud.yaml | 4 ++++ 70 files changed, 298 insertions(+), 14 deletions(-) diff --git a/static/bidder-info/adf.yaml b/static/bidder-info/adf.yaml index 776e208f562..2e312bff788 100644 --- a/static/bidder-info/adf.yaml +++ b/static/bidder-info/adf.yaml @@ -12,3 +12,7 @@ capabilities: - banner - native - video +userSync: + redirect: + url: "https://cm.adform.net/cookie?redirect_url={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/adform.yaml b/static/bidder-info/adform.yaml index 461714ac44d..c8c1d91565a 100644 --- a/static/bidder-info/adform.yaml +++ b/static/bidder-info/adform.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://cm.adform.net/cookie?redirect_url={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/adkernel.yaml b/static/bidder-info/adkernel.yaml index a78b3cde498..8d9096b271c 100644 --- a/static/bidder-info/adkernel.yaml +++ b/static/bidder-info/adkernel.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://sync.adkernel.com/user-sync?t=image&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "{UID}" \ No newline at end of file diff --git a/static/bidder-info/adkernelAdn.yaml b/static/bidder-info/adkernelAdn.yaml index 1f54f8e5a8f..e401cb83923 100644 --- a/static/bidder-info/adkernelAdn.yaml +++ b/static/bidder-info/adkernelAdn.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "{UID}" diff --git a/static/bidder-info/adman.yaml b/static/bidder-info/adman.yaml index 2db7c07584c..d7869fdc536 100644 --- a/static/bidder-info/adman.yaml +++ b/static/bidder-info/adman.yaml @@ -9,4 +9,8 @@ capabilities: site: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + redirect: + url: "https://sync.admanmedia.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/admixer.yaml b/static/bidder-info/admixer.yaml index 9faf0eb3a3e..5cb07ef5579 100644 --- a/static/bidder-info/admixer.yaml +++ b/static/bidder-info/admixer.yaml @@ -13,4 +13,8 @@ capabilities: - banner - video - native - - audio \ No newline at end of file + - audio +userSync: + redirect: + url: "https://inv-nets.admixer.net/adxcm.aspx?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir=1&rurl={{.RedirectURL}}" + userMacro: "$$visitor_cookie$$" diff --git a/static/bidder-info/adpone.yaml b/static/bidder-info/adpone.yaml index 7c5b473770b..c5fb83fb39c 100644 --- a/static/bidder-info/adpone.yaml +++ b/static/bidder-info/adpone.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://usersync.adpone.com/csync?redir={{.RedirectURL}}" + userMacro: "{uid}" \ No newline at end of file diff --git a/static/bidder-info/advangelists.yaml b/static/bidder-info/advangelists.yaml index aed9900d0e7..35495d4d97f 100644 --- a/static/bidder-info/advangelists.yaml +++ b/static/bidder-info/advangelists.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video - +userSync: + iframe: + url: "https://nep.advangelists.com/xp/user-sync?acctid={aid}&&redirect={{.RedirectURL}}" + userMacro: "$UID" + diff --git a/static/bidder-info/adyoulike.yaml b/static/bidder-info/adyoulike.yaml index d9b23d6c1a1..a8769ddd854 100644 --- a/static/bidder-info/adyoulike.yaml +++ b/static/bidder-info/adyoulike.yaml @@ -8,3 +8,7 @@ capabilities: - banner - video - native +userSync: + redirect: + url: "http://visitor.omnitagjs.com/visitor/bsync?uid=19340f4f097d16f41f34fc0274981ca4&name=PrebidServer&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url={{.RedirectURL}}" + userMacro: "[BUYER_USERID]" diff --git a/static/bidder-info/aja.yaml b/static/bidder-info/aja.yaml index 53f43689172..1d18dd7e75d 100644 --- a/static/bidder-info/aja.yaml +++ b/static/bidder-info/aja.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ad.as.amanad.adtdp.com/v1/sync/ssp?ssp=4&gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "%s" diff --git a/static/bidder-info/amx.yaml b/static/bidder-info/amx.yaml index f9fdfbb4a41..43c33c43ce0 100644 --- a/static/bidder-info/amx.yaml +++ b/static/bidder-info/amx.yaml @@ -9,4 +9,8 @@ capabilities: app: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + redirect: + url: "https://prebid.a-mo.net/cchain/0?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/appnexus.yaml b/static/bidder-info/appnexus.yaml index f2f4a1266df..dd7949f521b 100644 --- a/static/bidder-info/appnexus.yaml +++ b/static/bidder-info/appnexus.yaml @@ -12,3 +12,8 @@ capabilities: - banner - video - native +userSync: + key: "adnxs" + redirect: + url: "https://ib.adnxs.com/getuid?{{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/avocet.yaml b/static/bidder-info/avocet.yaml index 42147c96ebf..ad8ca157a75 100644 --- a/static/bidder-info/avocet.yaml +++ b/static/bidder-info/avocet.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ads.avct.cloud/getuid?&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url={{.RedirectURL}}" + userMacro: "{{UUID}}" diff --git a/static/bidder-info/beachfront.yaml b/static/bidder-info/beachfront.yaml index 06991698090..64e04dc6eec 100644 --- a/static/bidder-info/beachfront.yaml +++ b/static/bidder-info/beachfront.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + url: "https://sync.bfmio.com/sync_s2s?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&url={{.RedirectURL}}" + userMacro: "[io_cid]" diff --git a/static/bidder-info/beintoo.yaml b/static/bidder-info/beintoo.yaml index 905d89a44c4..d804a29818d 100644 --- a/static/bidder-info/beintoo.yaml +++ b/static/bidder-info/beintoo.yaml @@ -5,3 +5,8 @@ capabilities: site: mediaTypes: - banner +userSync: + key: "Beintoo" + iframe: + url: "https://ib.beintoo.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/between.yaml b/static/bidder-info/between.yaml index 71bd8ba6256..5649c328d62 100644 --- a/static/bidder-info/between.yaml +++ b/static/bidder-info/between.yaml @@ -7,4 +7,8 @@ capabilities: - banner app: mediaTypes: - - banner \ No newline at end of file + - banner +userSync: + redirect: + url: "https://ads.betweendigital.com/match?bidder_id=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&callback_url={{.RedirectURL}}" + userMacro: "${USER_ID}" diff --git a/static/bidder-info/brightroll.yaml b/static/bidder-info/brightroll.yaml index a2ea0c74b77..db822bc4f7b 100644 --- a/static/bidder-info/brightroll.yaml +++ b/static/bidder-info/brightroll.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://pr-bh.ybp.yahoo.com/sync/appnexusprebidserver/?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/colossus.yaml b/static/bidder-info/colossus.yaml index 901c824c603..11189c62cf3 100644 --- a/static/bidder-info/colossus.yaml +++ b/static/bidder-info/colossus.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://sync.colossusssp.com/pbs.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" \ No newline at end of file diff --git a/static/bidder-info/connectad.yaml b/static/bidder-info/connectad.yaml index fa0ae4520fc..116ab79100e 100644 --- a/static/bidder-info/connectad.yaml +++ b/static/bidder-info/connectad.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + iframe: + url: "https://cdn.connectad.io/connectmyusers.php?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/consumable.yaml b/static/bidder-info/consumable.yaml index e9b5f72623c..17df0c15b0d 100644 --- a/static/bidder-info/consumable.yaml +++ b/static/bidder-info/consumable.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://e.serverbid.com/udb/9969/match?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/conversant.yaml b/static/bidder-info/conversant.yaml index aa3d3822802..cd55f01fef5 100644 --- a/static/bidder-info/conversant.yaml +++ b/static/bidder-info/conversant.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://prebid-match.dotomi.com/match/bounce/current?version=1&networkId=72582&rurl={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/cpmstar.yaml b/static/bidder-info/cpmstar.yaml index 097dfddd5b0..46a7451bd1c 100644 --- a/static/bidder-info/cpmstar.yaml +++ b/static/bidder-info/cpmstar.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://server.cpmstar.com/usersync.aspx?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" \ No newline at end of file diff --git a/static/bidder-info/datablocks.yaml b/static/bidder-info/datablocks.yaml index 43f00a63eae..415dbc3546e 100644 --- a/static/bidder-info/datablocks.yaml +++ b/static/bidder-info/datablocks.yaml @@ -11,3 +11,7 @@ capabilities: - banner - native - video +userSync: + redirect: + url: "https://sync.v5prebid.datablocks.net/s2ssync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "${uid}" diff --git a/static/bidder-info/deepintent.yaml b/static/bidder-info/deepintent.yaml index a8f17a55d6f..10ef458f133 100644 --- a/static/bidder-info/deepintent.yaml +++ b/static/bidder-info/deepintent.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://match.deepintent.com/usersync/136?id=unk&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/e_volution.yaml b/static/bidder-info/e_volution.yaml index 6ea9dc7bac2..ddfef3dff36 100644 --- a/static/bidder-info/e_volution.yaml +++ b/static/bidder-info/e_volution.yaml @@ -11,4 +11,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://sync.e-volution.ai/pbserver?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/emx_digital.yaml b/static/bidder-info/emx_digital.yaml index ec0d090fb4c..a4cbd095934 100644 --- a/static/bidder-info/emx_digital.yaml +++ b/static/bidder-info/emx_digital.yaml @@ -9,4 +9,8 @@ capabilities: app: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + iframe: + url: "https://cs.emxdgt.com/um?ssp=pbs&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/engagebdr.yaml b/static/bidder-info/engagebdr.yaml index fd08367acc7..f79bca69953 100644 --- a/static/bidder-info/engagebdr.yaml +++ b/static/bidder-info/engagebdr.yaml @@ -7,3 +7,8 @@ capabilities: - banner - video - native +userSync: + iframe: + url: "https://match.bnmla.com/usersync/s2s_sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "${UUID}" + diff --git a/static/bidder-info/eplanning.yaml b/static/bidder-info/eplanning.yaml index ab0b7609dbb..960244446aa 100644 --- a/static/bidder-info/eplanning.yaml +++ b/static/bidder-info/eplanning.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + iframe: + url: "https://ads.us.e-planning.net/uspd/1/?du={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/gamoshi.yaml b/static/bidder-info/gamoshi.yaml index 0cfd495762f..c72d1770082 100644 --- a/static/bidder-info/gamoshi.yaml +++ b/static/bidder-info/gamoshi.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://rtb.gamoshi.io/user_sync_prebid?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl={{.RedirectURL}}" + userMacro: "[gusr]" diff --git a/static/bidder-info/grid.yaml b/static/bidder-info/grid.yaml index 31c7b7320e3..b6c13bc7000 100644 --- a/static/bidder-info/grid.yaml +++ b/static/bidder-info/grid.yaml @@ -9,4 +9,8 @@ capabilities: site: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + redirect: + url: "https://x.bidswitch.net/check_uuid/{{.RedirectURL}}?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}" + userMacro: "${BSW_UUID}" diff --git a/static/bidder-info/gumgum.yaml b/static/bidder-info/gumgum.yaml index bfefe63ab40..81f42d31877 100644 --- a/static/bidder-info/gumgum.yaml +++ b/static/bidder-info/gumgum.yaml @@ -5,4 +5,8 @@ capabilities: site: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + iframe: + url: "https://rtb.gumgum.com/usync/prbds2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/improvedigital.yaml b/static/bidder-info/improvedigital.yaml index f7fea4a8402..649246abf09 100644 --- a/static/bidder-info/improvedigital.yaml +++ b/static/bidder-info/improvedigital.yaml @@ -12,3 +12,7 @@ capabilities: - banner - video - native +userSync: + redirect: + url: "https://ad.360yield.com/server_match?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "{PUB_USER_ID}" diff --git a/static/bidder-info/ix.yaml b/static/bidder-info/ix.yaml index 1e89c72e5bb..99dd5c5954f 100644 --- a/static/bidder-info/ix.yaml +++ b/static/bidder-info/ix.yaml @@ -14,3 +14,7 @@ capabilities: - video - native - audio +userSync: + redirect: + url: "https://ssum.casalemedia.com/usermatchredir?s=194962&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/jixie.yaml b/static/bidder-info/jixie.yaml index ac38f313da1..0553f1f7393 100644 --- a/static/bidder-info/jixie.yaml +++ b/static/bidder-info/jixie.yaml @@ -6,3 +6,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://id.jixie.io/api/sync?pid=&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "%%JXUID%%" diff --git a/static/bidder-info/krushmedia.yaml b/static/bidder-info/krushmedia.yaml index 342e11df2c7..82055a29988 100644 --- a/static/bidder-info/krushmedia.yaml +++ b/static/bidder-info/krushmedia.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://cs.krushmedia.com/4e4abdd5ecc661643458a730b1aa927d.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/lockerdome.yaml b/static/bidder-info/lockerdome.yaml index b005e7c1f85..550b4a80b38 100644 --- a/static/bidder-info/lockerdome.yaml +++ b/static/bidder-info/lockerdome.yaml @@ -7,3 +7,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://lockerdome.com/usync/prebidserver?pid="+cfg.Adapters["lockerdome"].PlatformID+"&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "{{uid}}" diff --git a/static/bidder-info/logicad.yaml b/static/bidder-info/logicad.yaml index c087516c061..4748cc75f09 100644 --- a/static/bidder-info/logicad.yaml +++ b/static/bidder-info/logicad.yaml @@ -7,4 +7,8 @@ capabilities: app: mediaTypes: - banner +userSync: + redirect: + url: "https://cr-p31.ladsp.jp/cookiesender/31?r=true&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ru={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/lunamedia.yaml b/static/bidder-info/lunamedia.yaml index 4cabdc4a381..216fae9becf 100644 --- a/static/bidder-info/lunamedia.yaml +++ b/static/bidder-info/lunamedia.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video - +userSync: + iframe: + url: "https://api.lunamedia.io/xp/user-sync?redirect={{.RedirectURL}}" + userMacro: "$UID" + diff --git a/static/bidder-info/marsmedia.yaml b/static/bidder-info/marsmedia.yaml index 143b893ed9b..3080962d37c 100644 --- a/static/bidder-info/marsmedia.yaml +++ b/static/bidder-info/marsmedia.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://dmp.rtbsrv.com/dmp/profiles/cm?p_id=179&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "${UUID}" diff --git a/static/bidder-info/mgid.yaml b/static/bidder-info/mgid.yaml index bddb8b8598e..a9e20d631c0 100644 --- a/static/bidder-info/mgid.yaml +++ b/static/bidder-info/mgid.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - native +userSync: + redirect: + url: "https://cm.mgid.com/m?cdsp=363893&adu={{.RedirectURL}}" + userMacro: "{muidn}" diff --git a/static/bidder-info/nanointeractive.yaml b/static/bidder-info/nanointeractive.yaml index d199e9e8ff5..3b4db38dff8 100644 --- a/static/bidder-info/nanointeractive.yaml +++ b/static/bidder-info/nanointeractive.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://ad.audiencemanager.de/hbs/cookie_sync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/ninthdecimal.yaml b/static/bidder-info/ninthdecimal.yaml index eda7d222a5f..db1392d78c8 100755 --- a/static/bidder-info/ninthdecimal.yaml +++ b/static/bidder-info/ninthdecimal.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video - +userSync: + iframe: + url: "https://rtb.ninthdecimal.com/xp/user-sync?acctid={aid}&&redirect={{.RedirectURL}}" + userMacro: "$UID" + diff --git a/static/bidder-info/nobid.yaml b/static/bidder-info/nobid.yaml index 89f2a28abcd..6fa65fe93bc 100644 --- a/static/bidder-info/nobid.yaml +++ b/static/bidder-info/nobid.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ads.servenobid.com/getsync?tek=pbs&ver=1&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/onetag.yaml b/static/bidder-info/onetag.yaml index 697ef04d5a1..b0aeeafe347 100644 --- a/static/bidder-info/onetag.yaml +++ b/static/bidder-info/onetag.yaml @@ -11,4 +11,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + iframe: + url: "https://onetag-sys.com/usync/?redir={{.RedirectURL}}" + userMacro: "${USER_TOKEN}" diff --git a/static/bidder-info/openx.yaml b/static/bidder-info/openx.yaml index 709f3db0147..c48d7de2933 100644 --- a/static/bidder-info/openx.yaml +++ b/static/bidder-info/openx.yaml @@ -11,3 +11,7 @@ capabilities: - banner - video modifyingVastXmlAllowed: true +userSync: + redirect: + url: "https://rtb.openx.net/sync/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r={{.RedirectURL}}" + userMacro: "${UID}" diff --git a/static/bidder-info/outbrain.yaml b/static/bidder-info/outbrain.yaml index e38ec915f49..d24c603a6dc 100644 --- a/static/bidder-info/outbrain.yaml +++ b/static/bidder-info/outbrain.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - native +userSync: + redirect: + url: "https://prebidtest.zemanta.com/usersync/prebidtest?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" + userMacro: "__ZUID__" diff --git a/static/bidder-info/pubmatic.yaml b/static/bidder-info/pubmatic.yaml index 45c0418af8a..695af9dfd81 100644 --- a/static/bidder-info/pubmatic.yaml +++ b/static/bidder-info/pubmatic.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + url: "https://ads.pubmatic.com/AdServer/js/user_sync.html?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&predirect={{.RedirectURL}}" + userMacro: "" diff --git a/static/bidder-info/pulsepoint.yaml b/static/bidder-info/pulsepoint.yaml index bda03efd99c..635ceb46006 100644 --- a/static/bidder-info/pulsepoint.yaml +++ b/static/bidder-info/pulsepoint.yaml @@ -14,3 +14,7 @@ capabilities: - video - audio - native +userSync: + redirect: + url: "https://bh.contextweb.com/rtset?pid=561205&ev=1&rurl={{.RedirectURL}}" + userMacro: "%%VGUID%%" diff --git a/static/bidder-info/rhythmone.yaml b/static/bidder-info/rhythmone.yaml index 852344db3e3..0ea79afc766 100644 --- a/static/bidder-info/rhythmone.yaml +++ b/static/bidder-info/rhythmone.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://sync.1rx.io/usersync2/rmphb?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[RX_UUID]" diff --git a/static/bidder-info/sharethrough.yaml b/static/bidder-info/sharethrough.yaml index 45dceb94d22..12c9c4bc833 100644 --- a/static/bidder-info/sharethrough.yaml +++ b/static/bidder-info/sharethrough.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - native - banner +userSync: + redirect: + url: "https://match.sharethrough.com/FGMrCMMc/v1?redirectUri={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/smartadserver.yaml b/static/bidder-info/smartadserver.yaml index f22c7149ff7..a6b06d95aa7 100644 --- a/static/bidder-info/smartadserver.yaml +++ b/static/bidder-info/smartadserver.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ssbsync-global.smartadserver.com/api/sync?callerId=5&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri={{.RedirectURL}}" + userMacro: "[ssb_sync_pid]" diff --git a/static/bidder-info/smartrtb.yaml b/static/bidder-info/smartrtb.yaml index c26184f91b7..d73afe0440c 100644 --- a/static/bidder-info/smartrtb.yaml +++ b/static/bidder-info/smartrtb.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://market-global.smrtb.com/sync/all?nid=smartrtb&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&rr={{.RedirectURL}}" + userMacro: "{XID}" diff --git a/static/bidder-info/smartyads.yaml b/static/bidder-info/smartyads.yaml index df4c1b7ffb5..2704f007ca3 100644 --- a/static/bidder-info/smartyads.yaml +++ b/static/bidder-info/smartyads.yaml @@ -11,4 +11,8 @@ capabilities: - banner - video - native +userSync: + redirect: + url: "https://as.ck-ie.com/prebid.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/somoaudience.yaml b/static/bidder-info/somoaudience.yaml index 83b64e5c58e..4a4813c42a5 100644 --- a/static/bidder-info/somoaudience.yaml +++ b/static/bidder-info/somoaudience.yaml @@ -11,3 +11,7 @@ capabilities: - banner - native - video +userSync: + redirect: + url: "https://publisher-east.mobileadtrading.com/usersync?ru={{.RedirectURL}}" + userMacro: "${UID}" diff --git a/static/bidder-info/sonobi.yaml b/static/bidder-info/sonobi.yaml index 22a0a158306..9961d322083 100644 --- a/static/bidder-info/sonobi.yaml +++ b/static/bidder-info/sonobi.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://sync.go.sonobi.com/us.gif?loc={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/sovrn.yaml b/static/bidder-info/sovrn.yaml index 4c6251bdf57..4741c8a19fe 100644 --- a/static/bidder-info/sovrn.yaml +++ b/static/bidder-info/sovrn.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + url: "https://ap.lijit.com/pixel?redir={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/synacormedia.yaml b/static/bidder-info/synacormedia.yaml index 33086cd8e6c..14a4bcd1244 100644 --- a/static/bidder-info/synacormedia.yaml +++ b/static/bidder-info/synacormedia.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + url: "https://sync.technoratimedia.com/services?srv=cs&pid=70&cb={{.RedirectURL}}" + userMacro: "[USER_ID]" diff --git a/static/bidder-info/tappx.yaml b/static/bidder-info/tappx.yaml index eb655aa6a0c..6a321af0727 100644 --- a/static/bidder-info/tappx.yaml +++ b/static/bidder-info/tappx.yaml @@ -9,4 +9,8 @@ capabilities: site: mediaTypes: - banner - - video \ No newline at end of file + - video +userSync: + iframe: + url: "https://ssp.api.tappx.com/cs/usersync.php?gdpr_optin={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&type=iframe&ruid={{.RedirectURL}}" + userMacro: "{{TPPXUID}}" diff --git a/static/bidder-info/telaria.yaml b/static/bidder-info/telaria.yaml index 736fb9720b3..7d9501dc086 100644 --- a/static/bidder-info/telaria.yaml +++ b/static/bidder-info/telaria.yaml @@ -8,3 +8,7 @@ capabilities: site: mediaTypes: - video +userSync: + redirect: + url: "https://pbs.publishers.tremorhub.com/pubsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redir={{.RedirectURL}}" + userMacro: "[tvid]" diff --git a/static/bidder-info/triplelift.yaml b/static/bidder-info/triplelift.yaml index fe0ad8b2203..99026b6f171 100644 --- a/static/bidder-info/triplelift.yaml +++ b/static/bidder-info/triplelift.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/triplelift_native.yaml b/static/bidder-info/triplelift_native.yaml index 40d9be8f294..f9cc30fbec1 100644 --- a/static/bidder-info/triplelift_native.yaml +++ b/static/bidder-info/triplelift_native.yaml @@ -8,3 +8,8 @@ capabilities: site: mediaTypes: - native +userSync: + key: "triplelift" + redirect: + url: "https://eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "$UID" \ No newline at end of file diff --git a/static/bidder-info/trustx.yaml b/static/bidder-info/trustx.yaml index 6cfdd9fa465..b6c13bc7000 100644 --- a/static/bidder-info/trustx.yaml +++ b/static/bidder-info/trustx.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://x.bidswitch.net/check_uuid/{{.RedirectURL}}?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}" + userMacro: "${BSW_UUID}" diff --git a/static/bidder-info/ucfunnel.yaml b/static/bidder-info/ucfunnel.yaml index e6be68a0261..ca366c395c6 100644 --- a/static/bidder-info/ucfunnel.yaml +++ b/static/bidder-info/ucfunnel.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://sync.aralego.com/idsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&usprivacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "SspCookieUserId" diff --git a/static/bidder-info/unruly.yaml b/static/bidder-info/unruly.yaml index e31ea600b3e..ebefcb51c7b 100644 --- a/static/bidder-info/unruly.yaml +++ b/static/bidder-info/unruly.yaml @@ -8,3 +8,7 @@ capabilities: app: mediaTypes: - video +userSync: + iframe: + url: "https://usermatch.targeting.unrulymedia.com/pbsync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&rurl={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/valueimpression.yaml b/static/bidder-info/valueimpression.yaml index 1d64abcb68f..0782f897f2a 100644 --- a/static/bidder-info/valueimpression.yaml +++ b/static/bidder-info/valueimpression.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://rtb.valueimpression.com/usersync?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/visx.yaml b/static/bidder-info/visx.yaml index 789ce478bea..77d4463ac7a 100644 --- a/static/bidder-info/visx.yaml +++ b/static/bidder-info/visx.yaml @@ -8,3 +8,7 @@ capabilities: app: mediaTypes: - banner +userSync: + redirect: + url: "https://t.visx.net/s2s_sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" + userMacro: "${UUID}" diff --git a/static/bidder-info/yieldlab.yaml b/static/bidder-info/yieldlab.yaml index 3030d8a1d42..14960089cd1 100644 --- a/static/bidder-info/yieldlab.yaml +++ b/static/bidder-info/yieldlab.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ad.yieldlab.net/mr?t=2&pid=9140838&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r={{.RedirectURL}}" + userMacro: "%%YL_UID%%" diff --git a/static/bidder-info/yieldmo.yaml b/static/bidder-info/yieldmo.yaml index b1385acbebc..c5feacc83b9 100644 --- a/static/bidder-info/yieldmo.yaml +++ b/static/bidder-info/yieldmo.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://ads.yieldmo.com/pbsync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/yieldone.yaml b/static/bidder-info/yieldone.yaml index 74aef46d24f..46215b5e8ec 100644 --- a/static/bidder-info/yieldone.yaml +++ b/static/bidder-info/yieldone.yaml @@ -9,3 +9,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://y.one.impact-ad.jp/hbs_cs?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirectUri={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/zeroclickfraud.yaml b/static/bidder-info/zeroclickfraud.yaml index 527c0065600..8f7149ea137 100644 --- a/static/bidder-info/zeroclickfraud.yaml +++ b/static/bidder-info/zeroclickfraud.yaml @@ -11,3 +11,7 @@ capabilities: - banner - native - video +userSync: + iframe: + url: "https://s.0cf.io/sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" + userMacro: "${uid}" From 24d04f70b304ae732e83f7a3a7d8e5c15ebda887 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 16 Jun 2021 02:35:45 -0400 Subject: [PATCH 24/50] User Sync Metrics --- endpoints/cookie_sync.go | 10 +- endpoints/cookie_sync_test.go | 18 +-- endpoints/setuid.go | 55 ++++----- endpoints/setuid_test.go | 72 +++++++---- exchange/exchange_test.go | 13 -- metrics/config/metrics.go | 47 +++++--- metrics/go_metrics.go | 81 ++++++------- metrics/go_metrics_test.go | 109 ++++++++++++++--- metrics/metrics.go | 97 ++++++++------- metrics/metrics_mock.go | 13 +- metrics/prometheus/preload.go | 25 ++-- metrics/prometheus/prometheus.go | 96 ++++++++------- metrics/prometheus/prometheus_test.go | 164 +++++++++++++++++--------- metrics/prometheus/type_conversion.go | 39 +++--- 14 files changed, 513 insertions(+), 326 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index c4390976df9..e04d90ba5c4 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -236,15 +236,15 @@ func (c *cookieSyncEndpoint) writeBidderMetrics(biddersEvaluated []usersync.Bidd for _, bidder := range biddersEvaluated { switch bidder.Status { case usersync.StatusOK: - c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerOK) + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerCookieSyncOK) case usersync.StatusBlockedByGDPR: - c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerPrivacyBlocked) + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerCookieSyncPrivacyBlocked) case usersync.StatusBlockedByCCPA: - c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerPrivacyBlocked) + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerCookieSyncPrivacyBlocked) case usersync.StatusAlreadySynced: - c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerAlreadySynced) + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerCookieSyncAlreadySynced) case usersync.StatusTypeNotSupported: - c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerTypeNotSupported) + c.metrics.RecordSyncerRequest(bidder.SyncerKey, metrics.SyncerCookieSyncTypeNotSupported) } } } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index e5d9785b085..56077a61c5a 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -103,7 +103,7 @@ func TestCookieSyncHandle(t *testing.T) { `]}` + "\n", setMetricsExpectations: func(m *metrics.MetricsEngineMock) { m.On("RecordCookieSync", metrics.CookieSyncOK).Once() - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncOK).Once() }, setAnalyticsExpectations: func(a *MockAnalytics) { expected := analytics.CookieSyncObject{ @@ -135,7 +135,7 @@ func TestCookieSyncHandle(t *testing.T) { `]}` + "\n", setMetricsExpectations: func(m *metrics.MetricsEngineMock) { m.On("RecordCookieSync", metrics.CookieSyncOK).Once() - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncOK).Once() }, setAnalyticsExpectations: func(a *MockAnalytics) { expected := analytics.CookieSyncObject{ @@ -865,35 +865,35 @@ func TestCookieSyncWriteBidderMetrics(t *testing.T) { description: "One - OK", given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusOK}}, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncOK).Once() }, }, { description: "One - Blocked By GDPR", given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusBlockedByGDPR}}, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerPrivacyBlocked).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncPrivacyBlocked).Once() }, }, { description: "One - Blocked By CCPA", given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusBlockedByCCPA}}, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerPrivacyBlocked).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncPrivacyBlocked).Once() }, }, { description: "One - Already Synced", given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusAlreadySynced}}, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerAlreadySynced).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncAlreadySynced).Once() }, }, { description: "One - Type Not Supported", given: []usersync.BidderEvaluation{{Bidder: "a", SyncerKey: "aSyncer", Status: usersync.StatusTypeNotSupported}}, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerTypeNotSupported).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncTypeNotSupported).Once() }, }, { @@ -903,8 +903,8 @@ func TestCookieSyncWriteBidderMetrics(t *testing.T) { {Bidder: "b", SyncerKey: "bSyncer", Status: usersync.StatusAlreadySynced}, }, setExpectations: func(m *metrics.MetricsEngineMock) { - m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerOK).Once() - m.On("RecordSyncerRequest", "bSyncer", metrics.SyncerAlreadySynced).Once() + m.On("RecordSyncerRequest", "aSyncer", metrics.SyncerCookieSyncOK).Once() + m.On("RecordSyncerRequest", "bSyncer", metrics.SyncerCookieSyncAlreadySynced).Once() }, }, } diff --git a/endpoints/setuid.go b/endpoints/setuid.go index cd3430022a0..ad65815d5e0 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -14,7 +14,6 @@ import ( "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/gdpr" "github.com/prebid/prebid-server/metrics" - "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/httputil" ) @@ -41,9 +40,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer pc := usersync.ParseCookieFromRequest(r, &cfg) if !pc.AllowSyncs() { w.WriteHeader(http.StatusUnauthorized) - metricsEngine.RecordUserIDSet(metrics.UserLabels{ - Action: metrics.RequestActionOptOut, - }) + metricsEngine.RecordSetUid(metrics.SetUidOptOut) so.Status = http.StatusUnauthorized return } @@ -54,21 +51,30 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) - metricsEngine.RecordUserIDSet(metrics.UserLabels{ - Action: metrics.RequestActionErr, - }) + metricsEngine.RecordSetUid(metrics.SetUidSyncerUnknown) so.Status = http.StatusBadRequest return } so.Bidder = syncerKey + responseFormat, err := getResponseFormat(query, syncers[syncerKey]) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(err.Error())) + metricsEngine.RecordSetUid(metrics.SetUidBadRequest) + so.Status = http.StatusBadRequest + return + } + if shouldReturn, status, body := preventSyncsGDPR(query.Get("gdpr"), query.Get("gdpr_consent"), perms); shouldReturn { w.WriteHeader(status) w.Write([]byte(body)) - metricsEngine.RecordUserIDSet(metrics.UserLabels{ - Action: metrics.RequestActionGDPR, - Bidder: openrtb_ext.BidderName(syncerKey), - }) + switch status { + case http.StatusBadRequest: + metricsEngine.RecordSetUid(metrics.SetUidBadRequest) + case http.StatusUnavailableForLegalReasons: + metricsEngine.RecordSetUid(metrics.SetUidGDPRHostCookieBlocked) + } so.Status = status return } @@ -78,33 +84,20 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer if uid == "" { pc.Unsync(syncerKey) + metricsEngine.RecordSetUid(metrics.SetUidOK) + metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidCleared) + so.Success = true } else { - err = pc.TrySync(syncerKey, uid) - } - - if err == nil { - labels := metrics.UserLabels{ - Action: metrics.RequestActionSet, - Bidder: openrtb_ext.BidderName(syncerKey), + if err = pc.TrySync(syncerKey, uid); err == nil { + metricsEngine.RecordSetUid(metrics.SetUidOK) + metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidOK) + so.Success = true } - metricsEngine.RecordUserIDSet(labels) - so.Success = true } setSiteCookie := siteCookieCheck(r.UserAgent()) pc.SetCookieOnResponse(w, setSiteCookie, &cfg, cookieTTL) - responseFormat, err := getResponseFormat(query, syncers[syncerKey]) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(err.Error())) - metricsEngine.RecordUserIDSet(metrics.UserLabels{ - Action: metrics.RequestActionErr, - }) - so.Status = http.StatusBadRequest - return - } - switch responseFormat { case "i": w.Header().Add("Content-Type", httputil.Pixel1x1PNG.ContentType) diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 4c512139e62..30c819babe5 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -217,7 +217,7 @@ func TestSetUIDEndpoint(t *testing.T) { syncers: []string{"pubmatic"}, existingSyncs: nil, gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedSyncs: nil, expectedStatusCode: http.StatusBadRequest, expectedBody: `"f" query param is invalid. must be "b" or "i"`, description: "Set uid for valid bidder with invalid format", @@ -259,72 +259,98 @@ func TestSetUIDEndpointMetrics(t *testing.T) { cookieWithOptOut.SetOptOut(true) testCases := []struct { + description string uri string cookies []*usersync.Cookie syncers []string gdprAllowsHostCookies bool - expectedMetricAction metrics.RequestAction - expectedMetricBidder openrtb_ext.BidderName expectedResponseCode int - description string + expectedMetrics func(*metrics.MetricsEngineMock) }{ { + description: "Success - Sync", uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{}, syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, - expectedMetricAction: metrics.RequestActionSet, - expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), expectedResponseCode: 200, - description: "Success - Sync", + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidOK).Once() + m.On("RecordSyncerSet", "pubmatic", metrics.SyncerSetUidOK).Once() + }, }, { + description: "Success - Unsync", uri: "/setuid?bidder=pubmatic&uid=", cookies: []*usersync.Cookie{}, syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, - expectedMetricAction: metrics.RequestActionSet, - expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), expectedResponseCode: 200, - description: "Success - Unsync", + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidOK).Once() + m.On("RecordSyncerSet", "pubmatic", metrics.SyncerSetUidCleared).Once() + }, }, { + description: "Cookie Opted Out", uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{cookieWithOptOut}, syncers: []string{"pubmatic"}, gdprAllowsHostCookies: true, - expectedMetricAction: metrics.RequestActionOptOut, expectedResponseCode: 401, - description: "Cookie Opted Out", + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidOptOut).Once() + }, }, { + description: "Unknown Syncer Key", uri: "/setuid?bidder=pubmatic&uid=123", cookies: []*usersync.Cookie{}, syncers: []string{}, gdprAllowsHostCookies: true, - expectedMetricAction: metrics.RequestActionErr, expectedResponseCode: 400, - description: "Unsupported Cookie Name", + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidSyncerUnknown).Once() + }, + }, + { + description: "Unknown Format", + uri: "/setuid?bidder=pubmatic&uid=123&f=z", + cookies: []*usersync.Cookie{}, + syncers: []string{"pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 400, + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidBadRequest).Once() + }, }, { + description: "Prevented By GDPR - Invalid Consent String", uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1", cookies: []*usersync.Cookie{}, syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: false, - expectedMetricAction: metrics.RequestActionGDPR, - expectedMetricBidder: openrtb_ext.BidderName("pubmatic"), + gdprAllowsHostCookies: true, expectedResponseCode: 400, - description: "Prevented By GDPR", + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidBadRequest).Once() + }, + }, + { + description: "Prevented By GDPR - Permission Denied By Consent String", + uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=any", + cookies: []*usersync.Cookie{}, + syncers: []string{"pubmatic"}, + gdprAllowsHostCookies: false, + expectedResponseCode: 451, + expectedMetrics: func(m *metrics.MetricsEngineMock) { + m.On("RecordSetUid", metrics.SetUidGDPRHostCookieBlocked).Once() + }, }, } for _, test := range testCases { metricsEngine := &metrics.MetricsEngineMock{} - expectedLabels := metrics.UserLabels{ - Action: test.expectedMetricAction, - Bidder: test.expectedMetricBidder, - } - metricsEngine.On("RecordUserIDSet", expectedLabels).Once() + test.expectedMetrics(metricsEngine) req := httptest.NewRequest("GET", test.uri, nil) for _, v := range test.cookies { diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index df0ac639426..22b1681f55d 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -3613,19 +3613,6 @@ func (e *emptyUsersync) HasAnyLiveSyncs() bool { return false } -type mockUsersync struct { - syncs map[string]string -} - -func (e *mockUsersync) GetId(bidder openrtb_ext.BidderName) (id string, exists bool) { - id, exists = e.syncs[string(bidder)] - return -} - -func (e *mockUsersync) HasAnyLiveSyncs() bool { - return len(e.syncs) > 0 -} - type panicingAdapter struct{} func (panicingAdapter) requestBid(ctx context.Context, request *openrtb2.BidRequest, name openrtb_ext.BidderName, bidAdjustment float64, conversions currency.Conversions, reqInfo *adapters.ExtraRequestInfo, accountDebugAllowed, headerDebugAllowed bool) (posb *pbsOrtbSeatBid, errs []error) { diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index 9f3ed9a8771..34d874d9553 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -181,6 +181,27 @@ func (me *MultiMetricsEngine) RecordCookieSync(status metrics.CookieSyncStatus) } } +// RecordSyncerRequest across all engines +func (me *MultiMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerCookieSyncStatus) { + for _, thisME := range *me { + thisME.RecordSyncerRequest(key, status) + } +} + +// RecordSetUid across all engines +func (me *MultiMetricsEngine) RecordSetUid(status metrics.SetUidStatus) { + for _, thisME := range *me { + thisME.RecordSetUid(status) + } +} + +// RecordSyncerSet across all engines +func (me *MultiMetricsEngine) RecordSyncerSet(key string, status metrics.SyncerSetUidStatus) { + for _, thisME := range *me { + thisME.RecordSyncerSet(key, status) + } +} + // RecordStoredReqCacheResult across all engines func (me *MultiMetricsEngine) RecordStoredReqCacheResult(cacheResult metrics.CacheResult, inc int) { for _, thisME := range *me { @@ -202,20 +223,6 @@ func (me *MultiMetricsEngine) RecordAccountCacheResult(cacheResult metrics.Cache } } -// RecordSyncerRequest across all engines -func (me *MultiMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerStatus) { - for _, thisME := range *me { - thisME.RecordSyncerRequest(key, status) - } -} - -// RecordUserIDSet across all engines -func (me *MultiMetricsEngine) RecordUserIDSet(userLabels metrics.UserLabels) { - for _, thisME := range *me { - thisME.RecordUserIDSet(userLabels) - } -} - // RecordPrebidCacheRequestTime across all engines func (me *MultiMetricsEngine) RecordPrebidCacheRequestTime(success bool, length time.Duration) { for _, thisME := range *me { @@ -322,12 +329,16 @@ func (me *DummyMetricsEngine) RecordAdapterTime(labels metrics.AdapterLabels, le func (me *DummyMetricsEngine) RecordCookieSync(status metrics.CookieSyncStatus) { } -// RecordSyncerCRecordSyncerRequestookieSync as a noop -func (me *DummyMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerStatus) { +// RecordSyncerRequest as a noop +func (me *DummyMetricsEngine) RecordSyncerRequest(key string, status metrics.SyncerCookieSyncStatus) { +} + +// RecordSetUid as a noop +func (me *DummyMetricsEngine) RecordSetUid(status metrics.SetUidStatus) { } -// RecordUserIDSet as a noop -func (me *DummyMetricsEngine) RecordUserIDSet(userLabels metrics.UserLabels) { +// RecordSyncerSet as a noop +func (me *DummyMetricsEngine) RecordSyncerSet(key string, status metrics.SyncerSetUidStatus) { } // RecordStoredReqCacheResult as a noop diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index 5bda015f7c6..615b83b8be9 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -39,11 +39,10 @@ type Metrics struct { AmpNoCookieMeter metrics.Meter CookieSyncMeter metrics.Meter CookieSyncStatusMeter map[CookieSyncStatus]metrics.Meter - SyncerRequestsMeter map[string]map[SyncerStatus]metrics.Meter - userSyncOptout metrics.Meter - userSyncBadRequest metrics.Meter - userSyncSet map[openrtb_ext.BidderName]metrics.Meter - userSyncGDPRPrevent map[openrtb_ext.BidderName]metrics.Meter + SyncerRequestsMeter map[string]map[SyncerCookieSyncStatus]metrics.Meter + SetUidMeter metrics.Meter + SetUidStatusMeter map[SetUidStatus]metrics.Meter + SyncerSetsMeter map[string]map[SyncerSetUidStatus]metrics.Meter // Media types found in the "imp" JSON object ImpsTypeBanner metrics.Meter @@ -140,11 +139,10 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderNa AmpNoCookieMeter: blankMeter, CookieSyncMeter: blankMeter, CookieSyncStatusMeter: make(map[CookieSyncStatus]metrics.Meter), - SyncerRequestsMeter: make(map[string]map[SyncerStatus]metrics.Meter), - userSyncOptout: blankMeter, - userSyncBadRequest: blankMeter, - userSyncSet: make(map[openrtb_ext.BidderName]metrics.Meter), - userSyncGDPRPrevent: make(map[openrtb_ext.BidderName]metrics.Meter), + SyncerRequestsMeter: make(map[string]map[SyncerCookieSyncStatus]metrics.Meter), + SetUidMeter: blankMeter, + SetUidStatusMeter: make(map[SetUidStatus]metrics.Meter), + SyncerSetsMeter: make(map[string]map[SyncerSetUidStatus]metrics.Meter), ImpsTypeBanner: blankMeter, ImpsTypeVideo: blankMeter, @@ -246,24 +244,29 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d newMetrics.AmpNoCookieMeter = metrics.GetOrRegisterMeter("amp_no_cookie_requests", registry) - newMetrics.CookieSyncMeter = metrics.GetOrRegisterMeter("cookie_sync_requests", registry) // total count - + newMetrics.CookieSyncMeter = metrics.GetOrRegisterMeter("cookie_sync_requests", registry) for _, s := range CookieSyncStatuses() { newMetrics.CookieSyncStatusMeter[s] = metrics.GetOrRegisterMeter(fmt.Sprintf("cookie_sync_requests.%s", s), registry) } + newMetrics.SetUidMeter = metrics.GetOrRegisterMeter("setuid_requests", registry) + for _, s := range SetUidStatuses() { + newMetrics.SetUidStatusMeter[s] = metrics.GetOrRegisterMeter(fmt.Sprintf("setuid_requests.%s", s), registry) + } + for _, syncerKey := range syncerKeys { - newMetrics.SyncerRequestsMeter[syncerKey] = make(map[SyncerStatus]metrics.Meter) - for _, status := range SyncerStatuses() { - newMetrics.SyncerRequestsMeter[syncerKey][status] = metrics.GetOrRegisterMeter(fmt.Sprintf("syncer_requests.%s.%s", syncerKey, status), registry) + newMetrics.SyncerRequestsMeter[syncerKey] = make(map[SyncerCookieSyncStatus]metrics.Meter) + for _, status := range SyncerRequestStatuses() { + newMetrics.SyncerRequestsMeter[syncerKey][status] = metrics.GetOrRegisterMeter(fmt.Sprintf("syncer.%s.request.%s", syncerKey, status), registry) + } + + newMetrics.SyncerSetsMeter[syncerKey] = make(map[SyncerSetUidStatus]metrics.Meter) + for _, status := range SyncerSetUidStatuses() { + newMetrics.SyncerSetsMeter[syncerKey][status] = metrics.GetOrRegisterMeter(fmt.Sprintf("syncer.%s.set.%s", syncerKey, status), registry) } } - newMetrics.userSyncBadRequest = metrics.GetOrRegisterMeter("usersync.bad_requests", registry) - newMetrics.userSyncOptout = metrics.GetOrRegisterMeter("usersync.opt_outs", registry) for _, a := range exchanges { - newMetrics.userSyncSet[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("usersync.%s.sets", string(a)), registry) - newMetrics.userSyncGDPRPrevent[a] = metrics.GetOrRegisterMeter(fmt.Sprintf("usersync.%s.gdpr_prevent", string(a)), registry) registerAdapterMetrics(registry, "adapter", string(a), newMetrics.AdapterMetrics[a]) } @@ -282,9 +285,6 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d newMetrics.RequestsQueueTimer["video"][true] = metrics.GetOrRegisterTimer("queued_requests.video.accepted", registry) newMetrics.RequestsQueueTimer["video"][false] = metrics.GetOrRegisterTimer("queued_requests.video.rejected", registry) - newMetrics.userSyncSet[unknownBidder] = metrics.GetOrRegisterMeter("usersync.unknown.sets", registry) - newMetrics.userSyncGDPRPrevent[unknownBidder] = metrics.GetOrRegisterMeter("usersync.unknown.gdpr_prevent", registry) - newMetrics.TimeoutNotificationSuccess = metrics.GetOrRegisterMeter("timeout_notification.ok", registry) newMetrics.TimeoutNotificationFailure = metrics.GetOrRegisterMeter("timeout_notification.failed", registry) @@ -625,14 +625,13 @@ func (me *Metrics) RecordAdapterTime(labels AdapterLabels, length time.Duration) // RecordCookieSync implements a part of the MetricsEngine interface. Records a cookie sync request func (me *Metrics) RecordCookieSync(status CookieSyncStatus) { me.CookieSyncMeter.Mark(1) - if meter, exists := me.CookieSyncStatusMeter[status]; exists { meter.Mark(1) } } // RecordSyncerRequest implements a part of the MetricsEngine interface. Records a cookie sync syncer request and status -func (me *Metrics) RecordSyncerRequest(key string, status SyncerStatus) { +func (me *Metrics) RecordSyncerRequest(key string, status SyncerCookieSyncStatus) { if keyMeter, exists := me.SyncerRequestsMeter[key]; exists { if statusMeter, exists := keyMeter[status]; exists { statusMeter.Mark(1) @@ -640,17 +639,20 @@ func (me *Metrics) RecordSyncerRequest(key string, status SyncerStatus) { } } -// RecordUserIDSet implements a part of the MetricsEngine interface. Records a cookie setuid request -func (me *Metrics) RecordUserIDSet(userLabels UserLabels) { - switch userLabels.Action { - case RequestActionOptOut: - me.userSyncOptout.Mark(1) - case RequestActionErr: - me.userSyncBadRequest.Mark(1) - case RequestActionSet: - doMark(userLabels.Bidder, me.userSyncSet) - case RequestActionGDPR: - doMark(userLabels.Bidder, me.userSyncGDPRPrevent) +// RecordSetUid implements a part of the MetricsEngine interface. Records a set uid sync request +func (me *Metrics) RecordSetUid(status SetUidStatus) { + me.SetUidMeter.Mark(1) + if meter, exists := me.SetUidStatusMeter[status]; exists { + meter.Mark(1) + } +} + +// RecordSyncerSet implements a part of the MetricsEngine interface. Records a set uid sync request and status +func (me *Metrics) RecordSyncerSet(key string, status SyncerSetUidStatus) { + if keyMeter, exists := me.SyncerSetsMeter[key]; exists { + if statusMeter, exists := keyMeter[status]; exists { + statusMeter.Mark(1) + } } } @@ -735,12 +737,3 @@ func (me *Metrics) RecordAdapterGDPRRequestBlocked(adapterName openrtb_ext.Bidde am.GDPRRequestBlocked.Mark(1) } - -func doMark(bidder openrtb_ext.BidderName, meters map[openrtb_ext.BidderName]metrics.Meter) { - met, ok := meters[bidder] - if ok { - met.Mark(1) - } else { - meters[unknownBidder].Mark(1) - } -} diff --git a/metrics/go_metrics_test.go b/metrics/go_metrics_test.go index 61a069a9898..346a64a737f 100644 --- a/metrics/go_metrics_test.go +++ b/metrics/go_metrics_test.go @@ -10,11 +10,10 @@ import ( "github.com/stretchr/testify/assert" ) -// todo: add tests for new sync metrics - func TestNewMetrics(t *testing.T) { registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, nil) + syncerKeys := []string{"foo"} + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, syncerKeys) ensureContains(t, registry, "app_requests", m.AppRequestMeter) ensureContains(t, registry, "no_cookie_requests", m.NoCookieMeter) @@ -23,9 +22,17 @@ func TestNewMetrics(t *testing.T) { ensureContainsAdapterMetrics(t, registry, "adapter.appnexus", m.AdapterMetrics["appnexus"]) ensureContainsAdapterMetrics(t, registry, "adapter.rubicon", m.AdapterMetrics["rubicon"]) ensureContains(t, registry, "cookie_sync_requests", m.CookieSyncMeter) - ensureContains(t, registry, "usersync.appnexus.gdpr_prevent", m.userSyncGDPRPrevent["appnexus"]) - ensureContains(t, registry, "usersync.rubicon.gdpr_prevent", m.userSyncGDPRPrevent["rubicon"]) - ensureContains(t, registry, "usersync.unknown.gdpr_prevent", m.userSyncGDPRPrevent["unknown"]) + ensureContains(t, registry, "cookie_sync_requests.ok", m.CookieSyncStatusMeter[CookieSyncOK]) + ensureContains(t, registry, "cookie_sync_requests.bad_request", m.CookieSyncStatusMeter[CookieSyncBadRequest]) + ensureContains(t, registry, "cookie_sync_requests.opt_out", m.CookieSyncStatusMeter[CookieSyncOptOut]) + ensureContains(t, registry, "cookie_sync_requests.gdpr_blocked_host_cookie", m.CookieSyncStatusMeter[CookieSyncGDPRHostCookieBlocked]) + ensureContains(t, registry, "setuid_requests", m.SetUidMeter) + ensureContains(t, registry, "setuid_requests.ok", m.SetUidStatusMeter[SetUidOK]) + ensureContains(t, registry, "setuid_requests.bad_request", m.SetUidStatusMeter[SetUidBadRequest]) + ensureContains(t, registry, "setuid_requests.opt_out", m.SetUidStatusMeter[SetUidOptOut]) + ensureContains(t, registry, "setuid_requests.gdpr_blocked_host_cookie", m.SetUidStatusMeter[SetUidGDPRHostCookieBlocked]) + ensureContains(t, registry, "setuid_requests.syncer_unknown", m.SetUidStatusMeter[SetUidSyncerUnknown]) + ensureContains(t, registry, "prebid_cache_request_time.ok", m.PrebidCacheRequestTimerSuccess) ensureContains(t, registry, "prebid_cache_request_time.err", m.PrebidCacheRequestTimerError) @@ -62,6 +69,13 @@ func TestNewMetrics(t *testing.T) { ensureContains(t, registry, "privacy.request.lmt", m.PrivacyLMTRequest) ensureContains(t, registry, "privacy.request.tcf.v2", m.PrivacyTCFRequestVersion[TCFVersionV2]) ensureContains(t, registry, "privacy.request.tcf.err", m.PrivacyTCFRequestVersion[TCFVersionErr]) + + ensureContains(t, registry, "syncer.foo.request.ok", m.SyncerRequestsMeter["foo"][SyncerCookieSyncOK]) + ensureContains(t, registry, "syncer.foo.request.privacy_blocked", m.SyncerRequestsMeter["foo"][SyncerCookieSyncPrivacyBlocked]) + ensureContains(t, registry, "syncer.foo.request.already_synced", m.SyncerRequestsMeter["foo"][SyncerCookieSyncAlreadySynced]) + ensureContains(t, registry, "syncer.foo.request.type_not_supported", m.SyncerRequestsMeter["foo"][SyncerCookieSyncTypeNotSupported]) + ensureContains(t, registry, "syncer.foo.set.ok", m.SyncerSetsMeter["foo"][SyncerSetUidOK]) + ensureContains(t, registry, "syncer.foo.set.cleared", m.SyncerSetsMeter["foo"][SyncerSetUidCleared]) } func TestRecordBidType(t *testing.T) { @@ -81,16 +95,6 @@ func TestRecordBidType(t *testing.T) { VerifyMetrics(t, "Appnexus Video Nurl Bids", m.AdapterMetrics[openrtb_ext.BidderAppnexus].MarkupMetrics[openrtb_ext.BidTypeVideo].NurlMeter.Count(), 1) } -func TestRecordGDPRRejection(t *testing.T) { - registry := metrics.NewRegistry() - m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, config.DisabledMetrics{}, nil) - m.RecordUserIDSet(UserLabels{ - Action: RequestActionGDPR, - Bidder: openrtb_ext.BidderAppnexus, - }) - VerifyMetrics(t, "GDPR sync rejects", m.userSyncGDPRPrevent[openrtb_ext.BidderAppnexus].Count(), 1) -} - func ensureContains(t *testing.T, registry metrics.Registry, name string, metric interface{}) { t.Helper() if inRegistry := registry.Get(name); inRegistry == nil { @@ -599,6 +603,79 @@ func TestRecordAdapterGDPRRequestBlocked(t *testing.T) { } } +func TestRecordCookieSync(t *testing.T) { + registry := metrics.NewRegistry() + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, nil) + + // Known + m.RecordCookieSync(CookieSyncBadRequest) + + // Unknown + m.RecordCookieSync(CookieSyncStatus("unknown status")) + + assert.Equal(t, m.CookieSyncMeter.Count(), int64(2)) + assert.Equal(t, m.CookieSyncStatusMeter[CookieSyncOK].Count(), int64(0)) + assert.Equal(t, m.CookieSyncStatusMeter[CookieSyncBadRequest].Count(), int64(1)) + assert.Equal(t, m.CookieSyncStatusMeter[CookieSyncOptOut].Count(), int64(0)) + assert.Equal(t, m.CookieSyncStatusMeter[CookieSyncGDPRHostCookieBlocked].Count(), int64(0)) +} + +func TestRecordSyncerRequest(t *testing.T) { + registry := metrics.NewRegistry() + syncerKeys := []string{"foo"} + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, syncerKeys) + + // Known + m.RecordSyncerRequest("foo", SyncerCookieSyncOK) + + // Unknown Bidder + m.RecordSyncerRequest("bar", SyncerCookieSyncOK) + + // Unknown Status + m.RecordSyncerRequest("foo", SyncerCookieSyncStatus("unknown status")) + + assert.Equal(t, m.SyncerRequestsMeter["foo"][SyncerCookieSyncOK].Count(), int64(1)) + assert.Equal(t, m.SyncerRequestsMeter["foo"][SyncerCookieSyncPrivacyBlocked].Count(), int64(0)) + assert.Equal(t, m.SyncerRequestsMeter["foo"][SyncerCookieSyncAlreadySynced].Count(), int64(0)) + assert.Equal(t, m.SyncerRequestsMeter["foo"][SyncerCookieSyncTypeNotSupported].Count(), int64(0)) +} + +func TestRecordSetUid(t *testing.T) { + registry := metrics.NewRegistry() + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, nil) + + // Known + m.RecordSetUid(SetUidOptOut) + + // Unknown + m.RecordSetUid(SetUidStatus("unknown status")) + + assert.Equal(t, m.SetUidMeter.Count(), int64(2)) + assert.Equal(t, m.SetUidStatusMeter[SetUidOK].Count(), int64(0)) + assert.Equal(t, m.SetUidStatusMeter[SetUidBadRequest].Count(), int64(0)) + assert.Equal(t, m.SetUidStatusMeter[SetUidOptOut].Count(), int64(1)) + assert.Equal(t, m.SetUidStatusMeter[SetUidGDPRHostCookieBlocked].Count(), int64(0)) + assert.Equal(t, m.SetUidStatusMeter[SetUidSyncerUnknown].Count(), int64(0)) +} + +func TestRecordSyncerSet(t *testing.T) { + registry := metrics.NewRegistry() + syncerKeys := []string{"foo"} + m := NewMetrics(registry, []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus, openrtb_ext.BidderRubicon}, config.DisabledMetrics{}, syncerKeys) + + // Known + m.RecordSyncerSet("foo", SyncerSetUidCleared) + + // Unknown Bidder + m.RecordSyncerSet("bar", SyncerSetUidCleared) + + // Unknown Status + m.RecordSyncerSet("foo", SyncerSetUidStatus("unknown status")) + + assert.Equal(t, m.SyncerSetsMeter["foo"][SyncerSetUidOK].Count(), int64(0)) + assert.Equal(t, m.SyncerSetsMeter["foo"][SyncerSetUidCleared].Count(), int64(1)) +} + func ensureContainsBidTypeMetrics(t *testing.T, registry metrics.Registry, prefix string, mdm map[openrtb_ext.BidType]*MarkupDeliveryMetrics) { ensureContains(t, registry, prefix+".banner.adm_bids_received", mdm[openrtb_ext.BidTypeBanner].AdmMeter) ensureContains(t, registry, prefix+".banner.nurl_bids_received", mdm[openrtb_ext.BidTypeBanner].NurlMeter) diff --git a/metrics/metrics.go b/metrics/metrics.go index a9a15e025e5..af45f9b4f5a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -267,33 +267,6 @@ func CacheResults() []CacheResult { } } -// UserLabels : Labels for /setuid endpoint -type UserLabels struct { - Action RequestAction - Bidder openrtb_ext.BidderName -} - -// RequestAction : The setuid request result -type RequestAction string - -// /setuid action labels -const ( - RequestActionSet RequestAction = "set" - RequestActionOptOut RequestAction = "opt_out" - RequestActionGDPR RequestAction = "gdpr" - RequestActionErr RequestAction = "err" -) - -// RequestActions returns possible setuid action labels -func RequestActions() []RequestAction { - return []RequestAction{ - RequestActionSet, - RequestActionOptOut, - RequestActionGDPR, - RequestActionErr, - } -} - // TCFVersionValue : The possible values for TCF versions type TCFVersionValue string @@ -319,6 +292,7 @@ func TCFVersionToValue(version int) TCFVersionValue { return TCFVersionErr } +// CookieSyncStatus is a status code resulting from a call to the /cookie_sync endpoint. type CookieSyncStatus string const ( @@ -338,22 +312,62 @@ func CookieSyncStatuses() []CookieSyncStatus { } } -type SyncerStatus string +// SyncerCookieSyncStatus is a status code from an invocation of a syncer resulting from a call to the /cookie_sync endpoint. +type SyncerCookieSyncStatus string + +const ( + SyncerCookieSyncOK SyncerCookieSyncStatus = "ok" + SyncerCookieSyncPrivacyBlocked SyncerCookieSyncStatus = "privacy_blocked" + SyncerCookieSyncAlreadySynced SyncerCookieSyncStatus = "already_synced" + SyncerCookieSyncTypeNotSupported SyncerCookieSyncStatus = "type_not_supported" +) + +// SyncerRequestStatuses returns possible syncer statuses. +func SyncerRequestStatuses() []SyncerCookieSyncStatus { + return []SyncerCookieSyncStatus{ + SyncerCookieSyncOK, + SyncerCookieSyncPrivacyBlocked, + SyncerCookieSyncAlreadySynced, + SyncerCookieSyncTypeNotSupported, + } +} + +// SetUidStatus is a status code resulting from a call to the /setuid endpoint. +type SetUidStatus string + +// /setuid action labels +const ( + SetUidOK SetUidStatus = "ok" + SetUidBadRequest SetUidStatus = "bad_request" + SetUidOptOut SetUidStatus = "opt_out" + SetUidGDPRHostCookieBlocked SetUidStatus = "gdpr_blocked_host_cookie" + SetUidSyncerUnknown SetUidStatus = "syncer_unknown" +) + +// SetUidStatuses returns possible setuid statuses. +func SetUidStatuses() []SetUidStatus { + return []SetUidStatus{ + SetUidOK, + SetUidBadRequest, + SetUidOptOut, + SetUidGDPRHostCookieBlocked, + SetUidSyncerUnknown, + } +} + +// SyncerSetUidStatus is a status code from an invocation of a syncer resulting from a call to the /setuid endpoint. +type SyncerSetUidStatus string const ( - SyncerOK SyncerStatus = "ok" - SyncerPrivacyBlocked SyncerStatus = "privacy_blocked" - SyncerAlreadySynced SyncerStatus = "already_synced" - SyncerTypeNotSupported SyncerStatus = "type_not_supported" + SyncerSetUidOK SyncerSetUidStatus = "ok" + SyncerSetUidCleared SyncerSetUidStatus = "cleared" ) -// SyncerStatuses returns possible syncer statuses. -func SyncerStatuses() []SyncerStatus { - return []SyncerStatus{ - SyncerOK, - SyncerPrivacyBlocked, - SyncerAlreadySynced, - SyncerTypeNotSupported, +// SyncerSetUidStatuses returns possible syncer set statuses. +func SyncerSetUidStatuses() []SyncerSetUidStatus { + return []SyncerSetUidStatus{ + SyncerSetUidOK, + SyncerSetUidCleared, } } @@ -381,8 +395,9 @@ type MetricsEngine interface { RecordAdapterPrice(labels AdapterLabels, cpm float64) RecordAdapterTime(labels AdapterLabels, length time.Duration) RecordCookieSync(status CookieSyncStatus) - RecordSyncerRequest(key string, status SyncerStatus) - RecordUserIDSet(userLabels UserLabels) // Function should verify bidder values + RecordSyncerRequest(key string, status SyncerCookieSyncStatus) + RecordSetUid(status SetUidStatus) + RecordSyncerSet(key string, status SyncerSetUidStatus) RecordStoredReqCacheResult(cacheResult CacheResult, inc int) RecordStoredImpCacheResult(cacheResult CacheResult, inc int) RecordAccountCacheResult(cacheResult CacheResult, inc int) diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index b0ff9955a88..b8ab23b768a 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -97,13 +97,18 @@ func (me *MetricsEngineMock) RecordCookieSync(status CookieSyncStatus) { } // RecordSyncerRequest mock -func (me *MetricsEngineMock) RecordSyncerRequest(key string, status SyncerStatus) { +func (me *MetricsEngineMock) RecordSyncerRequest(key string, status SyncerCookieSyncStatus) { me.Called(key, status) } -// RecordUserIDSet mock -func (me *MetricsEngineMock) RecordUserIDSet(userLabels UserLabels) { - me.Called(userLabels) +// RecordSetUid mock +func (me *MetricsEngineMock) RecordSetUid(status SetUidStatus) { + me.Called(status) +} + +// RecordSyncerSet mock +func (me *MetricsEngineMock) RecordSyncerSet(key string, status SyncerSetUidStatus) { + me.Called(key, status) } // RecordStoredReqCacheResult mock diff --git a/metrics/prometheus/preload.go b/metrics/prometheus/preload.go index 04532a15724..338b1e8b347 100644 --- a/metrics/prometheus/preload.go +++ b/metrics/prometheus/preload.go @@ -7,7 +7,7 @@ import ( func preloadLabelValues(m *Metrics, syncerKeys []string) { var ( - actionValues = actionsAsString() + setUidStatusValues = setUidStatusesAsString() adapterErrorValues = adapterErrorsAsString() adapterValues = adaptersAsString() bidTypeValues = []string{markupDeliveryAdm, markupDeliveryNurl} @@ -16,11 +16,12 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { connectionErrorValues = []string{connectionAcceptError, connectionCloseError} cookieValues = cookieTypesAsString() cookieSyncStatusValues = cookieSyncStatusesAsString() - requestStatusValues = requestStatusesAsString() requestTypeValues = requestTypesAsString() + requestStatusValues = requestStatusesAsString() storedDataFetchTypeValues = storedDataFetchTypesAsString() storedDataErrorValues = storedDataErrorsAsString() - syncerStatusValues = syncerStatusesAsString() + syncerRequestStatusValues = syncerRequestStatusesAsString() + syncerSetsStatusValues = syncerSetStatusesAsString() sourceValues = []string{sourceRequest} ) @@ -29,7 +30,11 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { }) preloadLabelValuesForCounter(m.cookieSync, map[string][]string{ - cookiesyncStatusLabel: cookieSyncStatusValues, + statusLabel: cookieSyncStatusValues, + }) + + preloadLabelValuesForCounter(m.setUid, map[string][]string{ + statusLabel: setUidStatusValues, }) preloadLabelValuesForCounter(m.impressions, map[string][]string{ @@ -150,14 +155,14 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { adapterLabel: adapterValues, }) - preloadLabelValuesForCounter(m.adapterUserSync, map[string][]string{ - adapterLabel: adapterValues, - actionLabel: actionValues, + preloadLabelValuesForCounter(m.syncerRequests, map[string][]string{ + syncerLabel: syncerKeys, + statusLabel: syncerRequestStatusValues, }) - preloadLabelValuesForCounter(m.syncerRequests, map[string][]string{ - syncerLabel: syncerKeys, - syncerStatusLabel: syncerStatusValues, + preloadLabelValuesForCounter(m.syncerSets, map[string][]string{ + syncerLabel: syncerKeys, + statusLabel: syncerSetsStatusValues, }) //to minimize memory usage, queuedTimeout metric is now supported for video endpoint only diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 9c4741b6d07..52470369094 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -19,6 +19,7 @@ type Metrics struct { connectionsError *prometheus.CounterVec connectionsOpened prometheus.Counter cookieSync *prometheus.CounterVec + setUid *prometheus.CounterVec impressions *prometheus.CounterVec impressionsLegacy prometheus.Counter prebidCacheWriteTimer *prometheus.HistogramVec @@ -54,7 +55,6 @@ type Metrics struct { adapterPrices *prometheus.HistogramVec adapterRequests *prometheus.CounterVec adapterRequestsTimer *prometheus.HistogramVec - adapterUserSync *prometheus.CounterVec adapterReusedConnections *prometheus.CounterVec adapterCreatedConnections *prometheus.CounterVec adapterConnectionWaitTime *prometheus.HistogramVec @@ -62,6 +62,7 @@ type Metrics struct { // Syncer Metrics syncerRequests *prometheus.CounterVec + syncerSets *prometheus.CounterVec // Account Metrics accountRequests *prometheus.CounterVec @@ -70,29 +71,28 @@ type Metrics struct { } const ( - accountLabel = "account" - actionLabel = "action" - adapterErrorLabel = "adapter_error" - adapterLabel = "adapter" - bidTypeLabel = "bid_type" - cacheResultLabel = "cache_result" - connectionErrorLabel = "connection_error" - cookieLabel = "cookie" - cookiesyncStatusLabel = "status" - hasBidsLabel = "has_bids" - isAudioLabel = "audio" - isBannerLabel = "banner" - isNativeLabel = "native" - isVideoLabel = "video" - markupDeliveryLabel = "delivery" - optOutLabel = "opt_out" - privacyBlockedLabel = "privacy_blocked" - requestStatusLabel = "request_status" - requestTypeLabel = "request_type" - successLabel = "success" - syncerLabel = "syncer" - syncerStatusLabel = "status" - versionLabel = "version" + accountLabel = "account" + actionLabel = "action" + adapterErrorLabel = "adapter_error" + adapterLabel = "adapter" + bidTypeLabel = "bid_type" + cacheResultLabel = "cache_result" + connectionErrorLabel = "connection_error" + cookieLabel = "cookie" + hasBidsLabel = "has_bids" + isAudioLabel = "audio" + isBannerLabel = "banner" + isNativeLabel = "native" + isVideoLabel = "video" + markupDeliveryLabel = "delivery" + optOutLabel = "opt_out" + privacyBlockedLabel = "privacy_blocked" + requestStatusLabel = "request_status" + requestTypeLabel = "request_type" + statusLabel = "status" + successLabel = "success" + syncerLabel = "syncer" + versionLabel = "version" ) const ( @@ -152,7 +152,12 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet metrics.cookieSync = newCounter(cfg, metrics.Registry, "cookie_sync_requests", "Count of cookie sync requests to Prebid Server.", - []string{cookiesyncStatusLabel}) + []string{statusLabel}) + + metrics.setUid = newCounter(cfg, metrics.Registry, + "setuid_requests", + "Count of set uid requests to Prebid Server.", + []string{statusLabel}) metrics.impressions = newCounter(cfg, metrics.Registry, "impressions_requests", @@ -347,15 +352,15 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet []string{adapterLabel}, standardTimeBuckets) - metrics.adapterUserSync = newCounter(cfg, metrics.Registry, - "adapter_user_sync", - "Count of user ID sync requests received labeled by adapter and action.", - []string{adapterLabel, actionLabel}) - metrics.syncerRequests = newCounter(cfg, metrics.Registry, "syncer_requests", - "Count of cookie sync requests received labeled by syncer key and status.", - []string{syncerLabel, syncerStatusLabel}) + "Count of cookie sync requests where a syncer is a candidate to be synced labeled by syncer key and status.", + []string{syncerLabel, statusLabel}) + + metrics.syncerSets = newCounter(cfg, metrics.Registry, + "syncer_sets", + "Count of setuid set requests for a syncer labeled by syncer key and status.", + []string{syncerLabel, statusLabel}) metrics.accountRequests = newCounter(cfg, metrics.Registry, "account_requests", @@ -612,25 +617,28 @@ func (m *Metrics) RecordAdapterTime(labels metrics.AdapterLabels, length time.Du func (m *Metrics) RecordCookieSync(status metrics.CookieSyncStatus) { m.cookieSync.With(prometheus.Labels{ - cookiesyncStatusLabel: string(status), + statusLabel: string(status), }).Inc() } -func (m *Metrics) RecordSyncerRequest(key string, status metrics.SyncerStatus) { +func (m *Metrics) RecordSyncerRequest(key string, status metrics.SyncerCookieSyncStatus) { m.syncerRequests.With(prometheus.Labels{ - syncerLabel: key, - syncerStatusLabel: string(status), + syncerLabel: key, + statusLabel: string(status), }).Inc() } -func (m *Metrics) RecordUserIDSet(labels metrics.UserLabels) { - adapter := string(labels.Bidder) - if adapter != "" { - m.adapterUserSync.With(prometheus.Labels{ - adapterLabel: adapter, - actionLabel: string(labels.Action), - }).Inc() - } +func (m *Metrics) RecordSetUid(status metrics.SetUidStatus) { + m.setUid.With(prometheus.Labels{ + statusLabel: string(status), + }).Inc() +} + +func (m *Metrics) RecordSyncerSet(key string, status metrics.SyncerSetUidStatus) { + m.syncerSets.With(prometheus.Labels{ + syncerLabel: key, + statusLabel: string(status), + }).Inc() } func (m *Metrics) RecordStoredReqCacheResult(cacheResult metrics.CacheResult, inc int) { diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index e7dd786585b..0fe852b81df 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -923,57 +923,6 @@ func TestAdapterTimeMetric(t *testing.T) { } } -func TestSyncerRequests(t *testing.T) { - m := createMetricsForTesting() - syncerKey := "anyKey" - syncerStatus := metrics.SyncerAlreadySynced - - m.RecordSyncerRequest(syncerKey, syncerStatus) - - assertCounterVecValue(t, "", "syncer_requests", m.syncerRequests, - float64(1), - prometheus.Labels{ - syncerLabel: syncerKey, - syncerStatusLabel: string(syncerStatus), - }) -} - -func TestUserIDSetMetric(t *testing.T) { - m := createMetricsForTesting() - adapterName := "anyName" - action := metrics.RequestActionSet - - m.RecordUserIDSet(metrics.UserLabels{ - Bidder: openrtb_ext.BidderName(adapterName), - Action: action, - }) - - expectedCount := float64(1) - assertCounterVecValue(t, "", "adapterUserSync", m.adapterUserSync, - expectedCount, - prometheus.Labels{ - adapterLabel: adapterName, - actionLabel: string(action), - }) -} - -func TestUserIDSetMetricWhenBidderEmpty(t *testing.T) { - m := createMetricsForTesting() - action := metrics.RequestActionErr - - m.RecordUserIDSet(metrics.UserLabels{ - Bidder: openrtb_ext.BidderName(""), - Action: action, - }) - - expectedTotalCount := float64(0) - actualTotalCount := float64(0) - processMetrics(m.adapterUserSync, func(m dto.Metric) { - actualTotalCount += m.GetCounter().GetValue() - }) - assert.Equal(t, expectedTotalCount, actualTotalCount, "total count") -} - func TestAdapterPanicMetric(t *testing.T) { m := createMetricsForTesting() adapterName := "anyName" @@ -1050,7 +999,7 @@ func TestAccountCacheResultMetric(t *testing.T) { }) } -func TestCookieMetric(t *testing.T) { +func TestCookieSyncMetric(t *testing.T) { tests := []struct { status metrics.CookieSyncStatus label string @@ -1081,10 +1030,119 @@ func TestCookieMetric(t *testing.T) { assertCounterVecValue(t, "", "cookie_sync_requests:"+test.label, m.cookieSync, float64(1), prometheus.Labels{ - cookiesyncStatusLabel: string(test.status), + statusLabel: string(test.status), }) } +} +func TestRecordSyncerRequestMetric(t *testing.T) { + key := "anyKey" + + tests := []struct { + status metrics.SyncerCookieSyncStatus + label string + }{ + { + status: metrics.SyncerCookieSyncOK, + label: "ok", + }, + { + status: metrics.SyncerCookieSyncPrivacyBlocked, + label: "privacy_blocked", + }, + { + status: metrics.SyncerCookieSyncAlreadySynced, + label: "already_synced", + }, + { + status: metrics.SyncerCookieSyncTypeNotSupported, + label: "type_not_supported", + }, + } + + for _, test := range tests { + m := createMetricsForTesting() + + m.RecordSyncerRequest(key, test.status) + + assertCounterVecValue(t, "", "syncer_requests:"+test.label, m.syncerRequests, + float64(1), + prometheus.Labels{ + syncerLabel: key, + statusLabel: string(test.status), + }) + } +} + +func TestSetUidMetric(t *testing.T) { + tests := []struct { + status metrics.SetUidStatus + label string + }{ + { + status: metrics.SetUidOK, + label: "ok", + }, + { + status: metrics.SetUidBadRequest, + label: "bad_request", + }, + { + status: metrics.SetUidOptOut, + label: "opt_out", + }, + { + status: metrics.SetUidGDPRHostCookieBlocked, + label: "gdpr_blocked_host_cookie", + }, + { + status: metrics.SetUidSyncerUnknown, + label: "syncer_unknown", + }, + } + + for _, test := range tests { + m := createMetricsForTesting() + + m.RecordSetUid(test.status) + + assertCounterVecValue(t, "", "setuid_requests:"+test.label, m.setUid, + float64(1), + prometheus.Labels{ + statusLabel: string(test.status), + }) + } +} + +func TestRecordSyncerSetMetric(t *testing.T) { + key := "anyKey" + + tests := []struct { + status metrics.SyncerSetUidStatus + label string + }{ + { + status: metrics.SyncerSetUidOK, + label: "ok", + }, + { + status: metrics.SyncerSetUidCleared, + label: "cleared", + }, + } + + for _, test := range tests { + m := createMetricsForTesting() + + m.RecordSyncerSet(key, test.status) + + assertCounterVecValue(t, "", "syncer_sets:"+test.label, m.syncerSets, + float64(1), + prometheus.Labels{ + syncerLabel: key, + statusLabel: string(test.status), + }) + } } func TestPrebidCacheRequestTimeMetric(t *testing.T) { diff --git a/metrics/prometheus/type_conversion.go b/metrics/prometheus/type_conversion.go index 2a43e991c1a..4e2d1ff5ba1 100644 --- a/metrics/prometheus/type_conversion.go +++ b/metrics/prometheus/type_conversion.go @@ -7,15 +7,6 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" ) -func actionsAsString() []string { - values := metrics.RequestActions() - valuesAsString := make([]string, len(values)) - for i, v := range values { - valuesAsString[i] = string(v) - } - return valuesAsString -} - func adaptersAsString() []string { values := openrtb_ext.CoreBidderNames() valuesAsString := make([]string, len(values)) @@ -41,6 +32,15 @@ func boolValuesAsString() []string { } } +func cacheResultsAsString() []string { + values := metrics.CacheResults() + valuesAsString := make([]string, len(values)) + for i, v := range values { + valuesAsString[i] = string(v) + } + return valuesAsString +} + func cookieTypesAsString() []string { values := metrics.CookieTypes() valuesAsString := make([]string, len(values)) @@ -59,8 +59,8 @@ func cookieSyncStatusesAsString() []string { return valuesAsString } -func syncerStatusesAsString() []string { - values := metrics.SyncerStatuses() +func requestStatusesAsString() []string { + values := metrics.RequestStatuses() valuesAsString := make([]string, len(values)) for i, v := range values { valuesAsString[i] = string(v) @@ -68,8 +68,8 @@ func syncerStatusesAsString() []string { return valuesAsString } -func cacheResultsAsString() []string { - values := metrics.CacheResults() +func syncerRequestStatusesAsString() []string { + values := metrics.SyncerRequestStatuses() valuesAsString := make([]string, len(values)) for i, v := range values { valuesAsString[i] = string(v) @@ -77,8 +77,8 @@ func cacheResultsAsString() []string { return valuesAsString } -func requestStatusesAsString() []string { - values := metrics.RequestStatuses() +func syncerSetStatusesAsString() []string { + values := metrics.SyncerSetUidStatuses() valuesAsString := make([]string, len(values)) for i, v := range values { valuesAsString[i] = string(v) @@ -95,6 +95,15 @@ func requestTypesAsString() []string { return valuesAsString } +func setUidStatusesAsString() []string { + values := metrics.SetUidStatuses() + valuesAsString := make([]string, len(values)) + for i, v := range values { + valuesAsString[i] = string(v) + } + return valuesAsString +} + func storedDataTypesAsString() []string { values := metrics.StoredDataTypes() valuesAsString := make([]string, len(values)) From 973cfb22798ab343f1b98371c2a1b99562d97ab4 Mon Sep 17 00:00:00 2001 From: Veronika Solovei Date: Wed, 30 Jun 2021 20:06:50 -0700 Subject: [PATCH 25/50] Code review fixes --- static/bidder-info/appnexus.yaml | 6 +++--- static/bidder-info/connectad.yaml | 1 + static/bidder-info/consumable.yaml | 1 + static/bidder-info/conversant.yaml | 1 + static/bidder-info/gumgum.yaml | 1 + static/bidder-info/ix.yaml | 1 + static/bidder-info/lockerdome.yaml | 11 +++++++---- static/bidder-info/pubmatic.yaml | 1 + 8 files changed, 16 insertions(+), 7 deletions(-) diff --git a/static/bidder-info/appnexus.yaml b/static/bidder-info/appnexus.yaml index dd7949f521b..41838159bbe 100644 --- a/static/bidder-info/appnexus.yaml +++ b/static/bidder-info/appnexus.yaml @@ -14,6 +14,6 @@ capabilities: - native userSync: key: "adnxs" - redirect: - url: "https://ib.adnxs.com/getuid?{{.RedirectURL}}" - userMacro: "$UID" + redirect: + url: "https://ib.adnxs.com/getuid?{{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/connectad.yaml b/static/bidder-info/connectad.yaml index 116ab79100e..63cf05427de 100644 --- a/static/bidder-info/connectad.yaml +++ b/static/bidder-info/connectad.yaml @@ -12,3 +12,4 @@ userSync: iframe: url: "https://cdn.connectad.io/connectmyusers.php?gdpr={{.GDPR}}&consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" userMacro: "" + # connectad appends the user id to end of the redirect url and does not utilize a macro diff --git a/static/bidder-info/consumable.yaml b/static/bidder-info/consumable.yaml index 17df0c15b0d..5200c738200 100644 --- a/static/bidder-info/consumable.yaml +++ b/static/bidder-info/consumable.yaml @@ -12,3 +12,4 @@ userSync: redirect: url: "https://e.serverbid.com/udb/9969/match?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" userMacro: "" + # consumable appends the user id to end of the redirect url and does not utilize a macro diff --git a/static/bidder-info/conversant.yaml b/static/bidder-info/conversant.yaml index cd55f01fef5..1d298f8b582 100644 --- a/static/bidder-info/conversant.yaml +++ b/static/bidder-info/conversant.yaml @@ -14,3 +14,4 @@ userSync: redirect: url: "https://prebid-match.dotomi.com/match/bounce/current?version=1&networkId=72582&rurl={{.RedirectURL}}" userMacro: "" + # conversant appends the user id to end of the redirect url and does not utilize a macro diff --git a/static/bidder-info/gumgum.yaml b/static/bidder-info/gumgum.yaml index 81f42d31877..34b6f47d39e 100644 --- a/static/bidder-info/gumgum.yaml +++ b/static/bidder-info/gumgum.yaml @@ -10,3 +10,4 @@ userSync: iframe: url: "https://rtb.gumgum.com/usync/prbds2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r={{.RedirectURL}}" userMacro: "" + # gumgum appends the user id to end of the redirect url and does not utilize a macro diff --git a/static/bidder-info/ix.yaml b/static/bidder-info/ix.yaml index 99dd5c5954f..8c46525caf0 100644 --- a/static/bidder-info/ix.yaml +++ b/static/bidder-info/ix.yaml @@ -18,3 +18,4 @@ userSync: redirect: url: "https://ssum.casalemedia.com/usermatchredir?s=194962&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb={{.RedirectURL}}" userMacro: "" + # ix appends the user id to end of the redirect url and does not utilize a macro diff --git a/static/bidder-info/lockerdome.yaml b/static/bidder-info/lockerdome.yaml index 550b4a80b38..eebf3b87841 100644 --- a/static/bidder-info/lockerdome.yaml +++ b/static/bidder-info/lockerdome.yaml @@ -7,7 +7,10 @@ capabilities: site: mediaTypes: - banner -userSync: - redirect: - url: "https://lockerdome.com/usync/prebidserver?pid="+cfg.Adapters["lockerdome"].PlatformID+"&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" - userMacro: "{{uid}}" +# lockerdome requires a platform id for their user sync process. replace <> +# in the url below and uncomment the userSync section to enable. +# +#userSync: +# redirect: +# url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" +# userMacro: "{{uid}}" diff --git a/static/bidder-info/pubmatic.yaml b/static/bidder-info/pubmatic.yaml index 695af9dfd81..feba7faa110 100644 --- a/static/bidder-info/pubmatic.yaml +++ b/static/bidder-info/pubmatic.yaml @@ -14,3 +14,4 @@ userSync: iframe: url: "https://ads.pubmatic.com/AdServer/js/user_sync.html?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&predirect={{.RedirectURL}}" userMacro: "" + # pubmatic appends the user id to end of the redirect url and does not utilize a macro From a03163971b856c51727c8767c013a0ce55944a00 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 6 Jul 2021 12:02:00 -0400 Subject: [PATCH 26/50] Redirect URL Comment / Warning --- config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.go b/config/config.go index 7cc8952713e..0c73be719a6 100644 --- a/config/config.go +++ b/config/config.go @@ -688,6 +688,8 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("accounts.filesystem.directorypath", "./stored_requests/data/by_id") v.SetDefault("accounts.in_memory_cache.type", "none") + // some adapters append the user id to the end of the redirect url instead of using + // macro substitution. it is important for the uid to be the last query parameter. v.SetDefault("user_sync.redirect_url", "{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&f={{.SyncType}}&uid={{.UserMacro}}") for _, bidder := range openrtb_ext.CoreBidderNames() { From 168c5b707ef6a823018fa75d7aed2d15b33c1485 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 6 Jul 2021 13:16:28 -0400 Subject: [PATCH 27/50] Merge missed files. --- adapters/bidmyadz/usersync.go | 12 --------- adapters/bidmyadz/usersync_test.go | 33 ------------------------- adapters/sa_lunamedia/usersync.go | 12 --------- adapters/sa_lunamedia/usersync_test.go | 33 ------------------------- adapters/smilewanted/usersync.go | 12 --------- adapters/smilewanted/usersync_test.go | 34 -------------------------- 6 files changed, 136 deletions(-) delete mode 100644 adapters/bidmyadz/usersync.go delete mode 100644 adapters/bidmyadz/usersync_test.go delete mode 100644 adapters/sa_lunamedia/usersync.go delete mode 100644 adapters/sa_lunamedia/usersync_test.go delete mode 100644 adapters/smilewanted/usersync.go delete mode 100644 adapters/smilewanted/usersync_test.go diff --git a/adapters/bidmyadz/usersync.go b/adapters/bidmyadz/usersync.go deleted file mode 100644 index 755a184d6e4..00000000000 --- a/adapters/bidmyadz/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package bidmyadz - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewBidmyadzSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("bidmyadz", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/bidmyadz/usersync_test.go b/adapters/bidmyadz/usersync_test.go deleted file mode 100644 index 11b5fedd73f..00000000000 --- a/adapters/bidmyadz/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package bidmyadz - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestNewBidmyadzSyncer(t *testing.T) { - syncURL := "https://test.com/c0f68227d14ed938c6c49f3967cbe9bc?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewBidmyadzSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "allGdpr", - }, - CCPA: ccpa.Policy{ - Consent: "1---", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://test.com/c0f68227d14ed938c6c49f3967cbe9bc?gdpr=0&gdpr_consent=allGdpr&ccpa=1---", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/sa_lunamedia/usersync.go b/adapters/sa_lunamedia/usersync.go deleted file mode 100644 index f78b7944cb2..00000000000 --- a/adapters/sa_lunamedia/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package salunamedia - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSaLunamediaSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("salunamedia", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/sa_lunamedia/usersync_test.go b/adapters/sa_lunamedia/usersync_test.go deleted file mode 100644 index e3820fbc1af..00000000000 --- a/adapters/sa_lunamedia/usersync_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package salunamedia - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestNewSaLunamediaSyncer(t *testing.T) { - syncURL := "https://test.com/pserver?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - syncer := NewSaLunamediaSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "0", - Consent: "allGdpr", - }, - CCPA: ccpa.Policy{ - Consent: "1---", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "https://test.com/pserver?gdpr=0&gdpr_consent=allGdpr&ccpa=1---", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} diff --git a/adapters/smilewanted/usersync.go b/adapters/smilewanted/usersync.go deleted file mode 100644 index 8f29cb845d8..00000000000 --- a/adapters/smilewanted/usersync.go +++ /dev/null @@ -1,12 +0,0 @@ -package smilewanted - -import ( - "text/template" - - "github.com/prebid/prebid-server/adapters" - "github.com/prebid/prebid-server/usersync" -) - -func NewSmileWantedSyncer(temp *template.Template) usersync.Usersyncer { - return adapters.NewSyncer("smilewanted", temp, adapters.SyncTypeRedirect) -} diff --git a/adapters/smilewanted/usersync_test.go b/adapters/smilewanted/usersync_test.go deleted file mode 100644 index 497e5061554..00000000000 --- a/adapters/smilewanted/usersync_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package smilewanted - -import ( - "testing" - "text/template" - - "github.com/prebid/prebid-server/privacy" - "github.com/prebid/prebid-server/privacy/ccpa" - "github.com/prebid/prebid-server/privacy/gdpr" - "github.com/stretchr/testify/assert" -) - -func TestSmileWantedSyncer(t *testing.T) { - syncURL := "//csync.smilewanted.com/getuid?source=prebid-server&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dsmilewanted%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%24UID" - syncURLTemplate := template.Must( - template.New("sync-template").Parse(syncURL), - ) - - syncer := NewSmileWantedSyncer(syncURLTemplate) - syncInfo, err := syncer.GetUsersyncInfo(privacy.Policies{ - GDPR: gdpr.Policy{ - Signal: "1", - Consent: "COyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA", - }, - CCPA: ccpa.Policy{ - Consent: "1YNN", - }, - }) - - assert.NoError(t, err) - assert.Equal(t, "//csync.smilewanted.com/getuid?source=prebid-server&gdpr=1&gdpr_consent=COyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA&us_privacy=1YNN&redirect=https%3A%2F%2Flocalhost%2Fsetuid%3Fbidder%3Dsmilewanted%26gdpr%3D1%26gdpr_consent%3DCOyASAoOyASAoAfAAAENAfCAAAAAAAAAAAAAAAAAAAAA%26uid%3D%24UID", syncInfo.URL) - assert.Equal(t, "redirect", syncInfo.Type) - assert.Equal(t, false, syncInfo.SupportCORS) -} From 63ca092cf15672f39751a502e7f656bf06256635 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 7 Jul 2021 12:52:14 -0400 Subject: [PATCH 28/50] Remove TODO File --- metrics/config/todo | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 metrics/config/todo diff --git a/metrics/config/todo b/metrics/config/todo deleted file mode 100644 index 8908b5f4041..00000000000 --- a/metrics/config/todo +++ /dev/null @@ -1,14 +0,0 @@ -todo - - rename setuid metrics to match new cookiesync metric names - - GoMetrics - - prometheus - - - add metric tests - - GoMetrics - - prometheus - - - override bidder info tests - - - migrate all adapter user sync configs - - - remove todo file \ No newline at end of file From 5cb47e47af70e4ac1d6a83f7578c92b524cf6333 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 8 Jul 2021 16:58:24 -0400 Subject: [PATCH 29/50] Fix Regex + Add Tests --- usersync/syncer.go | 10 +++++----- usersync/syncer_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/usersync/syncer.go b/usersync/syncer.go index a2734273e81..133e06c57e0 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -139,11 +139,11 @@ func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { // macro substitution regex var ( - macroRegexExternalHost = regexp.MustCompile(`{{\s*.ExternalURL\s*}}`) - macroRegexSyncerKey = regexp.MustCompile(`{{\s*.SyncerKey\s*}}`) - macroRegexSyncType = regexp.MustCompile(`{{\s*.SyncType\s*}}`) - macroRegexUserMacro = regexp.MustCompile(`{{\s*.UserMacro\s*}}`) - macroRegexRedirect = regexp.MustCompile(`{{\s*.RedirectURL\s*}}`) + macroRegexExternalHost = regexp.MustCompile(`{{\s*\.ExternalURL\s*}}`) + macroRegexSyncerKey = regexp.MustCompile(`{{\s*\.SyncerKey\s*}}`) + macroRegexSyncType = regexp.MustCompile(`{{\s*\.SyncType\s*}}`) + macroRegexUserMacro = regexp.MustCompile(`{{\s*\.UserMacro\s*}}`) + macroRegexRedirect = regexp.MustCompile(`{{\s*\.RedirectURL\s*}}`) macroRegex = regexp.MustCompile(`{{.*?}}`) ) diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index c43e78a09c6..386e90b1350 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -285,6 +285,47 @@ func TestBuildTemplate(t *testing.T) { }, expectedError: "template: anykey_usersync_url:1: function \"malformed\" not defined", }, + + // The following tests protect against the . literal character vs . character class in regex. + { + description: "Invalid Macro - Redirect URL", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{xRedirectURL}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"xRedirectURL\" not defined", + }, + { + description: "Invalid Macro - External URL", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{xExternalURL}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"xExternalURL\" not defined", + }, + { + description: "Invalid Macro - Syncer Key", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{xSyncerKey}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"xSyncerKey\" not defined", + }, + { + description: "Invalid Macro - Sync Type", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{xSyncType}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"xSyncType\" not defined", + }, + { + description: "Invalid Macro - User Macro", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{xUserMacro}}", + }, + expectedError: "template: anykey_usersync_url:1: function \"xUserMacro\" not defined", + }, } for _, test := range testCases { From 762cd313e81b3ae22ad438bde84fb2f5d46e7eb2 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 3 Aug 2021 01:18:35 -0400 Subject: [PATCH 30/50] Update Comments. Provide Legacy Config Warning. --- router/router.go | 10 +++++++++- usersync/bidderchooser.go | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/router/router.go b/router/router.go index a8e8d7d94e5..145e576db34 100644 --- a/router/router.go +++ b/router/router.go @@ -335,7 +335,10 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg if adapterCfg, exists := adaptersCfg[bidderName]; exists { bidderInfo.Syncer = adapterCfg.Syncer.Override(bidderInfo.Syncer) - // comptability with legacy settings (if possible unambiguously) + // validate and try to apply the legacy usersync_url configuration in attempt to provide + // an easier upgrade path. be warned, this will break if the bidder adds a second syncer + // type and will eventually be removed after we've given hosts enough time to upgrade to + // the new config. if adapterCfg.UserSyncURL != "" { if bidderInfo.Syncer == nil { return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync", bidderName) @@ -355,9 +358,14 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define user sync endpoints", bidderName) } + // if the bidder defines both an iframe and redirect endpoint, we can't be sure which config value to + // override, and it wouldn't be both. this is a fatal configuration error. if endpointsCount > 1 { return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder defines multiple user sync endpoints", bidderName) } + + // provide a warning that this compatibility layer is temporary + glog.Warningf("legacy usersync_url setting for bidder %s will be removed in a future version of Prebid Server. please update to the latest user sync config values", bidderName) } bidderInfos[bidderName] = bidderInfo diff --git a/usersync/bidderchooser.go b/usersync/bidderchooser.go index 5e5ea81edf3..3b9449fd981 100644 --- a/usersync/bidderchooser.go +++ b/usersync/bidderchooser.go @@ -24,6 +24,10 @@ func (c standardBidderChooser) choose(requested, available []string, cooperative } func (c standardBidderChooser) chooseCooperative(requested, available []string, priorityGroups [][]string) []string { + // allocate enough memory for the slice to try to avoid re-allocation. the 50% overhead is a guess + // at a satisfactory value. since all available bidders are included in the slice, along with + // requested and prioritized bidders, expect there to be be many duplicates. the duplicate are + // resolved in the upstream chooser algorithm. biddersCapacity := int(float64(len(available)) * 1.5) bidders := make([]string, 0, biddersCapacity) From bfdb132a3f3c12135f0ccf26498bbfefd49438ea Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 3 Aug 2021 01:21:49 -0400 Subject: [PATCH 31/50] Add Suggested Test --- config/bidderinfo_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/bidderinfo_test.go b/config/bidderinfo_test.go index 4772574b345..19abdc43f59 100644 --- a/config/bidderinfo_test.go +++ b/config/bidderinfo_test.go @@ -232,6 +232,12 @@ func TestSyncerOverride(t *testing.T) { givenOverride: &Syncer{SupportCORS: &falseValue}, expected: &Syncer{SupportCORS: &falseValue}, }, + { + description: "Override Partial - Other Fields Untouched", + givenOriginal: &Syncer{Key: "originalKey", Default: "originalDefault"}, + givenOverride: &Syncer{Default: "overrideDefault"}, + expected: &Syncer{Key: "originalKey", Default: "overrideDefault"}, + }, } for _, test := range testCases { From 89e54bc6e4ef2414aae23552c31e987881485c7e Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 4 Aug 2021 11:22:54 -0400 Subject: [PATCH 32/50] Separated BidderFilter --- endpoints/cookie_sync.go | 14 +++--- endpoints/cookie_sync_test.go | 88 +++++++++++++++++------------------ usersync/bidderfilter.go | 57 +++++++++++++---------- usersync/bidderfilter_test.go | 8 ++-- usersync/chooser_test.go | 8 ++-- usersync/synctype_test.go | 4 +- 6 files changed, 94 insertions(+), 85 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index e04d90ba5c4..1e96bec7f0c 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -29,7 +29,7 @@ var ( errCookieSyncInvalidBiddersType = errors.New("invalid bidders type. must either be a string '*' or a string array of bidders") ) -var cookieSyncBidderFilterAllowAll = usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude) +var cookieSyncBidderFilterAllowAll = usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude) func NewCookieSyncEndpoint( syncers map[string]usersync.Syncer, @@ -199,27 +199,27 @@ func parseBidderFilter(filter *cookieSyncRequestFilter) (usersync.BidderFilter, case "exclude": mode = usersync.BidderFilterModeExclude default: - return usersync.BidderFilter{}, fmt.Errorf("invalid filter value '%s'. must be either 'include' or 'exclude'", filter.Mode) + return nil, fmt.Errorf("invalid filter value '%s'. must be either 'include' or 'exclude'", filter.Mode) } switch v := filter.Bidders.(type) { case string: if v == "*" { - return usersync.NewBidderFilterForAll(mode), nil + return usersync.NewUniformBidderFilter(mode), nil } - return usersync.BidderFilter{}, fmt.Errorf("invalid bidders value `%s`. must either be '*' or a string array", v) + return nil, fmt.Errorf("invalid bidders value `%s`. must either be '*' or a string array", v) case []interface{}: bidders := make([]string, len(v)) for i, x := range v { if bidder, ok := x.(string); ok { bidders[i] = bidder } else { - return usersync.BidderFilter{}, errCookieSyncInvalidBiddersType + return nil, errCookieSyncInvalidBiddersType } } - return usersync.NewBidderFilter(bidders, mode), nil + return usersync.NewSpecificBidderFilter(bidders, mode), nil default: - return usersync.BidderFilter{}, errCookieSyncInvalidBiddersType + return nil, errCookieSyncInvalidBiddersType } } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 6ab507a5968..bf1c4c63c65 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -313,8 +313,8 @@ func TestCookieSyncParseRequest(t *testing.T) { ccpaParsedPolicy: expectedCCPAParsedPolicy, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewSpecificBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), }, }, }, @@ -357,8 +357,8 @@ func TestCookieSyncParseRequest(t *testing.T) { ccpaParsedPolicy: expectedCCPAParsedPolicy, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -373,8 +373,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -399,8 +399,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -425,8 +425,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -451,8 +451,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -477,8 +477,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -503,8 +503,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -529,8 +529,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -545,8 +545,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -564,8 +564,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -603,8 +603,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalNo, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -628,8 +628,8 @@ func TestCookieSyncParseRequest(t *testing.T) { gdprSignal: gdpr.SignalAmbiguous, }, SyncTypeFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, }, @@ -684,16 +684,16 @@ func TestParseTypeFilter(t *testing.T) { description: "Nil", given: nil, expectedFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, { description: "Nil Object", given: &cookieSyncRequestFilterSettings{}, expectedFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, { @@ -702,8 +702,8 @@ func TestParseTypeFilter(t *testing.T) { IFrame: &cookieSyncRequestFilter{Bidders: []interface{}{"a"}, Mode: "exclude"}, }, expectedFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewSpecificBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, }, { @@ -712,8 +712,8 @@ func TestParseTypeFilter(t *testing.T) { Redirect: &cookieSyncRequestFilter{Bidders: []interface{}{"b"}, Mode: "exclude"}, }, expectedFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), - Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), + Redirect: usersync.NewSpecificBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), }, }, { @@ -723,8 +723,8 @@ func TestParseTypeFilter(t *testing.T) { Redirect: &cookieSyncRequestFilter{Bidders: []interface{}{"b"}, Mode: "exclude"}, }, expectedFilter: usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), - Redirect: usersync.NewBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), + IFrame: usersync.NewSpecificBidderFilter([]string{"a"}, usersync.BidderFilterModeExclude), + Redirect: usersync.NewSpecificBidderFilter([]string{"b"}, usersync.BidderFilterModeExclude), }, }, { @@ -768,17 +768,17 @@ func TestParseBidderFilter(t *testing.T) { { description: "Nil", given: nil, - expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + expectedFilter: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, { description: "All Bidders - Include", given: &cookieSyncRequestFilter{Bidders: "*", Mode: "include"}, - expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + expectedFilter: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), }, { description: "All Bidders - Exclude", given: &cookieSyncRequestFilter{Bidders: "*", Mode: "exclude"}, - expectedFilter: usersync.NewBidderFilterForAll(usersync.BidderFilterModeExclude), + expectedFilter: usersync.NewUniformBidderFilter(usersync.BidderFilterModeExclude), }, { description: "All Bidders - Invalid Mode", @@ -793,12 +793,12 @@ func TestParseBidderFilter(t *testing.T) { { description: "Specific Bidders - Include", given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", "b"}, Mode: "include"}, - expectedFilter: usersync.NewBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeInclude), + expectedFilter: usersync.NewSpecificBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeInclude), }, { description: "Specific Bidders - Exclude", given: &cookieSyncRequestFilter{Bidders: []interface{}{"a", "b"}, Mode: "exclude"}, - expectedFilter: usersync.NewBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeExclude), + expectedFilter: usersync.NewSpecificBidderFilter([]string{"a", "b"}, usersync.BidderFilterModeExclude), }, { description: "Specific Bidders - Invalid Mode", @@ -825,7 +825,7 @@ func TestParseBidderFilter(t *testing.T) { assert.Equal(t, test.expectedFilter, result, test.description+":result") } else { assert.EqualError(t, err, test.expectedError, test.description+":err") - assert.Empty(t, result, test.description+":result") + assert.Nil(t, result, test.description+":result") } } } @@ -922,8 +922,8 @@ func TestCookieSyncWriteBidderMetrics(t *testing.T) { func TestCookieSyncHandleResponse(t *testing.T) { syncTypeFilter := usersync.SyncTypeFilter{ - IFrame: usersync.NewBidderFilterForAll(usersync.BidderFilterModeExclude), - Redirect: usersync.NewBidderFilterForAll(usersync.BidderFilterModeInclude), + IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeExclude), + Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude), } syncTypeExpected := []usersync.SyncType{usersync.SyncTypeRedirect} privacyPolicies := privacy.Policies{CCPA: ccpa.Policy{Consent: "anyConsent"}} diff --git a/usersync/bidderfilter.go b/usersync/bidderfilter.go index 61a5ec97528..2d7d16ffe2b 100644 --- a/usersync/bidderfilter.go +++ b/usersync/bidderfilter.go @@ -1,5 +1,12 @@ package usersync +// BidderFilter determines if a bidder has permission to perform a user sync activity. +type BidderFilter interface { + // Allowed returns true if the filter determines the bidder has permission and false if either + // the bidder does not have permission or if the filter has an invalid mode. + Allowed(bidder string) bool +} + // BidderFilterMode represents the inclusion mode of a BidderFilter. type BidderFilterMode int @@ -8,46 +15,48 @@ const ( BidderFilterModeExclude ) -// BidderFilter determines if a bidder has permission to perform a user sync activity. -type BidderFilter struct { - biddersAll bool +// SpecificBidderFilter implements the BidderFilter which applies the same mode for a list of bidders. +type SpecificBidderFilter struct { biddersLookup map[string]struct{} mode BidderFilterMode } -// Allowed returns true if the filter determines the bidder has permission and false if the bidder -// does not have permission or if the BidderFilter is set to an unsupported BidderFilterMode. -func (t BidderFilter) Allowed(bidder string) bool { - switch t.mode { +// Allowed returns true if the bidder is specified and the mode is include or if the bidder is not specified +// and the mode is exclude and returns false in the opposite cases or when the mode is invalid. +func (f SpecificBidderFilter) Allowed(bidder string) bool { + _, exists := f.biddersLookup[bidder] + + switch f.mode { case BidderFilterModeInclude: - return t.bidderIncluded(bidder) + return exists case BidderFilterModeExclude: - return !t.bidderIncluded(bidder) + return !exists default: return false } } -func (t BidderFilter) bidderIncluded(bidder string) bool { - if t.biddersAll { - return true - } - - _, exists := t.biddersLookup[bidder] - return exists -} - -// NewBidderFilter returns a new BidderFilter which applies the same mode for a list of specific bidders. -func NewBidderFilter(bidders []string, mode BidderFilterMode) BidderFilter { +// NewSpecificBidderFilter returns a new instance of the NewSpecificBidderFilter filter. +func NewSpecificBidderFilter(bidders []string, mode BidderFilterMode) BidderFilter { biddersLookup := make(map[string]struct{}, len(bidders)) for _, bidder := range bidders { biddersLookup[bidder] = struct{}{} } - return BidderFilter{biddersLookup: biddersLookup, mode: mode} + return SpecificBidderFilter{biddersLookup: biddersLookup, mode: mode} +} + +// UniformBidderFilter implements the BidderFilter interface which applies the same mode for all bidders. +type UniformBidderFilter struct { + mode BidderFilterMode +} + +// Allowed returns true if the mode is include and false if the mode is either exclude or invalid. +func (f UniformBidderFilter) Allowed(bidder string) bool { + return f.mode == BidderFilterModeInclude } -// NewBidderFilterForAll returns a new BidderFilter which applies the same mode for all bidders. -func NewBidderFilterForAll(mode BidderFilterMode) BidderFilter { - return BidderFilter{biddersAll: true, mode: mode} +// NewUniformBidderFilter returns a new instance of the UniformBidderFilter filter. +func NewUniformBidderFilter(mode BidderFilterMode) BidderFilter { + return UniformBidderFilter{mode: mode} } diff --git a/usersync/bidderfilter_test.go b/usersync/bidderfilter_test.go index afed18c44ae..ddc757339a2 100644 --- a/usersync/bidderfilter_test.go +++ b/usersync/bidderfilter_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestNewBidderFilter(t *testing.T) { +func TestSpecificBidderFilter(t *testing.T) { bidder := "a" testCases := []struct { @@ -72,12 +72,12 @@ func TestNewBidderFilter(t *testing.T) { } for _, test := range testCases { - filter := NewBidderFilter(test.bidders, test.mode) + filter := NewSpecificBidderFilter(test.bidders, test.mode) assert.Equal(t, test.expected, filter.Allowed(bidder), test.description) } } -func TestBidderFilterForAll(t *testing.T) { +func TestUniformBidderFilter(t *testing.T) { bidder := "a" testCases := []struct { @@ -98,7 +98,7 @@ func TestBidderFilterForAll(t *testing.T) { } for _, test := range testCases { - filter := NewBidderFilterForAll(test.mode) + filter := NewUniformBidderFilter(test.mode) assert.Equal(t, test.expected, filter.Allowed(bidder), test.description) } } diff --git a/usersync/chooser_test.go b/usersync/chooser_test.go index 6274be40f30..8a129bd4cc8 100644 --- a/usersync/chooser_test.go +++ b/usersync/chooser_test.go @@ -51,8 +51,8 @@ func TestChooserChoose(t *testing.T) { syncerChoiceA := SyncerChoice{Bidder: "a", Syncer: fakeSyncerA} syncerChoiceB := SyncerChoice{Bidder: "b", Syncer: fakeSyncerB} syncTypeFilter := SyncTypeFilter{ - IFrame: NewBidderFilterForAll(BidderFilterModeInclude), - Redirect: NewBidderFilterForAll(BidderFilterModeExclude)} + IFrame: NewUniformBidderFilter(BidderFilterModeInclude), + Redirect: NewUniformBidderFilter(BidderFilterModeExclude)} cooperativeConfig := Cooperative{Enabled: true} @@ -234,8 +234,8 @@ func TestChooserEvaluate(t *testing.T) { fakeSyncerB := fakeSyncer{key: "keyB", supportsIFrame: false} bidderSyncerLookup := map[string]Syncer{"a": fakeSyncerA, "b": fakeSyncerB} syncTypeFilter := SyncTypeFilter{ - IFrame: NewBidderFilterForAll(BidderFilterModeInclude), - Redirect: NewBidderFilterForAll(BidderFilterModeExclude)} + IFrame: NewUniformBidderFilter(BidderFilterModeInclude), + Redirect: NewUniformBidderFilter(BidderFilterModeExclude)} cookieNeedsSync := Cookie{} cookieAlreadyHasSyncForA := Cookie{uids: map[string]uidWithExpiry{"keyA": {Expires: time.Now().Add(time.Duration(24) * time.Hour)}}} diff --git a/usersync/synctype_test.go b/usersync/synctype_test.go index 26161295dd1..35a241f182b 100644 --- a/usersync/synctype_test.go +++ b/usersync/synctype_test.go @@ -9,8 +9,8 @@ import ( func TestSyncTypeFilter(t *testing.T) { bidder := "foo" - bidderFilterAllowed := NewBidderFilterForAll(BidderFilterModeInclude) - bidderFilterNotAllowed := NewBidderFilterForAll(BidderFilterModeExclude) + bidderFilterAllowed := NewUniformBidderFilter(BidderFilterModeInclude) + bidderFilterNotAllowed := NewUniformBidderFilter(BidderFilterModeExclude) testCases := []struct { description string From 08334e5a97fd9a5be47ca6481f68db2105614481 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 4 Aug 2021 11:45:37 -0400 Subject: [PATCH 33/50] PR Feedback --- config/bidderinfo.go | 3 +-- endpoints/setuid.go | 10 ++++------ usersync/bidderchooser.go | 8 ++++---- usersync/bidderchooser_test.go | 12 ++++++++++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/config/bidderinfo.go b/config/bidderinfo.go index 29be6583155..e6fecb7bffa 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -48,8 +48,7 @@ type DebugInfo struct { // so it needs to have both yaml and mapstructure mappings. type Syncer struct { // Key is used as the record key for the user sync cookie. We recommend using the bidder name - // as the key for consistency, but that is not enforced as a requirement. Each bidder must - // have a unique key. + // as the key for consistency, but that is not enforced as a requirement. Key string `yaml:"key" mapstructure:"key"` // Default identifies which endpoint is preferred if both are allowed by the publisher. This is diff --git a/endpoints/setuid.go b/endpoints/setuid.go index ad65815d5e0..0cdcca777f9 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -87,12 +87,10 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer metricsEngine.RecordSetUid(metrics.SetUidOK) metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidCleared) so.Success = true - } else { - if err = pc.TrySync(syncerKey, uid); err == nil { - metricsEngine.RecordSetUid(metrics.SetUidOK) - metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidOK) - so.Success = true - } + } else if err = pc.TrySync(syncerKey, uid); err == nil { + metricsEngine.RecordSetUid(metrics.SetUidOK) + metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidOK) + so.Success = true } setSiteCookie := siteCookieCheck(r.UserAgent()) diff --git a/usersync/bidderchooser.go b/usersync/bidderchooser.go index 3b9449fd981..1b01e5116e9 100644 --- a/usersync/bidderchooser.go +++ b/usersync/bidderchooser.go @@ -12,14 +12,14 @@ type standardBidderChooser struct { } func (c standardBidderChooser) choose(requested, available []string, cooperative Cooperative) []string { - if len(requested) == 0 { - return c.shuffledCopy(available) - } - if cooperative.Enabled { return c.chooseCooperative(requested, available, cooperative.PriorityGroups) } + if len(requested) == 0 { + return c.shuffledCopy(available) + } + return c.shuffledCopy(requested) } diff --git a/usersync/bidderchooser_test.go b/usersync/bidderchooser_test.go index 29c2d56b07e..fe7d296f1f0 100644 --- a/usersync/bidderchooser_test.go +++ b/usersync/bidderchooser_test.go @@ -44,6 +44,18 @@ func TestBidderChooserChoose(t *testing.T) { givenCooperative: Cooperative{Enabled: false}, expected: []string{"r2", "r1"}, }, + { + description: "Coop - Nil", + givenRequested: nil, + givenCooperative: Cooperative{Enabled: true, PriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}}, + expected: []string{"pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, + }, + { + description: "Coop - Empty", + givenRequested: nil, + givenCooperative: Cooperative{Enabled: true, PriorityGroups: [][]string{{"pr1A", "pr1B"}, {"pr2A", "pr2B"}}}, + expected: []string{"pr1B", "pr1A", "pr2B", "pr2A", "a2", "a1"}, + }, { description: "Coop - Integration Test", givenRequested: []string{"r1", "r2"}, From 0e10d27fa46d6eeaac8636d6588d6aa6441b70af Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 5 Aug 2021 15:29:19 -0400 Subject: [PATCH 34/50] Tweak Handlng Of `f` Query Param --- endpoints/setuid.go | 14 +++++++++----- endpoints/setuid_test.go | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 0cdcca777f9..6e0ff811bb8 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -124,24 +124,28 @@ func getSyncerKey(query url.Values, syncers map[string]usersync.Syncer) (string, return key, nil } +// getResponseFormat reads the format query parameter or falls back to the syncer's default. +// Returns either "b" (iframe), "i" (redirect), or an empty string "" (legacy behavior of an +// empty response body with no content type). func getResponseFormat(query url.Values, syncer usersync.Syncer) (string, error) { - format := query.Get("f") + format, formatProvided := query["f"] + formatEmpty := len(format) == 0 || format[0] == "" - if format == "" { + if !formatProvided || formatEmpty { switch syncer.DefaultSyncType() { case usersync.SyncTypeIFrame: return "b", nil case usersync.SyncTypeRedirect: return "i", nil default: - return "", errors.New("invalid default sync type") + return "", nil } } - if !strings.EqualFold(format, "b") && !strings.EqualFold(format, "i") { + if !strings.EqualFold(format[0], "b") && !strings.EqualFold(format[0], "i") { return "", errors.New(`"f" query param is invalid. must be "b" or "i"`) } - return strings.ToLower(format), nil + return strings.ToLower(format[0]), nil } // siteCookieCheck scans the input User Agent string to check if browser is Chrome and browser version is greater than the minimum version for adding the SameSite cookie attribute diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 30c819babe5..56f538c1e8f 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -419,10 +419,10 @@ func TestGetResponseFormat(t *testing.T) { description: "parameter not provided, use default sync type redirect", }, { - urlValues: url.Values{}, - syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncType("invalid")}, - expectedError: "invalid default sync type", - description: "parameter not provided, use default sync type invalid", + urlValues: url.Values{}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncType("invalid")}, + expectedFormat: "", + description: "parameter not provided, default sync type is invalid", }, { urlValues: url.Values{"f": []string{"b"}}, @@ -454,6 +454,18 @@ func TestGetResponseFormat(t *testing.T) { expectedError: `"f" query param is invalid. must be "b" or "i"`, description: "parameter given invalid", }, + { + urlValues: url.Values{"f": []string{}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeRedirect}, + expectedFormat: "i", + description: "parameter given is empty (by slice), use default sync type redirect", + }, + { + urlValues: url.Values{"f": []string{""}}, + syncer: fakeSyncer{key: "a", defaultSyncType: usersync.SyncTypeRedirect}, + expectedFormat: "i", + description: "parameter given is empty (by empty item), use default sync type redirect", + }, } for _, test := range testCases { From 83f0f487e41b42f67ed1fe8db82dae8553585c58 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 5 Aug 2021 15:55:09 -0400 Subject: [PATCH 35/50] Treat Macro-Like User Sync Config As Literals --- usersync/syncer.go | 2 +- usersync/syncer_test.go | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/usersync/syncer.go b/usersync/syncer.go index 133e06c57e0..bd23e621eae 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -144,7 +144,7 @@ var ( macroRegexSyncType = regexp.MustCompile(`{{\s*\.SyncType\s*}}`) macroRegexUserMacro = regexp.MustCompile(`{{\s*\.UserMacro\s*}}`) macroRegexRedirect = regexp.MustCompile(`{{\s*\.RedirectURL\s*}}`) - macroRegex = regexp.MustCompile(`{{.*?}}`) + macroRegex = regexp.MustCompile(`{{\s*\..*?\s*}}`) ) func buildTemplate(key, syncTypeValue string, hostConfig config.UserSync, syncerEndpoint config.SyncerEndpoint) (*template.Template, error) { diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 386e90b1350..8c7fe4a5f45 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -285,8 +285,17 @@ func TestBuildTemplate(t *testing.T) { }, expectedError: "template: anykey_usersync_url:1: function \"malformed\" not defined", }, + { + description: "User Macro Is Go Template Macro-Like", + givenSyncerEndpoint: config.SyncerEndpoint{ + URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", + RedirectURL: "{{.ExternalURL}}/setuid?bidder={{.SyncerKey}}&f={{.SyncType}}&gdpr={{.GDPR}}&uid={{.UserMacro}}", + UserMacro: "{{UID}}", + }, + expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsetuid%3Fbidder%3DanyKey%26f%3Dx%26gdpr%3DA%26uid%3D%7B%7BUID%7D%7D", + }, - // The following tests protect against the . literal character vs . character class in regex. + // The following tests protect against the "." literal character vs the "."" character class in regex. { description: "Invalid Macro - Redirect URL", givenSyncerEndpoint: config.SyncerEndpoint{ @@ -300,7 +309,7 @@ func TestBuildTemplate(t *testing.T) { URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xExternalURL}}", }, - expectedError: "template: anykey_usersync_url:1: function \"xExternalURL\" not defined", + expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxExternalURL%7D%7D", }, { description: "Invalid Macro - Syncer Key", @@ -308,7 +317,7 @@ func TestBuildTemplate(t *testing.T) { URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xSyncerKey}}", }, - expectedError: "template: anykey_usersync_url:1: function \"xSyncerKey\" not defined", + expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxSyncerKey%7D%7D", }, { description: "Invalid Macro - Sync Type", @@ -316,7 +325,7 @@ func TestBuildTemplate(t *testing.T) { URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xSyncType}}", }, - expectedError: "template: anykey_usersync_url:1: function \"xSyncType\" not defined", + expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxSyncType%7D%7D", }, { description: "Invalid Macro - User Macro", @@ -324,7 +333,7 @@ func TestBuildTemplate(t *testing.T) { URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xUserMacro}}", }, - expectedError: "template: anykey_usersync_url:1: function \"xUserMacro\" not defined", + expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxUserMacro%7D%7D", }, } @@ -389,6 +398,11 @@ func TestEscapeTemplate(t *testing.T) { given: " &a {{ .Macro1 }} /b ", expected: "+%26a+{{ .Macro1 }}+%2Fb+", }, + { + description: "Double Curly Braces, But Not Macro", + given: "{{Macro}}", + expected: "%7B%7BMacro%7D%7D", + }, } for _, test := range testCases { From ed30a5566a2f69665742ebb14a49fcad20fb492e Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 5 Aug 2021 16:12:51 -0400 Subject: [PATCH 36/50] Fix User Sync CCPA Enforcement --- endpoints/cookie_sync.go | 3 ++- endpoints/cookie_sync_test.go | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 1e96bec7f0c..616f19a5e2e 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -368,5 +368,6 @@ func (p usersyncPrivacy) GDPRAllowsBidderSync(bidder string) bool { } func (p usersyncPrivacy) CCPAAllowsBidderSync(bidder string) bool { - return p.ccpaParsedPolicy.ShouldEnforce(bidder) + enforce := p.ccpaParsedPolicy.CanEnforce() && p.ccpaParsedPolicy.ShouldEnforce(bidder) + return !enforce } diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index bf1c4c63c65..cfcf71cdb98 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -1214,15 +1214,20 @@ func TestUsersyncPrivacyCCPAAllowsBidderSync(t *testing.T) { expected bool }{ { - description: "Allowed", - givenConsent: "1YYY", + description: "Allowed - No Opt-Out", + givenConsent: "1NNN", expected: true, }, { - description: "Not Allowed", - givenConsent: "1NNN", + description: "Not Allowed - Opt-Out", + givenConsent: "1NYN", expected: false, }, + { + description: "Not Specified", + givenConsent: "", + expected: true, + }, } for _, test := range testCases { From f9bf1c514cfe844f6c09063e0628bec810e4f020 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 5 Aug 2021 17:32:43 -0400 Subject: [PATCH 37/50] Fix SetUID Syncer Key Handling --- endpoints/setuid.go | 42 ++-- endpoints/setuid_test.go | 462 ++++++++++++++++++++------------------- 2 files changed, 261 insertions(+), 243 deletions(-) diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 6e0ff811bb8..3f774579c33 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -29,6 +29,13 @@ const ( func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour + // convert map of syncers by bidder to map of syncers by key + // - its safe to assume that if multiple bidders map to the same key, the syncers are interchangeable. + syncersByKey := make(map[string]usersync.Syncer, len(syncers)) + for _, v := range syncers { + syncersByKey[v.Key()] = v + } + return httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { so := analytics.SetUIDObject{ Status: http.StatusOK, @@ -47,7 +54,7 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer query := r.URL.Query() - syncerKey, err := getSyncerKey(query, syncers) + syncer, err := getSyncer(query, syncersByKey) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -55,9 +62,9 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer so.Status = http.StatusBadRequest return } - so.Bidder = syncerKey + so.Bidder = syncer.Key() - responseFormat, err := getResponseFormat(query, syncers[syncerKey]) + responseFormat, err := getResponseFormat(query, syncer) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) @@ -83,13 +90,13 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer so.UID = uid if uid == "" { - pc.Unsync(syncerKey) + pc.Unsync(syncer.Key()) metricsEngine.RecordSetUid(metrics.SetUidOK) - metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidCleared) + metricsEngine.RecordSyncerSet(syncer.Key(), metrics.SyncerSetUidCleared) so.Success = true - } else if err = pc.TrySync(syncerKey, uid); err == nil { + } else if err = pc.TrySync(syncer.Key(), uid); err == nil { metricsEngine.RecordSetUid(metrics.SetUidOK) - metricsEngine.RecordSyncerSet(syncerKey, metrics.SyncerSetUidOK) + metricsEngine.RecordSyncerSet(syncer.Key(), metrics.SyncerSetUidOK) so.Success = true } @@ -110,18 +117,19 @@ func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer }) } -func getSyncerKey(query url.Values, syncers map[string]usersync.Syncer) (string, error) { +func getSyncer(query url.Values, syncersByKey map[string]usersync.Syncer) (usersync.Syncer, error) { key := query.Get("bidder") if key == "" { - return "", errors.New(`"bidder" query param is required`) + return nil, errors.New(`"bidder" query param is required`) } - if _, ok := syncers[key]; !ok { - return "", errors.New("The bidder name provided is not supported by Prebid Server") + syncer, syncerExists := syncersByKey[key] + if !syncerExists { + return nil, errors.New("The bidder name provided is not supported by Prebid Server") } - return key, nil + return syncer, nil } // getResponseFormat reads the format query parameter or falls back to the syncer's default. @@ -159,6 +167,7 @@ func siteCookieCheck(ua string) bool { } else if criOSIndex != -1 { result = checkChromeBrowserVersion(ua, criOSIndex, chromeiOSStrLen) } + return result } @@ -177,7 +186,6 @@ func checkChromeBrowserVersion(ua string, index int, chromeStrLength int) bool { } func preventSyncsGDPR(gdprEnabled string, gdprConsent string, perms gdpr.Permissions) (shouldReturn bool, status int, body string) { - if gdprEnabled != "" && gdprEnabled != "0" && gdprEnabled != "1" { return true, http.StatusBadRequest, "the gdpr query param must be either 0 or 1. You gave " + gdprEnabled } @@ -198,10 +206,10 @@ func preventSyncsGDPR(gdprEnabled string, gdprConsent string, perms gdpr.Permiss return true, http.StatusBadRequest, "gdpr_consent was invalid. " + err.Error() } - // We can't really distinguish between requests that are for a new version of the global vendor list, and - // ones which are simply malformed (version number is much too large). - // Since we try to fetch new versions as requests come in for them, PBS *should* self-correct - // rather quickly, meaning that most of these will be malformed strings. + // We can't distinguish between requests for a new version of the global vendor list, and requests + // which are malformed (version number is much too large). Since we try to fetch new versions as we + // receive requests, PBS *should* self-correct quickly, allowing us to assume most of the errors + // caught here will be malformed strings. return true, http.StatusBadRequest, "No global vendor list was available to interpret this consent string. If this is a new, valid version, it should become available soon." } diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 56f538c1e8f..976946078c7 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -24,138 +24,148 @@ import ( func TestSetUIDEndpoint(t *testing.T) { testCases := []struct { - uri string - syncers []string - existingSyncs map[string]string - gdprAllowsHostCookies bool - gdprReturnsError bool - gdprMalformed bool - expectedSyncs map[string]string - expectedBody string - expectedStatusCode int - expectedHeaders map[string]string - description string + uri string + syncersBidderNameToKey map[string]string + existingSyncs map[string]string + gdprAllowsHostCookies bool + gdprReturnsError bool + gdprMalformed bool + expectedSyncs map[string]string + expectedBody string + expectedStatusCode int + expectedHeaders map[string]string + description string }{ { - uri: "/setuid?bidder=pubmatic&uid=123", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Set uid for valid bidder", - }, - { - uri: "/setuid?bidder=unsupported-bidder&uid=123", - syncers: []string{}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: "The bidder name provided is not supported by Prebid Server", - description: "Don't set uid for an unsupported bidder", - }, - { - uri: "/setuid?bidder=&uid=123", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: `"bidder" query param is required`, - description: "Don't set uid for an empty bidder", - }, - { - uri: "/setuid?bidder=unsupported-bidder&uid=123", - syncers: []string{}, - existingSyncs: map[string]string{"pubmatic": "1234"}, - gdprAllowsHostCookies: true, - expectedSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: "The bidder name provided is not supported by Prebid Server", + uri: "/setuid?bidder=pubmatic&uid=123", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Set uid for valid bidder", + }, + { + uri: "/setuid?bidder=adnxs&uid=123", + syncersBidderNameToKey: map[string]string{"appnexus": "adnxs"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"adnxs": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Set uid for valid bidder with different key", + }, + { + uri: "/setuid?bidder=unsupported-bidder&uid=123", + syncersBidderNameToKey: map[string]string{}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "The bidder name provided is not supported by Prebid Server", + description: "Don't set uid for an unsupported bidder", + }, + { + uri: "/setuid?bidder=&uid=123", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, + description: "Don't set uid for an empty bidder", + }, + { + uri: "/setuid?bidder=unsupported-bidder&uid=123", + syncersBidderNameToKey: map[string]string{}, + existingSyncs: map[string]string{"pubmatic": "1234"}, + gdprAllowsHostCookies: true, + expectedSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "The bidder name provided is not supported by Prebid Server", description: "No need to set existing syncs back in response for a request " + "to set uid for an unsupported bidder", }, { - uri: "/setuid?bidder=&uid=123", - syncers: []string{"pubmatic"}, - existingSyncs: map[string]string{"pubmatic": "1234"}, - gdprAllowsHostCookies: true, - expectedSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: `"bidder" query param is required`, + uri: "/setuid?bidder=&uid=123", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: map[string]string{"pubmatic": "1234"}, + gdprAllowsHostCookies: true, + expectedSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, description: "No need to set existing syncs back in response for a request " + "to set uid for an empty bidder", }, { - uri: "/setuid?bidder=pubmatic", - syncers: []string{"pubmatic"}, - existingSyncs: map[string]string{"pubmatic": "1234"}, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Unset uid for a bidder if the request contains an empty uid for that bidder", - }, - { - uri: "/setuid?bidder=pubmatic&uid=123", - syncers: []string{"pubmatic"}, - existingSyncs: map[string]string{"rubicon": "def"}, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123", "rubicon": "def"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Add the uid for the requested bidder to the list of existing syncs", - }, - { - uri: "/setuid?bidder=pubmatic&uid=123&gdpr=0", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Don't care about GDPR consent if GDPR is set to 0", - }, - { - uri: "/setuid?uid=123", - syncers: []string{"appnexus"}, - existingSyncs: nil, - expectedSyncs: nil, - gdprAllowsHostCookies: true, - expectedStatusCode: http.StatusBadRequest, - expectedBody: `"bidder" query param is required`, - description: "Return an error if the bidder param is missing from the request", - }, - { - uri: "/setuid?bidder=appnexus&uid=123&gdpr=2", - syncers: []string{"appnexus"}, - existingSyncs: nil, - expectedSyncs: nil, - gdprAllowsHostCookies: true, - expectedStatusCode: http.StatusBadRequest, - expectedBody: "the gdpr query param must be either 0 or 1. You gave 2", - description: "Return an error if GDPR is set to anything else other that 0 or 1", - }, - { - uri: "/setuid?bidder=appnexus&uid=123&gdpr=1", - syncers: []string{"appnexus"}, - existingSyncs: nil, - expectedSyncs: nil, - gdprAllowsHostCookies: true, - expectedStatusCode: http.StatusBadRequest, - expectedBody: "gdpr_consent is required when gdpr=1", - description: "Return an error if GDPR is set to 1 but GDPR consent string is missing", + uri: "/setuid?bidder=pubmatic", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: map[string]string{"pubmatic": "1234"}, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Unset uid for a bidder if the request contains an empty uid for that bidder", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: map[string]string{"rubicon": "def"}, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123", "rubicon": "def"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Add the uid for the requested bidder to the list of existing syncs", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&gdpr=0", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Don't care about GDPR consent if GDPR is set to 0", + }, + { + uri: "/setuid?uid=123", + syncersBidderNameToKey: map[string]string{"appnexus": "appnexus"}, + existingSyncs: nil, + expectedSyncs: nil, + gdprAllowsHostCookies: true, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"bidder" query param is required`, + description: "Return an error if the bidder param is missing from the request", + }, + { + uri: "/setuid?bidder=appnexus&uid=123&gdpr=2", + syncersBidderNameToKey: map[string]string{"appnexus": "appnexus"}, + existingSyncs: nil, + expectedSyncs: nil, + gdprAllowsHostCookies: true, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "the gdpr query param must be either 0 or 1. You gave 2", + description: "Return an error if GDPR is set to anything else other that 0 or 1", + }, + { + uri: "/setuid?bidder=appnexus&uid=123&gdpr=1", + syncersBidderNameToKey: map[string]string{"appnexus": "appnexus"}, + existingSyncs: nil, + expectedSyncs: nil, + gdprAllowsHostCookies: true, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "gdpr_consent is required when gdpr=1", + description: "Return an error if GDPR is set to 1 but GDPR consent string is missing", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - expectedSyncs: nil, - gdprReturnsError: true, - expectedStatusCode: http.StatusBadRequest, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + expectedSyncs: nil, + gdprReturnsError: true, + expectedStatusCode: http.StatusBadRequest, expectedBody: "No global vendor list was available to interpret this consent string. " + "If this is a new, valid version, it should become available soon.", description: "Return an error if the GDPR string is either malformed or using a newer version that isn't yet supported", @@ -163,71 +173,71 @@ func TestSetUIDEndpoint(t *testing.T) { { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - expectedSyncs: nil, - expectedStatusCode: http.StatusUnavailableForLegalReasons, - expectedBody: "The gdpr_consent string prevents cookies from being saved", - description: "Shouldn't set uid for a bidder if it is not allowed by the GDPR consent string", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + expectedSyncs: nil, + expectedStatusCode: http.StatusUnavailableForLegalReasons, + expectedBody: "The gdpr_consent string prevents cookies from being saved", + description: "Shouldn't set uid for a bidder if it is not allowed by the GDPR consent string", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + "BONciguONcjGKADACHENAOLS1rAHDAFAAEAASABQAMwAeACEAFw", - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - existingSyncs: nil, - expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Should set uid for a bidder that is allowed by the GDPR consent string", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + existingSyncs: nil, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Should set uid for a bidder that is allowed by the GDPR consent string", }, { uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=" + "malformed", - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - gdprMalformed: true, - existingSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: "gdpr_consent was invalid. malformed consent string malformed: some error", - description: "Should return an error if GDPR consent string is malformed", - }, - { - uri: "/setuid?bidder=pubmatic&uid=123&f=b", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, - description: "Set uid for valid bidder with iframe format", - }, - { - uri: "/setuid?bidder=pubmatic&uid=123&f=i", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: map[string]string{"pubmatic": "123"}, - expectedStatusCode: http.StatusOK, - expectedHeaders: map[string]string{"Content-Type": "image/png", "Content-Length": "86"}, - description: "Set uid for valid bidder with redirect format", - }, - { - uri: "/setuid?bidder=pubmatic&uid=123&f=x", - syncers: []string{"pubmatic"}, - existingSyncs: nil, - gdprAllowsHostCookies: true, - expectedSyncs: nil, - expectedStatusCode: http.StatusBadRequest, - expectedBody: `"f" query param is invalid. must be "b" or "i"`, - description: "Set uid for valid bidder with invalid format", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + gdprMalformed: true, + existingSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: "gdpr_consent was invalid. malformed consent string malformed: some error", + description: "Should return an error if GDPR consent string is malformed", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=b", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"}, + description: "Set uid for valid bidder with iframe format", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=i", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: map[string]string{"pubmatic": "123"}, + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "image/png", "Content-Length": "86"}, + description: "Set uid for valid bidder with redirect format", + }, + { + uri: "/setuid?bidder=pubmatic&uid=123&f=x", + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + existingSyncs: nil, + gdprAllowsHostCookies: true, + expectedSyncs: nil, + expectedStatusCode: http.StatusBadRequest, + expectedBody: `"f" query param is invalid. must be "b" or "i"`, + description: "Set uid for valid bidder with invalid format", }, } metrics := &metricsConf.DummyMetricsEngine{} for _, test := range testCases { response := doRequest(makeRequest(test.uri, test.existingSyncs), metrics, - test.syncers, test.gdprAllowsHostCookies, test.gdprReturnsError, test.gdprMalformed) + test.syncersBidderNameToKey, test.gdprAllowsHostCookies, test.gdprReturnsError, test.gdprMalformed) assert.Equal(t, test.expectedStatusCode, response.Code, "Test Case: %s. /setuid returned unexpected error code", test.description) if test.expectedSyncs != nil { @@ -259,89 +269,89 @@ func TestSetUIDEndpointMetrics(t *testing.T) { cookieWithOptOut.SetOptOut(true) testCases := []struct { - description string - uri string - cookies []*usersync.Cookie - syncers []string - gdprAllowsHostCookies bool - expectedResponseCode int - expectedMetrics func(*metrics.MetricsEngineMock) + description string + uri string + cookies []*usersync.Cookie + syncersBidderNameToKey map[string]string + gdprAllowsHostCookies bool + expectedResponseCode int + expectedMetrics func(*metrics.MetricsEngineMock) }{ { - description: "Success - Sync", - uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.Cookie{}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - expectedResponseCode: 200, + description: "Success - Sync", + uri: "/setuid?bidder=pubmatic&uid=123", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 200, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidOK).Once() m.On("RecordSyncerSet", "pubmatic", metrics.SyncerSetUidOK).Once() }, }, { - description: "Success - Unsync", - uri: "/setuid?bidder=pubmatic&uid=", - cookies: []*usersync.Cookie{}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - expectedResponseCode: 200, + description: "Success - Unsync", + uri: "/setuid?bidder=pubmatic&uid=", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 200, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidOK).Once() m.On("RecordSyncerSet", "pubmatic", metrics.SyncerSetUidCleared).Once() }, }, { - description: "Cookie Opted Out", - uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.Cookie{cookieWithOptOut}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - expectedResponseCode: 401, + description: "Cookie Opted Out", + uri: "/setuid?bidder=pubmatic&uid=123", + cookies: []*usersync.Cookie{cookieWithOptOut}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 401, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidOptOut).Once() }, }, { - description: "Unknown Syncer Key", - uri: "/setuid?bidder=pubmatic&uid=123", - cookies: []*usersync.Cookie{}, - syncers: []string{}, - gdprAllowsHostCookies: true, - expectedResponseCode: 400, + description: "Unknown Syncer Key", + uri: "/setuid?bidder=pubmatic&uid=123", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{}, + gdprAllowsHostCookies: true, + expectedResponseCode: 400, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidSyncerUnknown).Once() }, }, { - description: "Unknown Format", - uri: "/setuid?bidder=pubmatic&uid=123&f=z", - cookies: []*usersync.Cookie{}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - expectedResponseCode: 400, + description: "Unknown Format", + uri: "/setuid?bidder=pubmatic&uid=123&f=z", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 400, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidBadRequest).Once() }, }, { - description: "Prevented By GDPR - Invalid Consent String", - uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1", - cookies: []*usersync.Cookie{}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: true, - expectedResponseCode: 400, + description: "Prevented By GDPR - Invalid Consent String", + uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: true, + expectedResponseCode: 400, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidBadRequest).Once() }, }, { - description: "Prevented By GDPR - Permission Denied By Consent String", - uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=any", - cookies: []*usersync.Cookie{}, - syncers: []string{"pubmatic"}, - gdprAllowsHostCookies: false, - expectedResponseCode: 451, + description: "Prevented By GDPR - Permission Denied By Consent String", + uri: "/setuid?bidder=pubmatic&uid=123&gdpr=1&gdpr_consent=any", + cookies: []*usersync.Cookie{}, + syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"}, + gdprAllowsHostCookies: false, + expectedResponseCode: 451, expectedMetrics: func(m *metrics.MetricsEngineMock) { m.On("RecordSetUid", metrics.SetUidGDPRHostCookieBlocked).Once() }, @@ -356,7 +366,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { for _, v := range test.cookies { addCookie(req, v) } - response := doRequest(req, metricsEngine, test.syncers, test.gdprAllowsHostCookies, false, false) + response := doRequest(req, metricsEngine, test.syncersBidderNameToKey, test.gdprAllowsHostCookies, false, false) assert.Equal(t, test.expectedResponseCode, response.Code, test.description) metricsEngine.AssertExpectations(t) @@ -368,9 +378,9 @@ func TestOptedOut(t *testing.T) { cookie := usersync.NewCookie() cookie.SetOptOut(true) addCookie(request, cookie) - syncers := []string{"pubmatic"} + syncersBidderNameToKey := map[string]string{"pubmatic": "pubmatic"} metrics := &metricsConf.DummyMetricsEngine{} - response := doRequest(request, metrics, syncers, true, false, false) + response := doRequest(request, metrics, syncersBidderNameToKey, true, false, false) assert.Equal(t, http.StatusUnauthorized, response.Code) } @@ -506,7 +516,7 @@ func makeRequest(uri string, existingSyncs map[string]string) *http.Request { return request } -func doRequest(req *http.Request, metrics metrics.MetricsEngine, syncers []string, gdprAllowsHostCookies, gdprReturnsError, gdprReturnsMalformedError bool) *httptest.ResponseRecorder { +func doRequest(req *http.Request, metrics metrics.MetricsEngine, syncersBidderNameToKey map[string]string, gdprAllowsHostCookies, gdprReturnsError, gdprReturnsMalformedError bool) *httptest.ResponseRecorder { cfg := config.Configuration{} perms := &mockPermsSetUID{ allowHost: gdprAllowsHostCookies, @@ -515,12 +525,12 @@ func doRequest(req *http.Request, metrics metrics.MetricsEngine, syncers []strin personalInfoAllowed: true, } analytics := analyticsConf.NewPBSAnalytics(&cfg.Analytics) - syncersMap := make(map[string]usersync.Syncer) - for _, key := range syncers { - syncersMap[key] = fakeSyncer{key: key, defaultSyncType: usersync.SyncTypeIFrame} + syncersByBidder := make(map[string]usersync.Syncer) + for bidderName, syncerKey := range syncersBidderNameToKey { + syncersByBidder[bidderName] = fakeSyncer{key: syncerKey, defaultSyncType: usersync.SyncTypeIFrame} } - endpoint := NewSetUIDEndpoint(cfg.HostCookie, syncersMap, perms, analytics, metrics) + endpoint := NewSetUIDEndpoint(cfg.HostCookie, syncersByBidder, perms, analytics, metrics) response := httptest.NewRecorder() endpoint(response, req, nil) return response From 4f9568aa888ed0ab6fcf2974b994c658ed0fd134 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 9 Aug 2021 23:26:16 -0400 Subject: [PATCH 38/50] Fix ExternalURL Override Hierarchy --- router/router.go | 2 +- usersync/syncersbuilder.go | 10 ++++++++-- usersync/syncersbuilder_test.go | 23 +++++++++++++++++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/router/router.go b/router/router.go index 145e576db34..f5ec3df2fd4 100644 --- a/router/router.go +++ b/router/router.go @@ -220,7 +220,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } - syncers, err := usersync.BuildSyncers(cfg.UserSync, bidderInfos) + syncers, err := usersync.BuildSyncers(cfg, bidderInfos) if err != nil { return nil, err } diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index 33c1027f5b8..bf89b487aa8 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -14,7 +14,7 @@ type namedSyncerConfig struct { cfg config.Syncer } -func BuildSyncers(hostConfig config.UserSync, bidderInfos config.BidderInfos) (map[string]Syncer, error) { +func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, error) { // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { @@ -32,6 +32,12 @@ func BuildSyncers(hostConfig config.UserSync, bidderInfos config.BidderInfos) (m cfgBySyncerKey[cfg.Key] = append(cfgBySyncerKey[cfg.Key], namedSyncerConfig{bidder, cfg}) } + // resolve host endpoint + hostUserSyncConfig := hostConfig.UserSync + if hostUserSyncConfig.ExternalURL == "" { + hostUserSyncConfig.ExternalURL = hostConfig.ExternalURL + } + // create syncers errs := []error{} syncers := make(map[string]Syncer, len(bidderInfos)) @@ -42,7 +48,7 @@ func BuildSyncers(hostConfig config.UserSync, bidderInfos config.BidderInfos) (m continue } - syncer, err := NewSyncer(hostConfig, primaryCfg.cfg) + syncer, err := NewSyncer(hostUserSyncConfig, primaryCfg.cfg) if err != nil { errs = append(errs, fmt.Errorf("cannot create syncer for bidder %s with key %s. %v", primaryCfg.name, key, err)) continue diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index 838e0bbe4ac..29f00b79621 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -11,7 +11,7 @@ import ( func TestBuildSyncers(t *testing.T) { var ( - hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"} + hostConfig = config.Configuration{ExternalURL: "http://host.com", UserSync: config.UserSync{RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"}} iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} infoKeyAPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} infoKeyADisabled = config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} @@ -27,6 +27,7 @@ func TestBuildSyncers(t *testing.T) { testCases := []struct { description string + givenConfig config.Configuration givenBidderInfos config.BidderInfos expectedIFramesURLs map[string]string expectedErrorHeader string @@ -34,6 +35,7 @@ func TestBuildSyncers(t *testing.T) { }{ { description: "One", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated}, expectedIFramesURLs: map[string]string{ "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", @@ -41,6 +43,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "One - Missing Key - Defaults To Bidder Name", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyMissingPopulated}, expectedIFramesURLs: map[string]string{ "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fbidder1%2Fhost", @@ -48,6 +51,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "One - Syncer Error", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, expectedErrorHeader: "user sync (1 error)", expectedErrorSegments: []string{ @@ -56,6 +60,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Different Syncers", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyBPopulated}, expectedIFramesURLs: map[string]string{ "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", @@ -64,6 +69,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Same Syncers - One Primary", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAEmpty}, expectedIFramesURLs: map[string]string{ "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fa%2Fhost", @@ -72,6 +78,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Same Syncers - Many Primaries", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, expectedErrorHeader: "user sync (1 error)", expectedErrorSegments: []string{ @@ -80,6 +87,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Sync Error - Bidder Correct", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, expectedErrorHeader: "user sync (1 error)", expectedErrorSegments: []string{ @@ -88,6 +96,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Empty Syncers Ignored", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": {}, "bidder2": infoKeyBPopulated}, expectedIFramesURLs: map[string]string{ "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", @@ -95,6 +104,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Disabled Syncers Ignored", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyADisabled, "bidder2": infoKeyBPopulated}, expectedIFramesURLs: map[string]string{ "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", @@ -102,6 +112,7 @@ func TestBuildSyncers(t *testing.T) { }, { description: "Many - Multiple Errors", + givenConfig: hostConfig, givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, expectedErrorHeader: "user sync (2 errors)", expectedErrorSegments: []string{ @@ -109,10 +120,18 @@ func TestBuildSyncers(t *testing.T) { "cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", }, }, + { + description: "ExternalURL Host User Sync Override", + givenConfig: config.Configuration{ExternalURL: "http://host.com", UserSync: config.UserSync{ExternalURL: "http://hostoverride.com", RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"}}, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder1": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhostoverride.com%2Fa%2Fhost", + }, + }, } for _, test := range testCases { - result, err := BuildSyncers(hostConfig, test.givenBidderInfos) + result, err := BuildSyncers(&test.givenConfig, test.givenBidderInfos) if test.expectedErrorHeader == "" { assert.NoError(t, err, test.description+":err") From d6bb3661aaa39127ab0638be640f8e8beb65c2d0 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 10 Aug 2021 00:51:57 -0400 Subject: [PATCH 39/50] Reconciled + Verified User Sync Endpoints --- static/bidder-info/adagio.yaml | 6 +++++- static/bidder-info/bidmyadz.yaml | 6 +++++- static/bidder-info/operaads.yaml | 6 +++++- static/bidder-info/sa_lunamedia.yaml | 6 +++++- static/bidder-info/smilewanted.yaml | 4 ++++ static/bidder-info/triplelift_native.yaml | 1 - 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/static/bidder-info/adagio.yaml b/static/bidder-info/adagio.yaml index 3661191b3a1..891b2a6caea 100644 --- a/static/bidder-info/adagio.yaml +++ b/static/bidder-info/adagio.yaml @@ -9,4 +9,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://mp.4dex.io/sync?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "{{UID}}" diff --git a/static/bidder-info/bidmyadz.yaml b/static/bidder-info/bidmyadz.yaml index 70a995a2798..41cf4ecaa0a 100644 --- a/static/bidder-info/bidmyadz.yaml +++ b/static/bidder-info/bidmyadz.yaml @@ -10,4 +10,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://cookie-sync.bidmyadz.com/c0f68227d14ed938c6c49f3967cbe9bc?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}&red={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/operaads.yaml b/static/bidder-info/operaads.yaml index b95d81155c1..d28492b139a 100644 --- a/static/bidder-info/operaads.yaml +++ b/static/bidder-info/operaads.yaml @@ -11,4 +11,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://t.adx.opera.com/pbs/sync?gdpr={{.GDPR}}&us_privacy={{.USPrivacy}}&gdpr_consent={{.GDPRConsent}}&r={{.RedirectURL}}" + userMacro: "${UID}" diff --git a/static/bidder-info/sa_lunamedia.yaml b/static/bidder-info/sa_lunamedia.yaml index 181e1fd6c73..f5c2cf78e20 100644 --- a/static/bidder-info/sa_lunamedia.yaml +++ b/static/bidder-info/sa_lunamedia.yaml @@ -11,4 +11,8 @@ capabilities: mediaTypes: - banner - video - - native \ No newline at end of file + - native +userSync: + redirect: + url: "https://cookie.lmgssp.com/pserver?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&ccpa={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "[UID]" diff --git a/static/bidder-info/smilewanted.yaml b/static/bidder-info/smilewanted.yaml index 81b0585bb5e..e3d62bd5db6 100644 --- a/static/bidder-info/smilewanted.yaml +++ b/static/bidder-info/smilewanted.yaml @@ -10,3 +10,7 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + url: "https://csync.smilewanted.com/getuid?source=prebid-server&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + userMacro: "$UID" diff --git a/static/bidder-info/triplelift_native.yaml b/static/bidder-info/triplelift_native.yaml index f9cc30fbec1..32eb07efa14 100644 --- a/static/bidder-info/triplelift_native.yaml +++ b/static/bidder-info/triplelift_native.yaml @@ -9,7 +9,6 @@ capabilities: mediaTypes: - native userSync: - key: "triplelift" redirect: url: "https://eb2.3lift.com/getuid?gdpr={{.GDPR}}&cmp_cs={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir={{.RedirectURL}}" userMacro: "$UID" \ No newline at end of file From ebaa7e114af86c37e5cd79bc3f0751aa0464a630 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 10 Aug 2021 01:04:04 -0400 Subject: [PATCH 40/50] Improved Syncer Bidder Map Variable Name --- endpoints/auction.go | 28 ++++++++++++++-------------- endpoints/auction_test.go | 8 ++++---- endpoints/cookie_sync.go | 4 ++-- endpoints/cookie_sync_test.go | 6 +++--- endpoints/setuid.go | 6 +++--- exchange/exchange.go | 4 ++-- router/router.go | 18 +++++++++--------- 7 files changed, 37 insertions(+), 37 deletions(-) diff --git a/endpoints/auction.go b/endpoints/auction.go index 54e5a9c041c..10d2ced6c37 100644 --- a/endpoints/auction.go +++ b/endpoints/auction.go @@ -59,22 +59,22 @@ func writeAuctionError(w http.ResponseWriter, s string, err error) { } type auction struct { - cfg *config.Configuration - syncers map[string]usersync.Syncer - gdprPerms gdpr.Permissions - metricsEngine metrics.MetricsEngine - dataCache cache.Cache - exchanges map[string]adapters.Adapter + cfg *config.Configuration + syncersByBidder map[string]usersync.Syncer + gdprPerms gdpr.Permissions + metricsEngine metrics.MetricsEngine + dataCache cache.Cache + exchanges map[string]adapters.Adapter } -func Auction(cfg *config.Configuration, syncers map[string]usersync.Syncer, gdprPerms gdpr.Permissions, metricsEngine metrics.MetricsEngine, dataCache cache.Cache, exchanges map[string]adapters.Adapter) httprouter.Handle { +func Auction(cfg *config.Configuration, syncersByBidder map[string]usersync.Syncer, gdprPerms gdpr.Permissions, metricsEngine metrics.MetricsEngine, dataCache cache.Cache, exchanges map[string]adapters.Adapter) httprouter.Handle { a := &auction{ - cfg: cfg, - syncers: syncers, - gdprPerms: gdprPerms, - metricsEngine: metricsEngine, - dataCache: dataCache, - exchanges: exchanges, + cfg: cfg, + syncersByBidder: syncersByBidder, + gdprPerms: gdprPerms, + metricsEngine: metricsEngine, + dataCache: dataCache, + exchanges: exchanges, } return a.auction } @@ -482,7 +482,7 @@ func (a *auction) processUserSync(req *pbs.PBSRequest, bidder *pbs.PBSBidder, bl if syncerCode == "districtm" { syncerCode = "appnexus" } - syncer := a.syncers[syncerCode] + syncer := a.syncersByBidder[syncerCode] uid, _, _ := req.Cookie.GetUID(syncer.Key()) if uid == "" { bidder.NoCookie = true diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go index faa648d3990..522b38babcf 100644 --- a/endpoints/auction_test.go +++ b/endpoints/auction_test.go @@ -351,13 +351,13 @@ func TestCacheVideoOnly(t *testing.T) { if err != nil { t.Fatal(err.Error()) } - syncers := map[string]usersync.Syncer{} + syncersByBidder := map[string]usersync.Syncer{} gdprPerms := gdpr.NewPermissions(context.Background(), config.GDPR{ HostVendorID: 0, }, nil, nil) prebid_cache_client.InitPrebidCache(server.URL) var labels = &metrics.Labels{} - if err := cacheVideoOnly(bids, ctx, &auction{cfg: cfg, syncers: syncers, gdprPerms: gdprPerms, metricsEngine: &metricsConf.DummyMetricsEngine{}}, labels); err != nil { + if err := cacheVideoOnly(bids, ctx, &auction{cfg: cfg, syncersByBidder: syncersByBidder, gdprPerms: gdprPerms, metricsEngine: &metricsConf.DummyMetricsEngine{}}, labels); err != nil { t.Errorf("Prebid cache failed: %v \n", err) return } @@ -638,8 +638,8 @@ func TestWriteAuctionError(t *testing.T) { func TestPanicRecovery(t *testing.T) { dummy := auction{ - cfg: nil, - syncers: nil, + cfg: nil, + syncersByBidder: nil, gdprPerms: &auctionMockPermissions{ allowBidderSync: false, allowHostCookies: false, diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 616f19a5e2e..f6ec8152870 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -32,7 +32,7 @@ var ( var cookieSyncBidderFilterAllowAll = usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude) func NewCookieSyncEndpoint( - syncers map[string]usersync.Syncer, + syncersByBidder map[string]usersync.Syncer, config *config.Configuration, gdprPermissions gdpr.Permissions, metrics metrics.MetricsEngine, @@ -45,7 +45,7 @@ func NewCookieSyncEndpoint( } return &cookieSyncEndpoint{ - chooser: usersync.NewChooser(syncers), + chooser: usersync.NewChooser(syncersByBidder), config: config.UserSync, hostCookieConfig: &config.HostCookie, privacyConfig: usersyncPrivacyConfig{ diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index cfcf71cdb98..2ef2eb1770b 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -26,7 +26,7 @@ import ( func TestNewCookieSyncEndpoint(t *testing.T) { var ( - syncers = map[string]usersync.Syncer{"a": &MockSyncer{}} + syncersByBidder = map[string]usersync.Syncer{"a": &MockSyncer{}} gdprPerms = MockGDPRPerms{} configUserSync = config.UserSync{Cooperative: config.UserSyncCooperative{EnabledByDefault: true}} configHostCookie = config.HostCookie{Family: "foo"} @@ -38,7 +38,7 @@ func TestNewCookieSyncEndpoint(t *testing.T) { ) endpoint := NewCookieSyncEndpoint( - syncers, + syncersByBidder, &config.Configuration{ UserSync: configUserSync, HostCookie: configHostCookie, @@ -52,7 +52,7 @@ func TestNewCookieSyncEndpoint(t *testing.T) { ) expected := &cookieSyncEndpoint{ - chooser: usersync.NewChooser(syncers), + chooser: usersync.NewChooser(syncersByBidder), config: configUserSync, hostCookieConfig: &configHostCookie, privacyConfig: usersyncPrivacyConfig{ diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 3f774579c33..014ff3122c1 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -26,13 +26,13 @@ const ( chromeiOSStrLen = len(chromeiOSStr) ) -func NewSetUIDEndpoint(cfg config.HostCookie, syncers map[string]usersync.Syncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { +func NewSetUIDEndpoint(cfg config.HostCookie, syncersByBidder map[string]usersync.Syncer, perms gdpr.Permissions, pbsanalytics analytics.PBSAnalyticsModule, metricsEngine metrics.MetricsEngine) httprouter.Handle { cookieTTL := time.Duration(cfg.TTL) * 24 * time.Hour // convert map of syncers by bidder to map of syncers by key // - its safe to assume that if multiple bidders map to the same key, the syncers are interchangeable. - syncersByKey := make(map[string]usersync.Syncer, len(syncers)) - for _, v := range syncers { + syncersByKey := make(map[string]usersync.Syncer, len(syncersByBidder)) + for _, v := range syncersByBidder { syncersByKey[v.Key()] = v } diff --git a/exchange/exchange.go b/exchange/exchange.go index ff928cd8c01..4d4767b9b53 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -111,9 +111,9 @@ func (randomDeduplicateBidBooleanGenerator) Generate() bool { return rand.Intn(100) < 50 } -func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, syncers map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher) Exchange { +func NewExchange(adapters map[openrtb_ext.BidderName]adaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, syncersByBidder map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher) Exchange { bidderToSyncerKey := map[string]string{} - for bidder, syncer := range syncers { + for bidder, syncer := range syncersByBidder { bidderToSyncerKey[bidder] = syncer.Key() } diff --git a/router/router.go b/router/router.go index f5ec3df2fd4..3adb9a4647a 100644 --- a/router/router.go +++ b/router/router.go @@ -220,18 +220,18 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } - syncers, err := usersync.BuildSyncers(cfg, bidderInfos) + syncersByBidder, err := usersync.BuildSyncers(cfg, bidderInfos) if err != nil { return nil, err } - syncerKeys := make([]string, 0, len(syncers)) + syncerKeys := make([]string, 0, len(syncersByBidder)) syncerKeysHashSet := map[string]struct{}{} - for _, syncer := range syncers { + for _, syncer := range syncersByBidder { syncerKeysHashSet[syncer.Key()] = struct{}{} } - for syncerKey := range syncerKeysHashSet { - syncerKeys = append(syncerKeys, syncerKey) + for k := range syncerKeysHashSet { + syncerKeys = append(syncerKeys, k) } // Metrics engine @@ -270,7 +270,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, errs } - theExchange := exchange.NewExchange(adapters, cacheClient, cfg, syncers, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor, categoriesFetcher) + theExchange := exchange.NewExchange(adapters, cacheClient, cfg, syncersByBidder, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor, categoriesFetcher) openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, fetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders) if err != nil { @@ -292,14 +292,14 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R videoEndpoint = aspects.QueuedRequestTimeout(videoEndpoint, cfg.RequestTimeoutHeaders, r.MetricsEngine, metrics.ReqTypeVideo) } - r.POST("/auction", endpoints.Auction(cfg, syncers, gdprPerms, r.MetricsEngine, dataCache, exchanges)) + r.POST("/auction", endpoints.Auction(cfg, syncersByBidder, gdprPerms, r.MetricsEngine, dataCache, exchanges)) r.POST("/openrtb2/auction", openrtbEndpoint) r.POST("/openrtb2/video", videoEndpoint) r.GET("/openrtb2/amp", ampEndpoint) r.GET("/info/bidders", infoEndpoints.NewBiddersEndpoint(bidderInfos, defaultAliases)) r.GET("/info/bidders/:bidderName", infoEndpoints.NewBiddersDetailEndpoint(bidderInfos, cfg.Adapters, defaultAliases)) r.GET("/bidders/params", NewJsonDirectoryServer(schemaDirectory, paramsValidator, defaultAliases)) - r.POST("/cookie_sync", endpoints.NewCookieSyncEndpoint(syncers, cfg, gdprPerms, r.MetricsEngine, pbsAnalytics, activeBidders).Handle) + r.POST("/cookie_sync", endpoints.NewCookieSyncEndpoint(syncersByBidder, cfg, gdprPerms, r.MetricsEngine, pbsAnalytics, activeBidders).Handle) r.GET("/status", endpoints.NewStatusEndpoint(cfg.StatusResponse)) r.GET("/", serveIndex) r.ServeFiles("/static/*filepath", http.Dir("static")) @@ -322,7 +322,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R PBSAnalytics: pbsAnalytics, } - r.GET("/setuid", endpoints.NewSetUIDEndpoint(cfg.HostCookie, syncers, gdprPerms, pbsAnalytics, r.MetricsEngine)) + r.GET("/setuid", endpoints.NewSetUIDEndpoint(cfg.HostCookie, syncersByBidder, gdprPerms, pbsAnalytics, r.MetricsEngine)) r.GET("/getuids", endpoints.NewGetUIDsEndpoint(cfg.HostCookie)) r.POST("/optout", userSyncDeps.OptOut) r.GET("/optout", userSyncDeps.OptOut) From 8b2c36d2075cf9df524a35cc616cff5a8d9431d7 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 10 Aug 2021 01:20:06 -0400 Subject: [PATCH 41/50] Add Test Time Validation --- config/bidderinfo_validate_test.go | 46 ++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/config/bidderinfo_validate_test.go b/config/bidderinfo_validate_test.go index 23d6e3df034..b76c146fa1b 100644 --- a/config/bidderinfo_validate_test.go +++ b/config/bidderinfo_validate_test.go @@ -1,4 +1,4 @@ -package config +package config_test import ( "errors" @@ -8,7 +8,10 @@ import ( "strings" "testing" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/privacy" + "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" ) @@ -45,7 +48,7 @@ func TestBidderInfoFiles(t *testing.T) { content, err := ioutil.ReadAll(infoFileData) assert.NoError(t, err, "Failed to read static/bidder-info/%s: %v", fileInfo.Name(), err) - var fileInfoContent BidderInfo + var fileInfoContent config.BidderInfo err = yaml.Unmarshal(content, &fileInfoContent) assert.NoError(t, err, "Error interpreting content from static/bidder-info/%s: %v", fileInfo.Name(), err) @@ -54,7 +57,7 @@ func TestBidderInfoFiles(t *testing.T) { } } -func validateInfo(info *BidderInfo) error { +func validateInfo(info *config.BidderInfo) error { if err := validateMaintainer(info.Maintainer); err != nil { return err } @@ -63,17 +66,21 @@ func validateInfo(info *BidderInfo) error { return err } + if err := validateSyncer(info.Syncer); err != nil { + return err + } + return nil } -func validateMaintainer(info *MaintainerInfo) error { +func validateMaintainer(info *config.MaintainerInfo) error { if info == nil || info.Email == "" { return errors.New("missing required field: maintainer.email") } return nil } -func validateCapabilities(info *CapabilitiesInfo) error { +func validateCapabilities(info *config.CapabilitiesInfo) error { if info == nil { return errors.New("missing required field: capabilities") } @@ -96,7 +103,7 @@ func validateCapabilities(info *CapabilitiesInfo) error { return nil } -func validatePlatformInfo(info *PlatformInfo) error { +func validatePlatformInfo(info *config.PlatformInfo) error { if info == nil { return errors.New("object cannot be empty") } @@ -113,3 +120,30 @@ func validatePlatformInfo(info *PlatformInfo) error { return nil } + +func validateSyncer(syncerCfg *config.Syncer) error { + if syncerCfg == nil { + return nil + } + + hostConfig := config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} + + // emulate run time substitution of bidder name for empty keys + if syncerCfg.Key == "" { + syncerCfg.Key = "bidder" + } + + syncer, err := usersync.NewSyncer(hostConfig, *syncerCfg) + if err != nil { + return fmt.Errorf("syncer could not be created: %s", err) + } + + // ensure final macro substitution + privacyPolicies := privacy.Policies{} + _, err = syncer.GetSync([]usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect}, privacyPolicies) + if err != nil { + return fmt.Errorf("syncer has invalid macro: %s", err) + } + + return nil +} From a16417a3d7b017871faf6a7d30c4bda09f0318cb Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 11 Aug 2021 12:57:56 -0400 Subject: [PATCH 42/50] Improved Error Message --- router/router.go | 2 +- router/router_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/router/router.go b/router/router.go index 3adb9a4647a..5d160687b79 100644 --- a/router/router.go +++ b/router/router.go @@ -341,7 +341,7 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg // the new config. if adapterCfg.UserSyncURL != "" { if bidderInfo.Syncer == nil { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync", bidderName) + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync or is disabled", bidderName) } endpointsCount := 0 diff --git a/router/router_test.go b/router/router_test.go index 5865efddf41..ec7ab43e946 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -104,7 +104,7 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { description: "UserSyncURL Override Syncer Not Defined", givenBidderInfos: config.BidderInfos{"a": {}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync", + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync or is disabled", }, { description: "UserSyncURL Override Syncer Endpoints Not Defined", From 0a6655cd77ec1acb176c5eef4346a42f621e1ed6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 11 Aug 2021 13:20:56 -0400 Subject: [PATCH 43/50] Fixed Test Descriptions --- usersync/syncer_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 8c7fe4a5f45..2f4e3203905 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -295,7 +295,8 @@ func TestBuildTemplate(t *testing.T) { expectedRendered: "https://bidder.com/sync?redirect=http%3A%2F%2Fhost.com%2Fsetuid%3Fbidder%3DanyKey%26f%3Dx%26gdpr%3DA%26uid%3D%7B%7BUID%7D%7D", }, - // The following tests protect against the "." literal character vs the "."" character class in regex. + // The following tests protect against the "\"." literal character vs the "." character class in regex. Literal + // value which use {{ }} but do not match Go's naming pattern of {{ .Name }} are escaped. { description: "Invalid Macro - Redirect URL", givenSyncerEndpoint: config.SyncerEndpoint{ @@ -304,7 +305,7 @@ func TestBuildTemplate(t *testing.T) { expectedError: "template: anykey_usersync_url:1: function \"xRedirectURL\" not defined", }, { - description: "Invalid Macro - External URL", + description: "Macro-Like Literal Value - External URL", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xExternalURL}}", @@ -312,7 +313,7 @@ func TestBuildTemplate(t *testing.T) { expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxExternalURL%7D%7D", }, { - description: "Invalid Macro - Syncer Key", + description: "Macro-Like Literal Value - Syncer Key", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xSyncerKey}}", @@ -320,7 +321,7 @@ func TestBuildTemplate(t *testing.T) { expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxSyncerKey%7D%7D", }, { - description: "Invalid Macro - Sync Type", + description: "Macro-Like Literal Value - Sync Type", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xSyncType}}", @@ -328,7 +329,7 @@ func TestBuildTemplate(t *testing.T) { expectedRendered: "https://bidder.com/sync?redirect=%7B%7BxSyncType%7D%7D", }, { - description: "Invalid Macro - User Macro", + description: "Macro-Like Literal Value - User Macro", givenSyncerEndpoint: config.SyncerEndpoint{ URL: "https://bidder.com/sync?redirect={{.RedirectURL}}", RedirectURL: "{{xUserMacro}}", From 5649c77ce089140512ea0db9c3190bf3b49da910 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 11 Aug 2021 14:01:05 -0400 Subject: [PATCH 44/50] Update InMobi Syncer --- static/bidder-info/inmobi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/bidder-info/inmobi.yaml b/static/bidder-info/inmobi.yaml index 9104bf15acf..43040658964 100644 --- a/static/bidder-info/inmobi.yaml +++ b/static/bidder-info/inmobi.yaml @@ -13,5 +13,5 @@ capabilities: - video userSync: redirect: - url: "https://id5-sync.com/i/495/0.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&callback={{.RedirectURL}}" + url: "https://sync.inmobi.com/prebid?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" userMacro: "{ID5UID}" From 7d3bbb501d507f1fd7e8ad8ee4113f71f78d9900 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Thu, 12 Aug 2021 14:12:17 -0400 Subject: [PATCH 45/50] Allow Bidders To Represent Support Without Default --- router/router.go | 30 +++++++-- router/router_test.go | 53 ++++++++++++++- static/bidder-info/adocean.yaml | 5 ++ static/bidder-info/adtarget.yaml | 5 ++ static/bidder-info/adtelligent.yaml | 5 ++ static/bidder-info/adxcg.yaml | 5 ++ static/bidder-info/audienceNetwork.yaml | 5 ++ static/bidder-info/bmtm.yaml | 5 ++ static/bidder-info/criteo.yaml | 7 +- static/bidder-info/dmx.yaml | 5 ++ static/bidder-info/gamma.yaml | 5 ++ static/bidder-info/invibes.yaml | 5 ++ static/bidder-info/lockerdome.yaml | 16 +++-- static/bidder-info/mediafuse.yaml | 5 ++ static/bidder-info/rtbhouse.yaml | 5 ++ static/bidder-info/rubicon.yaml | 5 ++ static/bidder-info/smarthub.yaml | 5 ++ static/bidder-info/synacormedia.yaml | 5 ++ static/bidder-info/verizonmedia.yaml | 5 ++ static/bidder-info/viewdeos.yaml | 5 ++ static/bidder-info/vrtcal.yaml | 5 ++ usersync/syncer.go | 19 ++++-- usersync/syncer_test.go | 25 +++++++ usersync/syncersbuilder.go | 23 +++++-- usersync/syncersbuilder_test.go | 90 ++++++++++++++----------- 25 files changed, 281 insertions(+), 67 deletions(-) diff --git a/router/router.go b/router/router.go index 5d160687b79..e8eb12db734 100644 --- a/router/router.go +++ b/router/router.go @@ -220,9 +220,9 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } - syncersByBidder, err := usersync.BuildSyncers(cfg, bidderInfos) - if err != nil { - return nil, err + syncersByBidder, errs := usersync.BuildSyncers(cfg, bidderInfos) + if errsFiltered := filterSyncerErrors(errs); len(errsFiltered) > 0 { + return nil, errortypes.NewAggregateError("user sync", errsFiltered) } syncerKeys := make([]string, 0, len(syncersByBidder)) @@ -341,7 +341,7 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg // the new config. if adapterCfg.UserSyncURL != "" { if bidderInfo.Syncer == nil { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync or is disabled", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define a user sync", strings.ToLower(bidderName)) } endpointsCount := 0 @@ -355,17 +355,17 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg } if endpointsCount == 0 { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define user sync endpoints", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define user sync endpoints", strings.ToLower(bidderName)) } // if the bidder defines both an iframe and redirect endpoint, we can't be sure which config value to // override, and it wouldn't be both. this is a fatal configuration error. if endpointsCount > 1 { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder defines multiple user sync endpoints", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder defines multiple user sync endpoints", strings.ToLower(bidderName)) } // provide a warning that this compatibility layer is temporary - glog.Warningf("legacy usersync_url setting for bidder %s will be removed in a future version of Prebid Server. please update to the latest user sync config values", bidderName) + glog.Warningf("adapters.%s.usersync_url is deprecated and will be removed in a future version, please update to the latest user sync config values", strings.ToLower(bidderName)) } bidderInfos[bidderName] = bidderInfo @@ -374,6 +374,22 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg return nil } +func filterSyncerErrors(errs []error) []error { + var fatalErrors []error + + for _, err := range errs { + syncerBuildErr, ok := err.(usersync.SyncerBuildError) + if ok && syncerBuildErr.Err == usersync.ErrSyncerURLRequired { + bidderName := strings.ToLower(syncerBuildErr.Bidder) + glog.Warningf("bidder %s supports user syncing but has endpoints with no default url, no syncs will be performed with %s.", bidderName, bidderName) + } else { + fatalErrors = append(fatalErrors, err) + } + } + + return fatalErrors +} + // Fixes #648 // // These CORS options pose a security risk... but it's a calculated one. diff --git a/router/router_test.go b/router/router_test.go index ec7ab43e946..f3a167967ce 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -2,6 +2,7 @@ package router import ( "encoding/json" + "errors" "io/ioutil" "net/http" "net/http/httptest" @@ -9,7 +10,9 @@ import ( "testing" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" + "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" ) @@ -104,19 +107,19 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { description: "UserSyncURL Override Syncer Not Defined", givenBidderInfos: config.BidderInfos{"a": {}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync or is disabled", + expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define a user sync", }, { description: "UserSyncURL Override Syncer Endpoints Not Defined", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define user sync endpoints", }, { description: "UserSyncURL Override Ambiguous", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "originalIFrame"}, Redirect: &config.SyncerEndpoint{URL: "originalRedirect"}}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder defines multiple user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints", }, } @@ -131,6 +134,50 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { } } +func TestFilterSyncerErrors(t *testing.T) { + fatalErrorA := errors.New("errorA") + fatalErrorB := &errortypes.BadInput{Message: "errorB"} + buildErrorURLRequired := usersync.SyncerBuildError{Bidder: "anyBidder", SyncerKey: "anyKey", Err: usersync.ErrSyncerURLRequired} + buildErrorOther := usersync.SyncerBuildError{Bidder: "anyBidder", SyncerKey: "anyKey", Err: usersync.ErrSyncerKeyRequired} + + var testCases = []struct { + description string + givenErrors []error + expectedErrors []error + }{ + { + description: "Nil", + givenErrors: nil, + expectedErrors: nil, + }, + { + description: "Empty", + givenErrors: []error{}, + expectedErrors: nil, + }, + { + description: "One - Not Filtered", + givenErrors: []error{fatalErrorA}, + expectedErrors: []error{fatalErrorA}, + }, + { + description: "One - Filtered", + givenErrors: []error{buildErrorURLRequired}, + expectedErrors: nil, + }, + { + description: "Many - Mixed", + givenErrors: []error{fatalErrorA, fatalErrorB, buildErrorURLRequired, buildErrorOther}, + expectedErrors: []error{fatalErrorA, fatalErrorB, buildErrorOther}, + }, + } + + for _, test := range testCases { + filteredErrs := filterSyncerErrors(test.givenErrors) + assert.Equal(t, test.expectedErrors, filteredErrs, test.description) + } +} + // Prevents #648 func TestCORSSupport(t *testing.T) { const origin = "https://publisher-domain.com" diff --git a/static/bidder-info/adocean.yaml b/static/bidder-info/adocean.yaml index 6374d752c34..e53dcc32a5f 100644 --- a/static/bidder-info/adocean.yaml +++ b/static/bidder-info/adocean.yaml @@ -8,3 +8,8 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + # adocean supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/adtarget.yaml b/static/bidder-info/adtarget.yaml index cc064b9ca6b..694ed139095 100644 --- a/static/bidder-info/adtarget.yaml +++ b/static/bidder-info/adtarget.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + # adtarget supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/adtelligent.yaml b/static/bidder-info/adtelligent.yaml index 30f55ea912e..6cb2c514743 100644 --- a/static/bidder-info/adtelligent.yaml +++ b/static/bidder-info/adtelligent.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + # adtelligent supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/adxcg.yaml b/static/bidder-info/adxcg.yaml index e5535ae7258..d2b15524310 100644 --- a/static/bidder-info/adxcg.yaml +++ b/static/bidder-info/adxcg.yaml @@ -11,3 +11,8 @@ capabilities: - banner - video - native +userSync: + redirect: + # adxcg supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/audienceNetwork.yaml b/static/bidder-info/audienceNetwork.yaml index 324e5c6dff8..451959f7ec5 100644 --- a/static/bidder-info/audienceNetwork.yaml +++ b/static/bidder-info/audienceNetwork.yaml @@ -6,3 +6,8 @@ capabilities: - banner - video - native +userSync: + redirect: + # facebook's audience network supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/bmtm.yaml b/static/bidder-info/bmtm.yaml index c13bf17db73..526e7d857af 100644 --- a/static/bidder-info/bmtm.yaml +++ b/static/bidder-info/bmtm.yaml @@ -6,3 +6,8 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + # bmtm supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/criteo.yaml b/static/bidder-info/criteo.yaml index bfa098ba39d..0898e5938ec 100644 --- a/static/bidder-info/criteo.yaml +++ b/static/bidder-info/criteo.yaml @@ -7,4 +7,9 @@ capabilities: - banner site: mediaTypes: - - banner \ No newline at end of file + - banner +userSync: + redirect: + # criteo supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/dmx.yaml b/static/bidder-info/dmx.yaml index d29d699daeb..04e3612512b 100644 --- a/static/bidder-info/dmx.yaml +++ b/static/bidder-info/dmx.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + # dmx supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/gamma.yaml b/static/bidder-info/gamma.yaml index aedaf2cf749..ef230dc30c0 100644 --- a/static/bidder-info/gamma.yaml +++ b/static/bidder-info/gamma.yaml @@ -5,3 +5,8 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + # gamma supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/invibes.yaml b/static/bidder-info/invibes.yaml index 45184fb9f65..1da3c05ffbe 100644 --- a/static/bidder-info/invibes.yaml +++ b/static/bidder-info/invibes.yaml @@ -5,3 +5,8 @@ capabilities: site: mediaTypes: - banner +userSync: + iframe: + # invibes supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/lockerdome.yaml b/static/bidder-info/lockerdome.yaml index eebf3b87841..38e3e346be7 100644 --- a/static/bidder-info/lockerdome.yaml +++ b/static/bidder-info/lockerdome.yaml @@ -7,10 +7,12 @@ capabilities: site: mediaTypes: - banner -# lockerdome requires a platform id for their user sync process. replace <> -# in the url below and uncomment the userSync section to enable. -# -#userSync: -# redirect: -# url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" -# userMacro: "{{uid}}" +userSync: + redirect: + # lockerdome supports user syncing, but requires a platform id to be configured by + # the host. remove the empty url, replace <> with the value provided, and + # uncomment to enable. + # + # url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + url: "" + userMacro: "{{uid}}" diff --git a/static/bidder-info/mediafuse.yaml b/static/bidder-info/mediafuse.yaml index 98611402905..5e2cedc7932 100644 --- a/static/bidder-info/mediafuse.yaml +++ b/static/bidder-info/mediafuse.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + iframe: + # mediafuse supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/rtbhouse.yaml b/static/bidder-info/rtbhouse.yaml index 12744fdc75e..58196af4c3f 100644 --- a/static/bidder-info/rtbhouse.yaml +++ b/static/bidder-info/rtbhouse.yaml @@ -5,3 +5,8 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + # rtbhouse supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/rubicon.yaml b/static/bidder-info/rubicon.yaml index 0f19ddb9627..351918ca3cd 100644 --- a/static/bidder-info/rubicon.yaml +++ b/static/bidder-info/rubicon.yaml @@ -9,3 +9,8 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + # rubicon supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/smarthub.yaml b/static/bidder-info/smarthub.yaml index bfee9490840..32a7a1d0c40 100644 --- a/static/bidder-info/smarthub.yaml +++ b/static/bidder-info/smarthub.yaml @@ -11,3 +11,8 @@ capabilities: - banner - video - native +userSync: + redirect: + # smarthub supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/synacormedia.yaml b/static/bidder-info/synacormedia.yaml index 14a4bcd1244..774ba35720f 100644 --- a/static/bidder-info/synacormedia.yaml +++ b/static/bidder-info/synacormedia.yaml @@ -13,3 +13,8 @@ userSync: iframe: url: "https://sync.technoratimedia.com/services?srv=cs&pid=70&cb={{.RedirectURL}}" userMacro: "[USER_ID]" +userSync: + iframe: + # synacormedia supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/verizonmedia.yaml b/static/bidder-info/verizonmedia.yaml index 0b4642fcb6a..f629c5f7bf5 100644 --- a/static/bidder-info/verizonmedia.yaml +++ b/static/bidder-info/verizonmedia.yaml @@ -8,3 +8,8 @@ capabilities: site: mediaTypes: - banner +userSync: + redirect: + # verizonmedia supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/viewdeos.yaml b/static/bidder-info/viewdeos.yaml index 9483e281de0..3ce7f9cf26a 100644 --- a/static/bidder-info/viewdeos.yaml +++ b/static/bidder-info/viewdeos.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + redirect: + # viewdeos supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/static/bidder-info/vrtcal.yaml b/static/bidder-info/vrtcal.yaml index fcf9294adcb..e86513f38e1 100644 --- a/static/bidder-info/vrtcal.yaml +++ b/static/bidder-info/vrtcal.yaml @@ -4,3 +4,8 @@ capabilities: app: mediaTypes: - banner +userSync: + redirect: + # vrtcal supports user syncing, but requires configuration by the host. paste the + # url provided by the bidder below to enable. + url: "" \ No newline at end of file diff --git a/usersync/syncer.go b/usersync/syncer.go index bd23e621eae..220a0b5fa68 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -58,19 +58,26 @@ const ( setuidSyncTypeRedirect = "i" // i = image response ) -var errEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") -var errKeyRequired = errors.New("key is required") -var errDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") +var ErrSyncerEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") +var ErrSyncerURLRequired = errors.New("each endpoint defined (iframe and/or redirect) must specify a url") +var ErrSyncerKeyRequired = errors.New("key is required") +var ErrSyncerDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") // NewSyncer creates a new Syncer from the provided configuration, or an error if macro substition // fails or an endpoint url is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { if syncerConfig.Key == "" { - return nil, errKeyRequired + return nil, ErrSyncerKeyRequired } if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { - return nil, errEndpointRequired + return nil, ErrSyncerEndpointRequired + } + + // bidders may define an endpoint but leave the url empty as a way to communicate with the host + // that they support this endpoint, but don't provide a default value. + if (syncerConfig.IFrame != nil && syncerConfig.IFrame.URL == "") || (syncerConfig.Redirect != nil && syncerConfig.Redirect.URL == "") { + return nil, ErrSyncerURLRequired } syncer := standardSyncer{ @@ -112,7 +119,7 @@ func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { if syncerConfig.Default == "" { if syncerConfig.IFrame != nil && syncerConfig.Redirect != nil { - return SyncTypeUnknown, errDefaultSyncTypeRequired + return SyncTypeUnknown, ErrSyncerDefaultSyncTypeRequired } else if syncerConfig.IFrame != nil { return SyncTypeIFrame, nil } else { diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 2f4e3203905..05fc8506623 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -17,6 +17,7 @@ func TestNewSyncer(t *testing.T) { supportCORS = true hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} + emptyConfig = &config.SyncerEndpoint{} iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} redirectConfig = &config.SyncerEndpoint{URL: "https://bidder.com/redirect?redirect={{.RedirectURL}}"} errParseConfig = &config.SyncerEndpoint{URL: "{{malformed}}"} @@ -50,6 +51,14 @@ func TestNewSyncer(t *testing.T) { givenRedirectConfig: nil, expectedError: "at least one endpoint (iframe and/or redirect) is required", }, + { + description: "Missing URLs - IFrame & Redirect", + givenKey: "a", + givenDefault: "redirect", + givenIFrameConfig: emptyConfig, + givenRedirectConfig: emptyConfig, + expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", + }, { description: "Resolve Default Sync Type Error ", givenKey: "a", @@ -68,6 +77,14 @@ func TestNewSyncer(t *testing.T) { expectedIFrame: "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fhost", expectedRedirect: "https://bidder.com/redirect?redirect=http%3A%2F%2Fhost.com%2Fhost", }, + { + description: "IFrame - Missing URL Error", + givenKey: "a", + givenDefault: "iframe", + givenIFrameConfig: emptyConfig, + givenRedirectConfig: nil, + expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", + }, { description: "IFrame - Parse Error", givenKey: "a", @@ -84,6 +101,14 @@ func TestNewSyncer(t *testing.T) { givenRedirectConfig: nil, expectedError: "iframe composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", }, + { + description: "Redirect - Missing URL Error", + givenKey: "a", + givenDefault: "redirect", + givenIFrameConfig: nil, + givenRedirectConfig: emptyConfig, + expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", + }, { description: "Redirect - Parse Error", givenKey: "a", diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index bf89b487aa8..96b6b9d219e 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/errortypes" ) type namedSyncerConfig struct { @@ -14,7 +13,19 @@ type namedSyncerConfig struct { cfg config.Syncer } -func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, error) { +// SyncerBuildError represents an error with building a syncer. +type SyncerBuildError struct { + Bidder string + SyncerKey string + Err error +} + +// Error implements the standard error interface. +func (e SyncerBuildError) Error() string { + return fmt.Sprintf("cannot create syncer for bidder %s with key %s: %v", e.Bidder, e.SyncerKey, e.Err) +} + +func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, []error) { // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { @@ -50,7 +61,11 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf syncer, err := NewSyncer(hostUserSyncConfig, primaryCfg.cfg) if err != nil { - errs = append(errs, fmt.Errorf("cannot create syncer for bidder %s with key %s. %v", primaryCfg.name, key, err)) + errs = append(errs, SyncerBuildError{ + Bidder: primaryCfg.name, + SyncerKey: key, + Err: err, + }) continue } @@ -60,7 +75,7 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf } if len(errs) > 0 { - return nil, errortypes.NewAggregateError("user sync", errs) + return nil, errs } return syncers, nil } diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index 29f00b79621..f166c538524 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -1,7 +1,7 @@ package usersync import ( - "strings" + "errors" "testing" "github.com/prebid/prebid-server/config" @@ -9,6 +9,15 @@ import ( "github.com/stretchr/testify/assert" ) +func TestSyncerBuildError(t *testing.T) { + err := SyncerBuildError{ + Bidder: "anyBidder", + SyncerKey: "anyKey", + Err: errors.New("anyError"), + } + assert.Equal(t, err.Error(), "cannot create syncer for bidder anyBidder with key anyKey: anyError") +} + func TestBuildSyncers(t *testing.T) { var ( hostConfig = config.Configuration{ExternalURL: "http://host.com", UserSync: config.UserSync{RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"}} @@ -26,12 +35,11 @@ func TestBuildSyncers(t *testing.T) { // in these tests. Look carefully at the end of the expected iframe urls to see the syncer key. testCases := []struct { - description string - givenConfig config.Configuration - givenBidderInfos config.BidderInfos - expectedIFramesURLs map[string]string - expectedErrorHeader string - expectedErrorSegments []string + description string + givenConfig config.Configuration + givenBidderInfos config.BidderInfos + expectedIFramesURLs map[string]string + expectedErrors []string }{ { description: "One", @@ -50,12 +58,11 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "One - Syncer Error", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "One - Syncer Error", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", }, }, { @@ -77,21 +84,27 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Same Syncers - Many Primaries", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", + description: "Many - Same Syncers - Many Primaries - All Populated", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, + expectedErrors: []string{ + "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", + }, + }, + { + description: "Many - Same Syncers - Many Primaries - None Populated", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAEmpty}, + expectedErrors: []string{ + "bidders bidder1, bidder2 share the same syncer key, but none define endpoints (iframe and/or redirect)", }, }, { - description: "Many - Sync Error - Bidder Correct", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "Many - Sync Error - Bidder Correct", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder2 with key a: default is set to redirect but no redirect endpoint is configured", }, }, { @@ -111,13 +124,12 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Multiple Errors", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, - expectedErrorHeader: "user sync (2 errors)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", - "cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", + description: "Many - Multiple Errors", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", + "cannot create syncer for bidder bidder2 with key b: at least one endpoint (iframe and/or redirect) is required", }, }, { @@ -131,10 +143,10 @@ func TestBuildSyncers(t *testing.T) { } for _, test := range testCases { - result, err := BuildSyncers(&test.givenConfig, test.givenBidderInfos) + result, errs := BuildSyncers(&test.givenConfig, test.givenBidderInfos) - if test.expectedErrorHeader == "" { - assert.NoError(t, err, test.description+":err") + if len(test.expectedErrors) == 0 { + assert.Empty(t, errs, test.description+":err") resultRenderedIFrameURLS := map[string]string{} for k, v := range result { iframeRendered, err := v.GetSync([]SyncType{SyncTypeIFrame}, privacy.Policies{}) @@ -144,11 +156,11 @@ func TestBuildSyncers(t *testing.T) { } assert.Equal(t, test.expectedIFramesURLs, resultRenderedIFrameURLS, test.description+":result") } else { - errMessage := err.Error() - assert.True(t, strings.HasPrefix(errMessage, test.expectedErrorHeader), test.description+":err") - for _, s := range test.expectedErrorSegments { - assert.Contains(t, errMessage, s, test.description+":err") + errMessages := make([]string, 0, len(errs)) + for _, e := range errs { + errMessages = append(errMessages, e.Error()) } + assert.ElementsMatch(t, test.expectedErrors, errMessages, test.description+":err") assert.Empty(t, result, test.description+":result") } } From d7389d3aab5c109fa1183a244a719dfd45008d09 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 16 Aug 2021 15:21:38 -0400 Subject: [PATCH 46/50] Revert "Allow Bidders To Represent Support Without Default" This reverts commit 7d3bbb501d507f1fd7e8ad8ee4113f71f78d9900. --- router/router.go | 30 ++------- router/router_test.go | 53 +-------------- static/bidder-info/adocean.yaml | 5 -- static/bidder-info/adtarget.yaml | 5 -- static/bidder-info/adtelligent.yaml | 5 -- static/bidder-info/adxcg.yaml | 5 -- static/bidder-info/audienceNetwork.yaml | 5 -- static/bidder-info/bmtm.yaml | 5 -- static/bidder-info/criteo.yaml | 7 +- static/bidder-info/dmx.yaml | 5 -- static/bidder-info/gamma.yaml | 5 -- static/bidder-info/invibes.yaml | 5 -- static/bidder-info/lockerdome.yaml | 16 ++--- static/bidder-info/mediafuse.yaml | 5 -- static/bidder-info/rtbhouse.yaml | 5 -- static/bidder-info/rubicon.yaml | 5 -- static/bidder-info/smarthub.yaml | 5 -- static/bidder-info/synacormedia.yaml | 5 -- static/bidder-info/verizonmedia.yaml | 5 -- static/bidder-info/viewdeos.yaml | 5 -- static/bidder-info/vrtcal.yaml | 5 -- usersync/syncer.go | 19 ++---- usersync/syncer_test.go | 25 ------- usersync/syncersbuilder.go | 23 ++----- usersync/syncersbuilder_test.go | 90 +++++++++++-------------- 25 files changed, 67 insertions(+), 281 deletions(-) diff --git a/router/router.go b/router/router.go index e8eb12db734..5d160687b79 100644 --- a/router/router.go +++ b/router/router.go @@ -220,9 +220,9 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } - syncersByBidder, errs := usersync.BuildSyncers(cfg, bidderInfos) - if errsFiltered := filterSyncerErrors(errs); len(errsFiltered) > 0 { - return nil, errortypes.NewAggregateError("user sync", errsFiltered) + syncersByBidder, err := usersync.BuildSyncers(cfg, bidderInfos) + if err != nil { + return nil, err } syncerKeys := make([]string, 0, len(syncersByBidder)) @@ -341,7 +341,7 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg // the new config. if adapterCfg.UserSyncURL != "" { if bidderInfo.Syncer == nil { - return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define a user sync", strings.ToLower(bidderName)) + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync or is disabled", bidderName) } endpointsCount := 0 @@ -355,17 +355,17 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg } if endpointsCount == 0 { - return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define user sync endpoints", strings.ToLower(bidderName)) + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define user sync endpoints", bidderName) } // if the bidder defines both an iframe and redirect endpoint, we can't be sure which config value to // override, and it wouldn't be both. this is a fatal configuration error. if endpointsCount > 1 { - return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder defines multiple user sync endpoints", strings.ToLower(bidderName)) + return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder defines multiple user sync endpoints", bidderName) } // provide a warning that this compatibility layer is temporary - glog.Warningf("adapters.%s.usersync_url is deprecated and will be removed in a future version, please update to the latest user sync config values", strings.ToLower(bidderName)) + glog.Warningf("legacy usersync_url setting for bidder %s will be removed in a future version of Prebid Server. please update to the latest user sync config values", bidderName) } bidderInfos[bidderName] = bidderInfo @@ -374,22 +374,6 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg return nil } -func filterSyncerErrors(errs []error) []error { - var fatalErrors []error - - for _, err := range errs { - syncerBuildErr, ok := err.(usersync.SyncerBuildError) - if ok && syncerBuildErr.Err == usersync.ErrSyncerURLRequired { - bidderName := strings.ToLower(syncerBuildErr.Bidder) - glog.Warningf("bidder %s supports user syncing but has endpoints with no default url, no syncs will be performed with %s.", bidderName, bidderName) - } else { - fatalErrors = append(fatalErrors, err) - } - } - - return fatalErrors -} - // Fixes #648 // // These CORS options pose a security risk... but it's a calculated one. diff --git a/router/router_test.go b/router/router_test.go index f3a167967ce..ec7ab43e946 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -2,7 +2,6 @@ package router import ( "encoding/json" - "errors" "io/ioutil" "net/http" "net/http/httptest" @@ -10,9 +9,7 @@ import ( "testing" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" ) @@ -107,19 +104,19 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { description: "UserSyncURL Override Syncer Not Defined", givenBidderInfos: config.BidderInfos{"a": {}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define a user sync", + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync or is disabled", }, { description: "UserSyncURL Override Syncer Endpoints Not Defined", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define user sync endpoints", + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define user sync endpoints", }, { description: "UserSyncURL Override Ambiguous", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "originalIFrame"}, Redirect: &config.SyncerEndpoint{URL: "originalRedirect"}}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints", + expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder defines multiple user sync endpoints", }, } @@ -134,50 +131,6 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { } } -func TestFilterSyncerErrors(t *testing.T) { - fatalErrorA := errors.New("errorA") - fatalErrorB := &errortypes.BadInput{Message: "errorB"} - buildErrorURLRequired := usersync.SyncerBuildError{Bidder: "anyBidder", SyncerKey: "anyKey", Err: usersync.ErrSyncerURLRequired} - buildErrorOther := usersync.SyncerBuildError{Bidder: "anyBidder", SyncerKey: "anyKey", Err: usersync.ErrSyncerKeyRequired} - - var testCases = []struct { - description string - givenErrors []error - expectedErrors []error - }{ - { - description: "Nil", - givenErrors: nil, - expectedErrors: nil, - }, - { - description: "Empty", - givenErrors: []error{}, - expectedErrors: nil, - }, - { - description: "One - Not Filtered", - givenErrors: []error{fatalErrorA}, - expectedErrors: []error{fatalErrorA}, - }, - { - description: "One - Filtered", - givenErrors: []error{buildErrorURLRequired}, - expectedErrors: nil, - }, - { - description: "Many - Mixed", - givenErrors: []error{fatalErrorA, fatalErrorB, buildErrorURLRequired, buildErrorOther}, - expectedErrors: []error{fatalErrorA, fatalErrorB, buildErrorOther}, - }, - } - - for _, test := range testCases { - filteredErrs := filterSyncerErrors(test.givenErrors) - assert.Equal(t, test.expectedErrors, filteredErrs, test.description) - } -} - // Prevents #648 func TestCORSSupport(t *testing.T) { const origin = "https://publisher-domain.com" diff --git a/static/bidder-info/adocean.yaml b/static/bidder-info/adocean.yaml index e53dcc32a5f..6374d752c34 100644 --- a/static/bidder-info/adocean.yaml +++ b/static/bidder-info/adocean.yaml @@ -8,8 +8,3 @@ capabilities: site: mediaTypes: - banner -userSync: - redirect: - # adocean supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/adtarget.yaml b/static/bidder-info/adtarget.yaml index 694ed139095..cc064b9ca6b 100644 --- a/static/bidder-info/adtarget.yaml +++ b/static/bidder-info/adtarget.yaml @@ -10,8 +10,3 @@ capabilities: mediaTypes: - banner - video -userSync: - iframe: - # adtarget supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/adtelligent.yaml b/static/bidder-info/adtelligent.yaml index 6cb2c514743..30f55ea912e 100644 --- a/static/bidder-info/adtelligent.yaml +++ b/static/bidder-info/adtelligent.yaml @@ -10,8 +10,3 @@ capabilities: mediaTypes: - banner - video -userSync: - iframe: - # adtelligent supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/adxcg.yaml b/static/bidder-info/adxcg.yaml index d2b15524310..e5535ae7258 100644 --- a/static/bidder-info/adxcg.yaml +++ b/static/bidder-info/adxcg.yaml @@ -11,8 +11,3 @@ capabilities: - banner - video - native -userSync: - redirect: - # adxcg supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/audienceNetwork.yaml b/static/bidder-info/audienceNetwork.yaml index 451959f7ec5..324e5c6dff8 100644 --- a/static/bidder-info/audienceNetwork.yaml +++ b/static/bidder-info/audienceNetwork.yaml @@ -6,8 +6,3 @@ capabilities: - banner - video - native -userSync: - redirect: - # facebook's audience network supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/bmtm.yaml b/static/bidder-info/bmtm.yaml index 526e7d857af..c13bf17db73 100644 --- a/static/bidder-info/bmtm.yaml +++ b/static/bidder-info/bmtm.yaml @@ -6,8 +6,3 @@ capabilities: mediaTypes: - banner - video -userSync: - redirect: - # bmtm supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/criteo.yaml b/static/bidder-info/criteo.yaml index 0898e5938ec..bfa098ba39d 100644 --- a/static/bidder-info/criteo.yaml +++ b/static/bidder-info/criteo.yaml @@ -7,9 +7,4 @@ capabilities: - banner site: mediaTypes: - - banner -userSync: - redirect: - # criteo supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file + - banner \ No newline at end of file diff --git a/static/bidder-info/dmx.yaml b/static/bidder-info/dmx.yaml index 04e3612512b..d29d699daeb 100644 --- a/static/bidder-info/dmx.yaml +++ b/static/bidder-info/dmx.yaml @@ -10,8 +10,3 @@ capabilities: mediaTypes: - banner - video -userSync: - redirect: - # dmx supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/gamma.yaml b/static/bidder-info/gamma.yaml index ef230dc30c0..aedaf2cf749 100644 --- a/static/bidder-info/gamma.yaml +++ b/static/bidder-info/gamma.yaml @@ -5,8 +5,3 @@ capabilities: mediaTypes: - banner - video -userSync: - iframe: - # gamma supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/invibes.yaml b/static/bidder-info/invibes.yaml index 1da3c05ffbe..45184fb9f65 100644 --- a/static/bidder-info/invibes.yaml +++ b/static/bidder-info/invibes.yaml @@ -5,8 +5,3 @@ capabilities: site: mediaTypes: - banner -userSync: - iframe: - # invibes supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/lockerdome.yaml b/static/bidder-info/lockerdome.yaml index 38e3e346be7..eebf3b87841 100644 --- a/static/bidder-info/lockerdome.yaml +++ b/static/bidder-info/lockerdome.yaml @@ -7,12 +7,10 @@ capabilities: site: mediaTypes: - banner -userSync: - redirect: - # lockerdome supports user syncing, but requires a platform id to be configured by - # the host. remove the empty url, replace <> with the value provided, and - # uncomment to enable. - # - # url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" - url: "" - userMacro: "{{uid}}" +# lockerdome requires a platform id for their user sync process. replace <> +# in the url below and uncomment the userSync section to enable. +# +#userSync: +# redirect: +# url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" +# userMacro: "{{uid}}" diff --git a/static/bidder-info/mediafuse.yaml b/static/bidder-info/mediafuse.yaml index 5e2cedc7932..98611402905 100644 --- a/static/bidder-info/mediafuse.yaml +++ b/static/bidder-info/mediafuse.yaml @@ -10,8 +10,3 @@ capabilities: mediaTypes: - banner - video -userSync: - iframe: - # mediafuse supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/rtbhouse.yaml b/static/bidder-info/rtbhouse.yaml index 58196af4c3f..12744fdc75e 100644 --- a/static/bidder-info/rtbhouse.yaml +++ b/static/bidder-info/rtbhouse.yaml @@ -5,8 +5,3 @@ capabilities: site: mediaTypes: - banner -userSync: - redirect: - # rtbhouse supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/rubicon.yaml b/static/bidder-info/rubicon.yaml index 351918ca3cd..0f19ddb9627 100644 --- a/static/bidder-info/rubicon.yaml +++ b/static/bidder-info/rubicon.yaml @@ -9,8 +9,3 @@ capabilities: mediaTypes: - banner - video -userSync: - redirect: - # rubicon supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/smarthub.yaml b/static/bidder-info/smarthub.yaml index 32a7a1d0c40..bfee9490840 100644 --- a/static/bidder-info/smarthub.yaml +++ b/static/bidder-info/smarthub.yaml @@ -11,8 +11,3 @@ capabilities: - banner - video - native -userSync: - redirect: - # smarthub supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/synacormedia.yaml b/static/bidder-info/synacormedia.yaml index 774ba35720f..14a4bcd1244 100644 --- a/static/bidder-info/synacormedia.yaml +++ b/static/bidder-info/synacormedia.yaml @@ -13,8 +13,3 @@ userSync: iframe: url: "https://sync.technoratimedia.com/services?srv=cs&pid=70&cb={{.RedirectURL}}" userMacro: "[USER_ID]" -userSync: - iframe: - # synacormedia supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/verizonmedia.yaml b/static/bidder-info/verizonmedia.yaml index f629c5f7bf5..0b4642fcb6a 100644 --- a/static/bidder-info/verizonmedia.yaml +++ b/static/bidder-info/verizonmedia.yaml @@ -8,8 +8,3 @@ capabilities: site: mediaTypes: - banner -userSync: - redirect: - # verizonmedia supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/viewdeos.yaml b/static/bidder-info/viewdeos.yaml index 3ce7f9cf26a..9483e281de0 100644 --- a/static/bidder-info/viewdeos.yaml +++ b/static/bidder-info/viewdeos.yaml @@ -10,8 +10,3 @@ capabilities: mediaTypes: - banner - video -userSync: - redirect: - # viewdeos supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/static/bidder-info/vrtcal.yaml b/static/bidder-info/vrtcal.yaml index e86513f38e1..fcf9294adcb 100644 --- a/static/bidder-info/vrtcal.yaml +++ b/static/bidder-info/vrtcal.yaml @@ -4,8 +4,3 @@ capabilities: app: mediaTypes: - banner -userSync: - redirect: - # vrtcal supports user syncing, but requires configuration by the host. paste the - # url provided by the bidder below to enable. - url: "" \ No newline at end of file diff --git a/usersync/syncer.go b/usersync/syncer.go index 220a0b5fa68..bd23e621eae 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -58,26 +58,19 @@ const ( setuidSyncTypeRedirect = "i" // i = image response ) -var ErrSyncerEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") -var ErrSyncerURLRequired = errors.New("each endpoint defined (iframe and/or redirect) must specify a url") -var ErrSyncerKeyRequired = errors.New("key is required") -var ErrSyncerDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") +var errEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") +var errKeyRequired = errors.New("key is required") +var errDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") // NewSyncer creates a new Syncer from the provided configuration, or an error if macro substition // fails or an endpoint url is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { if syncerConfig.Key == "" { - return nil, ErrSyncerKeyRequired + return nil, errKeyRequired } if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { - return nil, ErrSyncerEndpointRequired - } - - // bidders may define an endpoint but leave the url empty as a way to communicate with the host - // that they support this endpoint, but don't provide a default value. - if (syncerConfig.IFrame != nil && syncerConfig.IFrame.URL == "") || (syncerConfig.Redirect != nil && syncerConfig.Redirect.URL == "") { - return nil, ErrSyncerURLRequired + return nil, errEndpointRequired } syncer := standardSyncer{ @@ -119,7 +112,7 @@ func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { if syncerConfig.Default == "" { if syncerConfig.IFrame != nil && syncerConfig.Redirect != nil { - return SyncTypeUnknown, ErrSyncerDefaultSyncTypeRequired + return SyncTypeUnknown, errDefaultSyncTypeRequired } else if syncerConfig.IFrame != nil { return SyncTypeIFrame, nil } else { diff --git a/usersync/syncer_test.go b/usersync/syncer_test.go index 05fc8506623..2f4e3203905 100644 --- a/usersync/syncer_test.go +++ b/usersync/syncer_test.go @@ -17,7 +17,6 @@ func TestNewSyncer(t *testing.T) { supportCORS = true hostConfig = config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} macroValues = macros.UserSyncTemplateParams{GDPR: "A", GDPRConsent: "B", USPrivacy: "C"} - emptyConfig = &config.SyncerEndpoint{} iframeConfig = &config.SyncerEndpoint{URL: "https://bidder.com/iframe?redirect={{.RedirectURL}}"} redirectConfig = &config.SyncerEndpoint{URL: "https://bidder.com/redirect?redirect={{.RedirectURL}}"} errParseConfig = &config.SyncerEndpoint{URL: "{{malformed}}"} @@ -51,14 +50,6 @@ func TestNewSyncer(t *testing.T) { givenRedirectConfig: nil, expectedError: "at least one endpoint (iframe and/or redirect) is required", }, - { - description: "Missing URLs - IFrame & Redirect", - givenKey: "a", - givenDefault: "redirect", - givenIFrameConfig: emptyConfig, - givenRedirectConfig: emptyConfig, - expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", - }, { description: "Resolve Default Sync Type Error ", givenKey: "a", @@ -77,14 +68,6 @@ func TestNewSyncer(t *testing.T) { expectedIFrame: "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fhost", expectedRedirect: "https://bidder.com/redirect?redirect=http%3A%2F%2Fhost.com%2Fhost", }, - { - description: "IFrame - Missing URL Error", - givenKey: "a", - givenDefault: "iframe", - givenIFrameConfig: emptyConfig, - givenRedirectConfig: nil, - expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", - }, { description: "IFrame - Parse Error", givenKey: "a", @@ -101,14 +84,6 @@ func TestNewSyncer(t *testing.T) { givenRedirectConfig: nil, expectedError: "iframe composed url: \"notAURL:http%3A%2F%2Fhost.com%2Fhost\" is invalid", }, - { - description: "Redirect - Missing URL Error", - givenKey: "a", - givenDefault: "redirect", - givenIFrameConfig: nil, - givenRedirectConfig: emptyConfig, - expectedError: "each endpoint defined (iframe and/or redirect) must specify a url", - }, { description: "Redirect - Parse Error", givenKey: "a", diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index 96b6b9d219e..bf89b487aa8 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" ) type namedSyncerConfig struct { @@ -13,19 +14,7 @@ type namedSyncerConfig struct { cfg config.Syncer } -// SyncerBuildError represents an error with building a syncer. -type SyncerBuildError struct { - Bidder string - SyncerKey string - Err error -} - -// Error implements the standard error interface. -func (e SyncerBuildError) Error() string { - return fmt.Sprintf("cannot create syncer for bidder %s with key %s: %v", e.Bidder, e.SyncerKey, e.Err) -} - -func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, []error) { +func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, error) { // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { @@ -61,11 +50,7 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf syncer, err := NewSyncer(hostUserSyncConfig, primaryCfg.cfg) if err != nil { - errs = append(errs, SyncerBuildError{ - Bidder: primaryCfg.name, - SyncerKey: key, - Err: err, - }) + errs = append(errs, fmt.Errorf("cannot create syncer for bidder %s with key %s. %v", primaryCfg.name, key, err)) continue } @@ -75,7 +60,7 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf } if len(errs) > 0 { - return nil, errs + return nil, errortypes.NewAggregateError("user sync", errs) } return syncers, nil } diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index f166c538524..29f00b79621 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -1,7 +1,7 @@ package usersync import ( - "errors" + "strings" "testing" "github.com/prebid/prebid-server/config" @@ -9,15 +9,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSyncerBuildError(t *testing.T) { - err := SyncerBuildError{ - Bidder: "anyBidder", - SyncerKey: "anyKey", - Err: errors.New("anyError"), - } - assert.Equal(t, err.Error(), "cannot create syncer for bidder anyBidder with key anyKey: anyError") -} - func TestBuildSyncers(t *testing.T) { var ( hostConfig = config.Configuration{ExternalURL: "http://host.com", UserSync: config.UserSync{RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"}} @@ -35,11 +26,12 @@ func TestBuildSyncers(t *testing.T) { // in these tests. Look carefully at the end of the expected iframe urls to see the syncer key. testCases := []struct { - description string - givenConfig config.Configuration - givenBidderInfos config.BidderInfos - expectedIFramesURLs map[string]string - expectedErrors []string + description string + givenConfig config.Configuration + givenBidderInfos config.BidderInfos + expectedIFramesURLs map[string]string + expectedErrorHeader string + expectedErrorSegments []string }{ { description: "One", @@ -58,11 +50,12 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "One - Syncer Error", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, - expectedErrors: []string{ - "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", + description: "One - Syncer Error", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", }, }, { @@ -84,27 +77,21 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Same Syncers - Many Primaries - All Populated", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, - expectedErrors: []string{ - "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", - }, - }, - { - description: "Many - Same Syncers - Many Primaries - None Populated", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAEmpty}, - expectedErrors: []string{ - "bidders bidder1, bidder2 share the same syncer key, but none define endpoints (iframe and/or redirect)", + description: "Many - Same Syncers - Many Primaries", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", }, }, { - description: "Many - Sync Error - Bidder Correct", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, - expectedErrors: []string{ - "cannot create syncer for bidder bidder2 with key a: default is set to redirect but no redirect endpoint is configured", + description: "Many - Sync Error - Bidder Correct", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, + expectedErrorHeader: "user sync (1 error)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", }, }, { @@ -124,12 +111,13 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Multiple Errors", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, - expectedErrors: []string{ - "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", - "cannot create syncer for bidder bidder2 with key b: at least one endpoint (iframe and/or redirect) is required", + description: "Many - Multiple Errors", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, + expectedErrorHeader: "user sync (2 errors)", + expectedErrorSegments: []string{ + "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + "cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", }, }, { @@ -143,10 +131,10 @@ func TestBuildSyncers(t *testing.T) { } for _, test := range testCases { - result, errs := BuildSyncers(&test.givenConfig, test.givenBidderInfos) + result, err := BuildSyncers(&test.givenConfig, test.givenBidderInfos) - if len(test.expectedErrors) == 0 { - assert.Empty(t, errs, test.description+":err") + if test.expectedErrorHeader == "" { + assert.NoError(t, err, test.description+":err") resultRenderedIFrameURLS := map[string]string{} for k, v := range result { iframeRendered, err := v.GetSync([]SyncType{SyncTypeIFrame}, privacy.Policies{}) @@ -156,11 +144,11 @@ func TestBuildSyncers(t *testing.T) { } assert.Equal(t, test.expectedIFramesURLs, resultRenderedIFrameURLS, test.description+":result") } else { - errMessages := make([]string, 0, len(errs)) - for _, e := range errs { - errMessages = append(errMessages, e.Error()) + errMessage := err.Error() + assert.True(t, strings.HasPrefix(errMessage, test.expectedErrorHeader), test.description+":err") + for _, s := range test.expectedErrorSegments { + assert.Contains(t, errMessage, s, test.description+":err") } - assert.ElementsMatch(t, test.expectedErrors, errMessages, test.description+":err") assert.Empty(t, result, test.description+":result") } } From 745eb3acd60fb48bcc2cd194070b2a1a9e7877c7 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Mon, 16 Aug 2021 15:41:37 -0400 Subject: [PATCH 47/50] Improved Errors --- router/router.go | 14 ++--- router/router_test.go | 6 +-- usersync/syncer.go | 12 ++--- usersync/syncersbuilder.go | 23 +++++++-- usersync/syncersbuilder_test.go | 90 +++++++++++++++++++-------------- 5 files changed, 86 insertions(+), 59 deletions(-) diff --git a/router/router.go b/router/router.go index 5d160687b79..6a822f5ef8c 100644 --- a/router/router.go +++ b/router/router.go @@ -220,9 +220,9 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } - syncersByBidder, err := usersync.BuildSyncers(cfg, bidderInfos) - if err != nil { - return nil, err + syncersByBidder, errs := usersync.BuildSyncers(cfg, bidderInfos) + if len(errs) > 0 { + return nil, errortypes.NewAggregateError("user sync", errs) } syncerKeys := make([]string, 0, len(syncersByBidder)) @@ -341,7 +341,7 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg // the new config. if adapterCfg.UserSyncURL != "" { if bidderInfo.Syncer == nil { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define a user sync or is disabled", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define a user sync", strings.ToLower(bidderName)) } endpointsCount := 0 @@ -355,17 +355,17 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg } if endpointsCount == 0 { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder does not define user sync endpoints", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define user sync endpoints", strings.ToLower(bidderName)) } // if the bidder defines both an iframe and redirect endpoint, we can't be sure which config value to // override, and it wouldn't be both. this is a fatal configuration error. if endpointsCount > 1 { - return fmt.Errorf("failed to apply legacy usersync_url setting for bidder %s, bidder defines multiple user sync endpoints", bidderName) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder defines multiple user sync endpoints", strings.ToLower(bidderName)) } // provide a warning that this compatibility layer is temporary - glog.Warningf("legacy usersync_url setting for bidder %s will be removed in a future version of Prebid Server. please update to the latest user sync config values", bidderName) + glog.Warningf("adapters.%s.usersync_url is deprecated and will be removed in a future version, please update to the latest user sync config values", strings.ToLower(bidderName)) } bidderInfos[bidderName] = bidderInfo diff --git a/router/router_test.go b/router/router_test.go index ec7ab43e946..475e504da62 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -104,19 +104,19 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { description: "UserSyncURL Override Syncer Not Defined", givenBidderInfos: config.BidderInfos{"a": {}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define a user sync or is disabled", + expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define a user sync", }, { description: "UserSyncURL Override Syncer Endpoints Not Defined", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder does not define user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define user sync endpoints", }, { description: "UserSyncURL Override Ambiguous", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "originalIFrame"}, Redirect: &config.SyncerEndpoint{URL: "originalRedirect"}}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "failed to apply legacy usersync_url setting for bidder a, bidder defines multiple user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints", }, } diff --git a/usersync/syncer.go b/usersync/syncer.go index bd23e621eae..2db220a2340 100644 --- a/usersync/syncer.go +++ b/usersync/syncer.go @@ -58,19 +58,19 @@ const ( setuidSyncTypeRedirect = "i" // i = image response ) -var errEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") -var errKeyRequired = errors.New("key is required") -var errDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") +var ErrSyncerEndpointRequired = errors.New("at least one endpoint (iframe and/or redirect) is required") +var ErrSyncerKeyRequired = errors.New("key is required") +var ErrSyncerDefaultSyncTypeRequired = errors.New("default sync type is required when more then one sync endpoint is configured") // NewSyncer creates a new Syncer from the provided configuration, or an error if macro substition // fails or an endpoint url is invalid. func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, error) { if syncerConfig.Key == "" { - return nil, errKeyRequired + return nil, ErrSyncerKeyRequired } if syncerConfig.IFrame == nil && syncerConfig.Redirect == nil { - return nil, errEndpointRequired + return nil, ErrSyncerEndpointRequired } syncer := standardSyncer{ @@ -112,7 +112,7 @@ func NewSyncer(hostConfig config.UserSync, syncerConfig config.Syncer) (Syncer, func resolveDefaultSyncType(syncerConfig config.Syncer) (SyncType, error) { if syncerConfig.Default == "" { if syncerConfig.IFrame != nil && syncerConfig.Redirect != nil { - return SyncTypeUnknown, errDefaultSyncTypeRequired + return SyncTypeUnknown, ErrSyncerDefaultSyncTypeRequired } else if syncerConfig.IFrame != nil { return SyncTypeIFrame, nil } else { diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index bf89b487aa8..96b6b9d219e 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/errortypes" ) type namedSyncerConfig struct { @@ -14,7 +13,19 @@ type namedSyncerConfig struct { cfg config.Syncer } -func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, error) { +// SyncerBuildError represents an error with building a syncer. +type SyncerBuildError struct { + Bidder string + SyncerKey string + Err error +} + +// Error implements the standard error interface. +func (e SyncerBuildError) Error() string { + return fmt.Sprintf("cannot create syncer for bidder %s with key %s: %v", e.Bidder, e.SyncerKey, e.Err) +} + +func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInfos) (map[string]Syncer, []error) { // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { @@ -50,7 +61,11 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf syncer, err := NewSyncer(hostUserSyncConfig, primaryCfg.cfg) if err != nil { - errs = append(errs, fmt.Errorf("cannot create syncer for bidder %s with key %s. %v", primaryCfg.name, key, err)) + errs = append(errs, SyncerBuildError{ + Bidder: primaryCfg.name, + SyncerKey: key, + Err: err, + }) continue } @@ -60,7 +75,7 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf } if len(errs) > 0 { - return nil, errortypes.NewAggregateError("user sync", errs) + return nil, errs } return syncers, nil } diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index 29f00b79621..be3b7d705cd 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -1,7 +1,7 @@ package usersync import ( - "strings" + "errors" "testing" "github.com/prebid/prebid-server/config" @@ -9,6 +9,15 @@ import ( "github.com/stretchr/testify/assert" ) +func TestSyncerBuildError(t *testing.T) { + err := SyncerBuildError{ + Bidder: "anyBidder", + SyncerKey: "anyKey", + Err: errors.New("anyError"), + } + assert.Equal(t, err.Error(), "cannot create syncer for bidder anyBidder with key anyKey: anyError") +} + func TestBuildSyncers(t *testing.T) { var ( hostConfig = config.Configuration{ExternalURL: "http://host.com", UserSync: config.UserSync{RedirectURL: "{{.ExternalURL}}/{{.SyncerKey}}/host"}} @@ -26,12 +35,11 @@ func TestBuildSyncers(t *testing.T) { // in these tests. Look carefully at the end of the expected iframe urls to see the syncer key. testCases := []struct { - description string - givenConfig config.Configuration - givenBidderInfos config.BidderInfos - expectedIFramesURLs map[string]string - expectedErrorHeader string - expectedErrorSegments []string + description string + givenConfig config.Configuration + givenBidderInfos config.BidderInfos + expectedIFramesURLs map[string]string + expectedErrors []string }{ { description: "One", @@ -50,12 +58,11 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "One - Syncer Error", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "One - Syncer Error", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", }, }, { @@ -77,21 +84,27 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Same Syncers - Many Primaries", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints\n", + description: "Many - Same Syncers - Many Primaries", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAPopulated, "bidder2": infoKeyAPopulated}, + expectedErrors: []string{ + "bidders bidder1, bidder2 define endpoints (iframe and/or redirect) for the same syncer key, but only one bidder is permitted to define endpoints", + }, + }, + { + description: "Many - Same Syncers - Many Primaries - None Populated", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAEmpty}, + expectedErrors: []string{ + "bidders bidder1, bidder2 share the same syncer key, but none define endpoints (iframe and/or redirect)", }, }, { - description: "Many - Sync Error - Bidder Correct", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, - expectedErrorHeader: "user sync (1 error)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder2 with key a. default is set to redirect but no redirect endpoint is configured\n", + description: "Many - Sync Error - Bidder Correct", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAEmpty, "bidder2": infoKeyAError}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder2 with key a: default is set to redirect but no redirect endpoint is configured", }, }, { @@ -111,13 +124,12 @@ func TestBuildSyncers(t *testing.T) { }, }, { - description: "Many - Multiple Errors", - givenConfig: hostConfig, - givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, - expectedErrorHeader: "user sync (2 errors)", - expectedErrorSegments: []string{ - "cannot create syncer for bidder bidder1 with key a. default is set to redirect but no redirect endpoint is configured\n", - "cannot create syncer for bidder bidder2 with key b. at least one endpoint (iframe and/or redirect) is required\n", + description: "Many - Multiple Errors", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyAError, "bidder2": infoKeyBEmpty}, + expectedErrors: []string{ + "cannot create syncer for bidder bidder1 with key a: default is set to redirect but no redirect endpoint is configured", + "cannot create syncer for bidder bidder2 with key b: at least one endpoint (iframe and/or redirect) is required", }, }, { @@ -131,10 +143,10 @@ func TestBuildSyncers(t *testing.T) { } for _, test := range testCases { - result, err := BuildSyncers(&test.givenConfig, test.givenBidderInfos) + result, errs := BuildSyncers(&test.givenConfig, test.givenBidderInfos) - if test.expectedErrorHeader == "" { - assert.NoError(t, err, test.description+":err") + if len(test.expectedErrors) == 0 { + assert.Empty(t, errs, test.description+":err") resultRenderedIFrameURLS := map[string]string{} for k, v := range result { iframeRendered, err := v.GetSync([]SyncType{SyncTypeIFrame}, privacy.Policies{}) @@ -144,11 +156,11 @@ func TestBuildSyncers(t *testing.T) { } assert.Equal(t, test.expectedIFramesURLs, resultRenderedIFrameURLS, test.description+":result") } else { - errMessage := err.Error() - assert.True(t, strings.HasPrefix(errMessage, test.expectedErrorHeader), test.description+":err") - for _, s := range test.expectedErrorSegments { - assert.Contains(t, errMessage, s, test.description+":err") + errMessages := make([]string, 0, len(errs)) + for _, e := range errs { + errMessages = append(errMessages, e.Error()) } + assert.ElementsMatch(t, test.expectedErrors, errMessages, test.description+":err") assert.Empty(t, result, test.description+":result") } } From c12d936055920073e306a381e146980920bac924 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 17 Aug 2021 16:35:31 -0400 Subject: [PATCH 48/50] User Sync Supports Field --- config/bidderinfo.go | 11 ++- config/bidderinfo_validate_test.go | 55 ++++++----- router/router.go | 17 +++- router/router_test.go | 22 ++++- static/bidder-info/adocean.yaml | 5 + static/bidder-info/adtarget.yaml | 5 + static/bidder-info/adtelligent.yaml | 5 + static/bidder-info/adxcg.yaml | 5 + static/bidder-info/audienceNetwork.yaml | 4 + static/bidder-info/bmtm.yaml | 5 + static/bidder-info/criteo.yaml | 7 +- static/bidder-info/dmx.yaml | 5 + static/bidder-info/gamma.yaml | 5 + static/bidder-info/invibes.yaml | 5 + static/bidder-info/lockerdome.yaml | 26 ++++-- static/bidder-info/mediafuse.yaml | 5 + static/bidder-info/rtbhouse.yaml | 5 + static/bidder-info/rubicon.yaml | 5 + static/bidder-info/smarthub.yaml | 5 + static/bidder-info/synacormedia.yaml | 5 + static/bidder-info/verizonmedia.yaml | 5 + static/bidder-info/viewdeos.yaml | 5 + static/bidder-info/vrtcal.yaml | 5 + usersync/syncersbuilder.go | 16 +++- usersync/syncersbuilder_test.go | 119 ++++++++++++++++++++++++ util/sliceutil/sliceutil.go | 14 +++ util/sliceutil/sliceutil_test.go | 70 ++++++++++++++ 27 files changed, 405 insertions(+), 36 deletions(-) create mode 100644 util/sliceutil/sliceutil.go create mode 100644 util/sliceutil/sliceutil_test.go diff --git a/config/bidderinfo.go b/config/bidderinfo.go index e6fecb7bffa..750604c31b3 100644 --- a/config/bidderinfo.go +++ b/config/bidderinfo.go @@ -56,6 +56,11 @@ type Syncer struct { // `iframe` and `redirect`. Default string `yaml:"default" mapstructure:"default"` + // Supports allows bidders to specify which user sync endpoints they support but which don't have + // good defaults. Host companies should contact the bidder for the endpoint configuration. Hosts + // may not override this value. + Supports []string `yaml:"supports"` + // IFrame configures an iframe endpoint for user syncing. IFrame *SyncerEndpoint `yaml:"iframe" mapstructure:"iframe"` @@ -67,6 +72,9 @@ type Syncer struct { SupportCORS *bool `yaml:"supportCors" mapstructure:"support_cors"` } +// Override returns a new Syncer object where values in the original are replaced by non-empty/non-default +// values in the override, except for the Supports field which may not be overridden. No changes are made +// to the original or override Syncer. func (s *Syncer) Override(original *Syncer) *Syncer { if s == nil && original == nil { return nil @@ -163,7 +171,8 @@ type SyncerEndpoint struct { UserMacro string `yaml:"userMacro" mapstructure:"user_macro"` } -// Override returns a new SyncerEndpoint with original values overridden by non empty values. +// Override returns a new SyncerEndpoint object where values in the original are replaced by non-empty/non-default +// values in the override. No changes are made to the original or override SyncerEndpoint. func (s *SyncerEndpoint) Override(original *SyncerEndpoint) *SyncerEndpoint { if s == nil && original == nil { return nil diff --git a/config/bidderinfo_validate_test.go b/config/bidderinfo_validate_test.go index b76c146fa1b..6a1b18ab092 100644 --- a/config/bidderinfo_validate_test.go +++ b/config/bidderinfo_validate_test.go @@ -5,12 +5,12 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "strings" "testing" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" - "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/usersync" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v2" @@ -38,7 +38,8 @@ func TestBidderInfoFiles(t *testing.T) { expectedFileInfosLength := len(openrtb_ext.CoreBidderNames()) assert.Len(t, fileInfos, expectedFileInfosLength, "static/bidder-info contains %d files, but there are %d known bidders. Did you forget to add a YAML file for your bidder?", len(fileInfos), expectedFileInfosLength) - // Validate Contents + // Load & Validate Contents + bidderInfos := make(config.BidderInfos) for _, fileInfo := range fileInfos { path := fmt.Sprintf(bidderInfoRelativePath + "/" + fileInfo.Name()) @@ -54,7 +55,16 @@ func TestBidderInfoFiles(t *testing.T) { err = validateInfo(&fileInfoContent) assert.NoError(t, err, "Invalid content in static/bidder-info/%s: %v", fileInfo.Name(), err) + + err = validateSyncer(fileInfoContent) + assert.NoError(t, err, "Invalid syncer config in static/bidder-info/%s: %v", fileInfo.Name(), err) + + fileNameWithoutExtension := fileInfo.Name()[:len(fileInfo.Name())-len(filepath.Ext(fileInfo.Name()))] + bidderInfos[fileNameWithoutExtension] = fileInfoContent } + + errs := validateSyncers(t, bidderInfos) + assert.Empty(t, errs, "syncer errors") } func validateInfo(info *config.BidderInfo) error { @@ -66,10 +76,6 @@ func validateInfo(info *config.BidderInfo) error { return err } - if err := validateSyncer(info.Syncer); err != nil { - return err - } - return nil } @@ -121,28 +127,33 @@ func validatePlatformInfo(info *config.PlatformInfo) error { return nil } -func validateSyncer(syncerCfg *config.Syncer) error { - if syncerCfg == nil { - return nil +func validateSyncers(t *testing.T, bidderInfos config.BidderInfos) []error { + hostConfig := &config.Configuration{ + UserSync: config.UserSync{ + ExternalURL: "http://host.com", + RedirectURL: "{{.ExternalURL}}/host", + }, } - hostConfig := config.UserSync{ExternalURL: "http://host.com", RedirectURL: "{{.ExternalURL}}/host"} - - // emulate run time substitution of bidder name for empty keys - if syncerCfg.Key == "" { - syncerCfg.Key = "bidder" + // enable all bidders to allow BuildSyncers to build all syncers + for k, v := range bidderInfos { + v.Enabled = true + bidderInfos[k] = v } - syncer, err := usersync.NewSyncer(hostConfig, *syncerCfg) - if err != nil { - return fmt.Errorf("syncer could not be created: %s", err) + _, errs := usersync.BuildSyncers(hostConfig, bidderInfos) + return errs +} + +func validateSyncer(bidderInfo config.BidderInfo) error { + if bidderInfo.Syncer == nil { + return nil } - // ensure final macro substitution - privacyPolicies := privacy.Policies{} - _, err = syncer.GetSync([]usersync.SyncType{usersync.SyncTypeIFrame, usersync.SyncTypeRedirect}, privacyPolicies) - if err != nil { - return fmt.Errorf("syncer has invalid macro: %s", err) + for _, v := range bidderInfo.Syncer.Supports { + if !strings.EqualFold(v, "iframe") && !strings.EqualFold(v, "redirect") { + return fmt.Errorf("syncer could not be created, invalid supported endpoint: %s", v) + } } return nil diff --git a/router/router.go b/router/router.go index 6a822f5ef8c..25ad26944db 100644 --- a/router/router.go +++ b/router/router.go @@ -44,6 +44,7 @@ import ( "github.com/prebid/prebid-server/server/ssl" storedRequestsConf "github.com/prebid/prebid-server/stored_requests/config" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/sliceutil" "github.com/golang/glog" "github.com/julienschmidt/httprouter" @@ -354,14 +355,26 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg endpointsCount++ } + // use Supports as a hint if there are no good defaults provided if endpointsCount == 0 { - return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define user sync endpoints", strings.ToLower(bidderName)) + if sliceutil.ContainsStringIgnoreCase(bidderInfo.Syncer.Supports, "iframe") { + bidderInfo.Syncer.IFrame = &config.SyncerEndpoint{URL: adapterCfg.UserSyncURL} + endpointsCount++ + } + if sliceutil.ContainsStringIgnoreCase(bidderInfo.Syncer.Supports, "redirect") { + bidderInfo.Syncer.Redirect = &config.SyncerEndpoint{URL: adapterCfg.UserSyncURL} + endpointsCount++ + } + } + + if endpointsCount == 0 { + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder does not define user sync endpoints and does not define supported endpoints", strings.ToLower(bidderName)) } // if the bidder defines both an iframe and redirect endpoint, we can't be sure which config value to // override, and it wouldn't be both. this is a fatal configuration error. if endpointsCount > 1 { - return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder defines multiple user sync endpoints", strings.ToLower(bidderName)) + return fmt.Errorf("adapters.%s.usersync_url cannot be applied, bidder defines multiple user sync endpoints or supports multiple endpoints", strings.ToLower(bidderName)) } // provide a warning that this compatibility layer is temporary diff --git a/router/router_test.go b/router/router_test.go index 475e504da62..fc90fdf01f3 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -94,8 +94,20 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "override"}}}}, }, + { + description: "UserSyncURL Supports IFrame", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Supports: []string{"iframe"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Supports: []string{"iframe"}, IFrame: &config.SyncerEndpoint{URL: "override"}}}}, + }, { description: "UserSyncURL Override Redirect", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Supports: []string{"redirect"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Supports: []string{"redirect"}, Redirect: &config.SyncerEndpoint{URL: "override"}}}}, + }, + { + description: "UserSyncURL Supports Redirect", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Redirect: &config.SyncerEndpoint{URL: "original"}}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, expectedBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Redirect: &config.SyncerEndpoint{URL: "override"}}}}, @@ -110,13 +122,19 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { description: "UserSyncURL Override Syncer Endpoints Not Defined", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder does not define user sync endpoints and does not define supported endpoints", }, { description: "UserSyncURL Override Ambiguous", givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{IFrame: &config.SyncerEndpoint{URL: "originalIFrame"}, Redirect: &config.SyncerEndpoint{URL: "originalRedirect"}}}}, givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, - expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints", + expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints or supports multiple endpoints", + }, + { + description: "UserSyncURL Supports Ambiguous", + givenBidderInfos: config.BidderInfos{"a": {Syncer: &config.Syncer{Supports: []string{"iframe", "redirect"}}}}, + givenAdaptersCfg: map[string]config.Adapter{"a": {UserSyncURL: "override"}}, + expectedError: "adapters.a.usersync_url cannot be applied, bidder defines multiple user sync endpoints or supports multiple endpoints", }, } diff --git a/static/bidder-info/adocean.yaml b/static/bidder-info/adocean.yaml index 6374d752c34..680e7496725 100644 --- a/static/bidder-info/adocean.yaml +++ b/static/bidder-info/adocean.yaml @@ -8,3 +8,8 @@ capabilities: site: mediaTypes: - banner +userSync: + # adocean supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/adtarget.yaml b/static/bidder-info/adtarget.yaml index cc064b9ca6b..9ef7e259d22 100644 --- a/static/bidder-info/adtarget.yaml +++ b/static/bidder-info/adtarget.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # adtarget supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/adtelligent.yaml b/static/bidder-info/adtelligent.yaml index 30f55ea912e..f8ce434d481 100644 --- a/static/bidder-info/adtelligent.yaml +++ b/static/bidder-info/adtelligent.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # adtelligent supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/adxcg.yaml b/static/bidder-info/adxcg.yaml index e5535ae7258..14c7c8bb572 100644 --- a/static/bidder-info/adxcg.yaml +++ b/static/bidder-info/adxcg.yaml @@ -11,3 +11,8 @@ capabilities: - banner - video - native +userSync: + # adxcg supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/audienceNetwork.yaml b/static/bidder-info/audienceNetwork.yaml index 324e5c6dff8..5e567318a89 100644 --- a/static/bidder-info/audienceNetwork.yaml +++ b/static/bidder-info/audienceNetwork.yaml @@ -6,3 +6,7 @@ capabilities: - banner - video - native +userSync: + # facebook's audience network supports user syncing, but requires configuration by the host. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/bmtm.yaml b/static/bidder-info/bmtm.yaml index c13bf17db73..bc7d2aed806 100644 --- a/static/bidder-info/bmtm.yaml +++ b/static/bidder-info/bmtm.yaml @@ -6,3 +6,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # bmtm supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/criteo.yaml b/static/bidder-info/criteo.yaml index bfa098ba39d..5eb7eb37512 100644 --- a/static/bidder-info/criteo.yaml +++ b/static/bidder-info/criteo.yaml @@ -7,4 +7,9 @@ capabilities: - banner site: mediaTypes: - - banner \ No newline at end of file + - banner +userSync: + # criteo supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/dmx.yaml b/static/bidder-info/dmx.yaml index d29d699daeb..fa31886d229 100644 --- a/static/bidder-info/dmx.yaml +++ b/static/bidder-info/dmx.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # dmx supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/gamma.yaml b/static/bidder-info/gamma.yaml index aedaf2cf749..4a79fafb0fd 100644 --- a/static/bidder-info/gamma.yaml +++ b/static/bidder-info/gamma.yaml @@ -5,3 +5,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # gamma supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/invibes.yaml b/static/bidder-info/invibes.yaml index 45184fb9f65..6b7cd6a1f3c 100644 --- a/static/bidder-info/invibes.yaml +++ b/static/bidder-info/invibes.yaml @@ -5,3 +5,8 @@ capabilities: site: mediaTypes: - banner +userSync: + # invibes supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/lockerdome.yaml b/static/bidder-info/lockerdome.yaml index eebf3b87841..cdb0434935d 100644 --- a/static/bidder-info/lockerdome.yaml +++ b/static/bidder-info/lockerdome.yaml @@ -7,10 +7,22 @@ capabilities: site: mediaTypes: - banner -# lockerdome requires a platform id for their user sync process. replace <> -# in the url below and uncomment the userSync section to enable. -# -#userSync: -# redirect: -# url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" -# userMacro: "{{uid}}" +userSync: + # lockerdome supports user syncing, but requires configuration by the host. replace <> in + # the url below and configure it as either an environment variable or in pbs.yaml. + # + # environment variable: + # ---------------------- + # PBS_ADAPTERS_LOCKERDOME_USERSYNC_REDIRECT_URL: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + # PBS_ADAPTERS_LOCKERDOME_USERSYNC_REDIRECT_USER_MACRO: "{{uid}}" + # + # pbs.yaml: + # --------- + # adapters: + # lockerdome: + # usersync: + # redirect: + # url: "https://lockerdome.com/usync/prebidserver?pid=<>&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect={{.RedirectURL}}" + # userMacro: "{{uid}}" + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/mediafuse.yaml b/static/bidder-info/mediafuse.yaml index 98611402905..3c69bfeacaf 100644 --- a/static/bidder-info/mediafuse.yaml +++ b/static/bidder-info/mediafuse.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # mediafuse supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/rtbhouse.yaml b/static/bidder-info/rtbhouse.yaml index 12744fdc75e..8d5ad3f03fb 100644 --- a/static/bidder-info/rtbhouse.yaml +++ b/static/bidder-info/rtbhouse.yaml @@ -5,3 +5,8 @@ capabilities: site: mediaTypes: - banner +userSync: + # rtbhouse supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/rubicon.yaml b/static/bidder-info/rubicon.yaml index 0f19ddb9627..6d1568d6b62 100644 --- a/static/bidder-info/rubicon.yaml +++ b/static/bidder-info/rubicon.yaml @@ -9,3 +9,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # rubicon supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/smarthub.yaml b/static/bidder-info/smarthub.yaml index bfee9490840..6932780f195 100644 --- a/static/bidder-info/smarthub.yaml +++ b/static/bidder-info/smarthub.yaml @@ -11,3 +11,8 @@ capabilities: - banner - video - native +userSync: + # smarthub supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/synacormedia.yaml b/static/bidder-info/synacormedia.yaml index 14a4bcd1244..7ca12d5323a 100644 --- a/static/bidder-info/synacormedia.yaml +++ b/static/bidder-info/synacormedia.yaml @@ -13,3 +13,8 @@ userSync: iframe: url: "https://sync.technoratimedia.com/services?srv=cs&pid=70&cb={{.RedirectURL}}" userMacro: "[USER_ID]" +userSync: + # synacormedia supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - iframe \ No newline at end of file diff --git a/static/bidder-info/verizonmedia.yaml b/static/bidder-info/verizonmedia.yaml index 0b4642fcb6a..80db16ed28b 100644 --- a/static/bidder-info/verizonmedia.yaml +++ b/static/bidder-info/verizonmedia.yaml @@ -8,3 +8,8 @@ capabilities: site: mediaTypes: - banner +userSync: + # verizonmedia supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/viewdeos.yaml b/static/bidder-info/viewdeos.yaml index 9483e281de0..bf0c1789fe3 100644 --- a/static/bidder-info/viewdeos.yaml +++ b/static/bidder-info/viewdeos.yaml @@ -10,3 +10,8 @@ capabilities: mediaTypes: - banner - video +userSync: + # viewdeos supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/static/bidder-info/vrtcal.yaml b/static/bidder-info/vrtcal.yaml index fcf9294adcb..9a33e9f941a 100644 --- a/static/bidder-info/vrtcal.yaml +++ b/static/bidder-info/vrtcal.yaml @@ -4,3 +4,8 @@ capabilities: app: mediaTypes: - banner +userSync: + # vrtcal supports user syncing, but requires configuration by the host. contact this + # bidder directly at the email address in this file to ask about enabling user sync. + supports: + - redirect \ No newline at end of file diff --git a/usersync/syncersbuilder.go b/usersync/syncersbuilder.go index 96b6b9d219e..95b3e98962c 100644 --- a/usersync/syncersbuilder.go +++ b/usersync/syncersbuilder.go @@ -29,7 +29,7 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf // map syncer config by bidder cfgByBidder := make(map[string]config.Syncer, len(bidderInfos)) for bidder, cfg := range bidderInfos { - if cfg.Enabled && cfg.Syncer != nil { + if shouldCreateSyncer(cfg) { cfgByBidder[bidder] = *cfg.Syncer } } @@ -80,6 +80,20 @@ func BuildSyncers(hostConfig *config.Configuration, bidderInfos config.BidderInf return syncers, nil } +func shouldCreateSyncer(cfg config.BidderInfo) bool { + if !cfg.Enabled { + return false + } + + if cfg.Syncer == nil { + return false + } + + // a syncer may provide just a Supports field to provide hints to the host. we should only try to create a syncer + // if there is at least one non-Supports value populated. + return cfg.Syncer.Key != "" || cfg.Syncer.Default != "" || cfg.Syncer.IFrame != nil || cfg.Syncer.Redirect != nil || cfg.Syncer.SupportCORS != nil +} + func chooseSyncerConfig(biddersSyncerConfig []namedSyncerConfig) (namedSyncerConfig, error) { if len(biddersSyncerConfig) == 1 { return biddersSyncerConfig[0], nil diff --git a/usersync/syncersbuilder_test.go b/usersync/syncersbuilder_test.go index be3b7d705cd..e36b1c2ee52 100644 --- a/usersync/syncersbuilder_test.go +++ b/usersync/syncersbuilder_test.go @@ -26,6 +26,7 @@ func TestBuildSyncers(t *testing.T) { infoKeyADisabled = config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "a", IFrame: iframeConfig}} infoKeyAEmpty = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a"}} infoKeyAError = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "a", Default: "redirect", IFrame: iframeConfig}} // Error caused by invalid default sync type + infoKeyASupportsOnly = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Supports: []string{"iframe"}}} infoKeyBPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "b", IFrame: iframeConfig}} infoKeyBEmpty = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "b"}} infoKeyMissingPopulated = config.BidderInfo{Enabled: true, Syncer: &config.Syncer{IFrame: iframeConfig}} @@ -123,6 +124,14 @@ func TestBuildSyncers(t *testing.T) { "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", }, }, + { + description: "Many - Supports Only Syncers Ignored", + givenConfig: hostConfig, + givenBidderInfos: map[string]config.BidderInfo{"bidder1": infoKeyASupportsOnly, "bidder2": infoKeyBPopulated}, + expectedIFramesURLs: map[string]string{ + "bidder2": "https://bidder.com/iframe?redirect=http%3A%2F%2Fhost.com%2Fb%2Fhost", + }, + }, { description: "Many - Multiple Errors", givenConfig: hostConfig, @@ -166,6 +175,116 @@ func TestBuildSyncers(t *testing.T) { } } +func TestShouldCreateSyncer(t *testing.T) { + var ( + anySupports = []string{"iframe"} + anyEndpoint = &config.SyncerEndpoint{} + anyCORS = true + ) + + testCases := []struct { + description string + given config.BidderInfo + expected bool + }{ + { + description: "Enabled, No Syncer", + given: config.BidderInfo{Enabled: true, Syncer: nil}, + expected: false, + }, + { + description: "Enabled, Syncer", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "anyKey"}}, + expected: true, + }, + { + description: "Enabled, Syncer - Fully Loaded", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "anyKey", Default: "iframe", Supports: anySupports, IFrame: anyEndpoint, Redirect: anyEndpoint, SupportCORS: &anyCORS}}, + expected: true, + }, + { + description: "Enabled, Syncer - Only Key", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Key: "anyKey"}}, + expected: true, + }, + { + description: "Enabled, Syncer - Only Default", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Default: "iframe"}}, + expected: true, + }, + { + description: "Enabled, Syncer - Only Supports", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Supports: anySupports}}, + expected: false, + }, + { + description: "Enabled, Syncer - Only IFrame", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{IFrame: anyEndpoint}}, + expected: true, + }, + { + description: "Enabled, Syncer - Only Redirect", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{Redirect: anyEndpoint}}, + expected: true, + }, + { + description: "Enabled, Syncer - Only SupportCORS", + given: config.BidderInfo{Enabled: true, Syncer: &config.Syncer{SupportCORS: &anyCORS}}, + expected: true, + }, + { + description: "Disabled, No Syncer", + given: config.BidderInfo{Enabled: false, Syncer: nil}, + expected: false, + }, + { + description: "Disabled, Syncer", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "anyKey"}}, + expected: false, + }, + { + description: "Disabled, Syncer - Fully Loaded", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "anyKey", Default: "iframe", Supports: anySupports, IFrame: anyEndpoint, Redirect: anyEndpoint, SupportCORS: &anyCORS}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only Key", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Key: "anyKey"}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only Default", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Default: "iframe"}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only Supports", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Supports: anySupports}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only IFrame", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{IFrame: anyEndpoint}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only Redirect", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{Redirect: anyEndpoint}}, + expected: false, + }, + { + description: "Disabled, Syncer - Only SupportCORS", + given: config.BidderInfo{Enabled: false, Syncer: &config.Syncer{SupportCORS: &anyCORS}}, + expected: false, + }, + } + + for _, test := range testCases { + result := shouldCreateSyncer(test.given) + assert.Equal(t, test.expected, result, test.description) + } +} + func TestChooseSyncerConfig(t *testing.T) { var ( bidderAPopulated = namedSyncerConfig{name: "bidderA", cfg: config.Syncer{Key: "a", IFrame: &config.SyncerEndpoint{URL: "anyURL"}}} diff --git a/util/sliceutil/sliceutil.go b/util/sliceutil/sliceutil.go new file mode 100644 index 00000000000..523d35e0980 --- /dev/null +++ b/util/sliceutil/sliceutil.go @@ -0,0 +1,14 @@ +package sliceutil + +import ( + "strings" +) + +func ContainsStringIgnoreCase(s []string, v string) bool { + for _, i := range s { + if strings.EqualFold(i, v) { + return true + } + } + return false +} diff --git a/util/sliceutil/sliceutil_test.go b/util/sliceutil/sliceutil_test.go new file mode 100644 index 00000000000..62936605901 --- /dev/null +++ b/util/sliceutil/sliceutil_test.go @@ -0,0 +1,70 @@ +package sliceutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContainsStringIgnoreCase(t *testing.T) { + testCases := []struct { + description string + givenSlice []string + givenValue string + expected bool + }{ + { + description: "Nil", + givenSlice: nil, + givenValue: "a", + expected: false, + }, + { + description: "Empty", + givenSlice: []string{}, + givenValue: "a", + expected: false, + }, + { + description: "One - Match - Same Case", + givenSlice: []string{"a"}, + givenValue: "a", + expected: true, + }, + { + description: "One - Match - Different Case", + givenSlice: []string{"a"}, + givenValue: "A", + expected: true, + }, + { + description: "One - No Match", + givenSlice: []string{"a"}, + givenValue: "z", + expected: false, + }, + { + description: "Many - Match - Same Case", + givenSlice: []string{"a", "b"}, + givenValue: "b", + expected: true, + }, + { + description: "Many - Match - Different Case", + givenSlice: []string{"a", "b"}, + givenValue: "B", + expected: true, + }, + { + description: "Many - No Match", + givenSlice: []string{"a", "b"}, + givenValue: "z", + expected: false, + }, + } + + for _, test := range testCases { + result := ContainsStringIgnoreCase(test.givenSlice, test.givenValue) + assert.Equal(t, test.expected, result, test.description) + } +} From c5fb9b54c3ee8602ee4a35647b97a1fb1be3f4b6 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 18 Aug 2021 14:41:55 -0400 Subject: [PATCH 49/50] Emit Warning For Unconfigured Supported User Sync --- router/router.go | 29 +++++++++++++ router/router_test.go | 97 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/router/router.go b/router/router.go index 25ad26944db..446d4280b09 100644 --- a/router/router.go +++ b/router/router.go @@ -221,6 +221,10 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R return nil, err } + if err := checkSupportedUserSyncEndpoints(bidderInfos); err != nil { + return nil, err + } + syncersByBidder, errs := usersync.BuildSyncers(cfg, bidderInfos) if len(errs) > 0 { return nil, errortypes.NewAggregateError("user sync", errs) @@ -387,6 +391,31 @@ func applyBidderInfoConfigOverrides(bidderInfos config.BidderInfos, adaptersCfg return nil } +func checkSupportedUserSyncEndpoints(bidderInfos config.BidderInfos) error { + for name, info := range bidderInfos { + if info.Syncer == nil { + continue + } + + for _, endpoint := range info.Syncer.Supports { + endpointLower := strings.ToLower(endpoint) + switch endpointLower { + case "iframe": + if info.Syncer.IFrame == nil { + glog.Warningf("bidder %s supports iframe user sync, but doesn't have a default and must be configured by the host", name) + } + case "redirect": + if info.Syncer.Redirect == nil { + glog.Warningf("bidder %s supports redirect user sync, but doesn't have a default and must be configured by the host", name) + } + default: + return fmt.Errorf("failed to load bidder info for %s, user sync supported endpoint '%s' is unrecognized", name, endpoint) + } + } + } + return nil +} + // Fixes #648 // // These CORS options pose a security risk... but it's a calculated one. diff --git a/router/router_test.go b/router/router_test.go index fc90fdf01f3..e6c4396e49c 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -149,6 +149,103 @@ func TestApplyBidderInfoConfigOverrides(t *testing.T) { } } +func TestCheckSupportedUserSyncEndpoints(t *testing.T) { + anyEndpoint := &config.SyncerEndpoint{URL: "anyURL"} + + var testCases = []struct { + description string + givenBidderInfos config.BidderInfos + expectedError string + }{ + { + description: "None", + givenBidderInfos: config.BidderInfos{}, + expectedError: "", + }, + { + description: "One - No Syncer", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: nil}, + }, + expectedError: "", + }, + { + description: "One - Invalid Supported Endpoint", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"invalid"}}}, + }, + expectedError: "failed to load bidder info for a, user sync supported endpoint 'invalid' is unrecognized", + }, + { + description: "One - IFrame Supported - Not Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"iframe"}, IFrame: nil}}, + }, + expectedError: "", + }, + { + description: "One - IFrame Supported - Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"iframe"}, IFrame: anyEndpoint}}, + }, + expectedError: "", + }, + { + description: "One - Redirect Supported - Not Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"redirect"}, Redirect: nil}}, + }, + expectedError: "", + }, + { + description: "One - IFrame Supported - Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"redirect"}, Redirect: anyEndpoint}}, + }, + expectedError: "", + }, + { + description: "One - IFrame + Redirect Supported - Not Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"iframe", "redirect"}, IFrame: nil, Redirect: nil}}, + }, + expectedError: "", + }, + { + description: "One - IFrame + Redirect Supported - Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"iframe", "redirect"}, IFrame: anyEndpoint, Redirect: anyEndpoint}}, + }, + expectedError: "", + }, + { + description: "Many - With Invalid Supported Endpoint", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{}, + "b": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"invalid"}}}, + }, + expectedError: "failed to load bidder info for foo, user sync supported endpoint 'invalid' is unrecognized", + }, + { + description: "Many - Specified + Not Specified", + givenBidderInfos: config.BidderInfos{ + "a": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"iframe"}, IFrame: anyEndpoint}}, + "b": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"redirect"}, Redirect: nil}}, + }, + expectedError: "", + }, + } + + for _, test := range testCases { + resultErr := checkSupportedUserSyncEndpoints(test.givenBidderInfos) + if test.expectedError == "" { + assert.NoError(t, resultErr, test.description) + } else { + assert.EqualError(t, resultErr, test.expectedError, test.description) + } + } +} + // Prevents #648 func TestCORSSupport(t *testing.T) { const origin = "https://publisher-domain.com" From 2cc6ec37206f722584025cfb5b21a1168b7067b7 Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Wed, 18 Aug 2021 14:50:31 -0400 Subject: [PATCH 50/50] Fixed Test Typo --- router/router_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/router/router_test.go b/router/router_test.go index e6c4396e49c..07e5938800e 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -224,7 +224,7 @@ func TestCheckSupportedUserSyncEndpoints(t *testing.T) { "a": config.BidderInfo{}, "b": config.BidderInfo{Syncer: &config.Syncer{Supports: []string{"invalid"}}}, }, - expectedError: "failed to load bidder info for foo, user sync supported endpoint 'invalid' is unrecognized", + expectedError: "failed to load bidder info for b, user sync supported endpoint 'invalid' is unrecognized", }, { description: "Many - Specified + Not Specified",