Adding Events support in bid responses

endpoints/events/vtrack.go

@@ -4,6 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
 	"github.com/golang/glog"
 	"github.com/julienschmidt/httprouter"
 	accountService "github.com/prebid/prebid-server/account"
@@ -13,12 +19,6 @@ import (
 	"github.com/prebid/prebid-server/errortypes"
 	"github.com/prebid/prebid-server/prebid_cache_client"
 	"github.com/prebid/prebid-server/stored_requests"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"sort"
-	"strings"
-	"time"
 )
 
 const (
@@ -230,7 +230,7 @@ func (v *vtrackEndpoint) cachePutObjects(ctx context.Context, req *BidCacheReque
 		}
 
 		if _, ok := biddersAllowingVastUpdate[c.Bidder]; ok && nc.Data != nil {
-			nc.Data = modifyVastXml(v.Cfg.ExternalURL, nc.Data, c.BidID, c.Bidder, accountId, c.Timestamp)
+			nc.Data = ModifyVastXmlJSON(v.Cfg.ExternalURL, nc.Data, c.BidID, c.Bidder, accountId, c.Timestamp)
 		}
 
 		cacheables = append(cacheables, *nc)
@@ -269,32 +269,36 @@ func getAccountId(httpRequest *http.Request) string {
 	return httpRequest.URL.Query().Get(AccountParameter)
 }
 
-// modifyVastXml modifies BidCacheRequest element Vast XML data
-func modifyVastXml(externalUrl string, data json.RawMessage, bidid string, bidder string, accountId string, timestamp int64) json.RawMessage {
-	c := string(data)
-	ci := strings.Index(c, ImpressionCloseTag)
+// ModifyVastXmlString rewrites and returns the string vastXML and a flag indicating if it was modified
+func ModifyVastXmlString(externalUrl, vast, bidid, bidder, accountID string, timestamp int64) (string, bool) {
+	ci := strings.Index(vast, ImpressionCloseTag)
 
 	// no impression tag - pass it as it is
 	if ci == -1 {
-		return data
+		return vast, false
 	}
 
-	vastUrlTracking := GetVastUrlTracking(externalUrl, bidid, bidder, accountId, timestamp)
+	vastUrlTracking := GetVastUrlTracking(externalUrl, bidid, bidder, accountID, timestamp)
 	impressionUrl := "<![CDATA[" + vastUrlTracking + "]]>"
-	oi := strings.Index(c, ImpressionOpenTag)
+	oi := strings.Index(vast, ImpressionOpenTag)
 
 	if ci-oi == len(ImpressionOpenTag) {
-		return json.RawMessage(strings.Replace(c, ImpressionOpenTag, ImpressionOpenTag+impressionUrl, 1))
+		return strings.Replace(vast, ImpressionOpenTag, ImpressionOpenTag+impressionUrl, 1), true
 	}
 
-	return json.RawMessage(strings.Replace(c, ImpressionCloseTag, ImpressionCloseTag+ImpressionOpenTag+impressionUrl+ImpressionCloseTag, 1))
+	return strings.Replace(vast, ImpressionCloseTag, ImpressionCloseTag+ImpressionOpenTag+impressionUrl+ImpressionCloseTag, 1), true
 }
 
-func contains(s []string, e string) bool {
-	if len(s) == 0 {
-		return false
+// ModifyVastXmlJSON modifies BidCacheRequest element Vast XML data
+func ModifyVastXmlJSON(externalUrl string, data json.RawMessage, bidid, bidder, accountId string, timestamp int64) json.RawMessage {
+	var vast string
+	if err := json.Unmarshal(data, &vast); err != nil {
+		// failed to decode json, fall back to string
+		vast = string(data)
 	}
-
-	i := sort.SearchStrings(s, e)
-	return i < len(s) && s[i] == e
+	vast, ok := ModifyVastXmlString(externalUrl, vast, bidid, bidder, accountId, timestamp)
+	if !ok {
+		return data
+	}
+	return json.RawMessage(vast)
 }

endpoints/openrtb2/auction_benchmark_test.go

@@ -77,6 +77,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) {
 		nil,
 		&config.Configuration{},
 		newTestMetrics(),
+		infos,
 		gdpr.AlwaysAllow{},
 		currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
 		empty_fetcher.EmptyFetcher{},

exchange/auction.go

@@ -147,7 +147,7 @@ func (a *auction) setRoundedPrices(priceGranularity openrtb_ext.PriceGranularity
 	a.roundedPrices = roundedPrices
 }
 
-func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string, debugLog *DebugLog) []error {
+func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, evTracking *eventTracking, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string, debugLog *DebugLog) []error {
 	var bids, vast, includeBidderKeys, includeWinners bool = targData.includeCacheBids, targData.includeCacheVast, targData.includeBidderKeys, targData.includeWinners
 	if !((bids || vast) && (includeBidderKeys || includeWinners)) {
 		return nil
@@ -176,7 +176,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
 		expByImp[imp.ID] = imp.Exp
 	}
 	for _, topBidsPerImp := range a.winningBidsByBidder {
-		for _, topBidPerBidder := range topBidsPerImp {
+		for bidderName, topBidPerBidder := range topBidsPerImp {
 			impID := topBidPerBidder.bid.ImpID
 			isOverallWinner := a.winningBids[impID] == topBidPerBidder
 			if !includeBidderKeys && !isOverallWinner {
@@ -195,6 +195,10 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
 			}
 			if bids {
 				if jsonBytes, err := json.Marshal(topBidPerBidder.bid); err == nil {
+					jsonBytes, err = evTracking.modifyBidJSON(topBidPerBidder, bidderName, jsonBytes)
+					if err != nil {
+						errs = append(errs, err)
+					}
 					if useCustomCacheKey {
 						// not allowed if bids is true; log error and cache normally
 						errs = append(errs, errors.New("cannot use custom cache key for non-vast bids"))
@@ -210,8 +214,8 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
 				}
 			}
 			if vast && topBidPerBidder.bidType == openrtb_ext.BidTypeVideo {
-				vast := makeVAST(topBidPerBidder.bid)
-				if jsonBytes, err := json.Marshal(vast); err == nil {
+				vastXML := makeVAST(topBidPerBidder.bid)
+				if jsonBytes, err := json.Marshal(vastXML); err == nil {
 					if useCustomCacheKey {
 						toCache = append(toCache, prebid_cache_client.Cacheable{
 							Type:       prebid_cache_client.TypeXML,

exchange/auction_test.go

@@ -6,9 +6,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strconv"
-	"strings"
 	"testing"
 
 	"github.com/prebid/prebid-server/config"
@@ -100,57 +100,21 @@ func TestBuildCacheString(t *testing.T) {
 // TestCacheJSON executes tests for all the *.json files in cachetest.
 // customcachekey.json test here verifies custom cache key not used for non-vast video
 func TestCacheJSON(t *testing.T) {
-	if specFiles, err := ioutil.ReadDir("./cachetest"); err == nil {
-		for _, specFile := range specFiles {
-			fileName := "./cachetest/" + specFile.Name()
-			fileDisplayName := "exchange/cachetest/" + specFile.Name()
-			specData, err := loadCacheSpec(fileName)
-			if err != nil {
-				t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
+	for _, dir := range []string{"cachetest", "customcachekeytest", "impcustomcachekeytest", "eventscachetest"} {
+		if specFiles, err := ioutil.ReadDir(dir); err == nil {
+			for _, specFile := range specFiles {
+				fileName := filepath.Join(dir, specFile.Name())
+				fileDisplayName := "exchange/" + fileName
+				t.Run(fileDisplayName, func(t *testing.T) {
+					specData, err := loadCacheSpec(fileName)
+					if assert.NoError(t, err, "Failed to load contents of file %s: %v", fileDisplayName, err) {
+						runCacheSpec(t, fileDisplayName, specData)
+					}
+				})
 			}
-
-			runCacheSpec(t, fileDisplayName, specData)
-		}
-	} else {
-		t.Fatalf("Failed to read contents of directory exchange/cachetest/: %v", err)
-	}
-}
-
-// TestCacheJSON executes tests for all the *.json files in customcachekeytest.
-// customcachekey.json test here verifies custom cache key is used for vast video
-func TestCustomCacheKeyJSON(t *testing.T) {
-	if specFiles, err := ioutil.ReadDir("./customcachekeytest"); err == nil {
-		for _, specFile := range specFiles {
-			fileName := "./customcachekeytest/" + specFile.Name()
-			fileDisplayName := "exchange/customcachekeytest/" + specFile.Name()
-			specData, err := loadCacheSpec(fileName)
-			if err != nil {
-				t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
-			}
-
-			runCacheSpec(t, fileDisplayName, specData)
-		}
-	} else {
-		t.Fatalf("Failed to read contents of directory exchange/customcachekeytest/: %v", err)
-	}
-}
-
-// TestMultiImpCache executes multi-Imp test cases found in *.json files in
-// impcustomcachekeytest.
-func TestCustomCacheKeyMultiImp(t *testing.T) {
-	if specFiles, err := ioutil.ReadDir("./impcustomcachekeytest"); err == nil {
-		for _, specFile := range specFiles {
-			fileName := "./impcustomcachekeytest/" + specFile.Name()
-			fileDisplayName := "exchange/impcustomcachekeytest/" + specFile.Name()
-			multiImpSpecData, err := loadCacheSpec(fileName)
-			if err != nil {
-				t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
-			}
-
-			runCacheSpec(t, fileDisplayName, multiImpSpecData)
+		} else {
+			t.Fatalf("Failed to read contents of directory exchange/%s: %v", dir, err)
 		}
-	} else {
-		t.Fatalf("Failed to read contents of directory exchange/customcachekeytest/: %v", err)
 	}
 }
 
@@ -169,9 +133,8 @@ func loadCacheSpec(filename string) (*cacheSpec, error) {
 	return &spec, nil
 }
 
-// runCacheSpec has been modified to handle multi-Imp and multi-bid Json test files,
-// it cycles through the bids found in the test cases hardcoded in json files and
-// finds the highest bid of every Imp.
+// runCacheSpec cycles through the bids found in the json test cases and
+// finds the highest bid of every Imp, then tests doCache() with resulting auction object
 func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
 	var bid *pbsOrtbBid
 	winningBidsByImp := make(map[string]*pbsOrtbBid)
@@ -245,7 +208,14 @@ func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
 		winningBidsByBidder: winningBidsByBidder,
 		roundedPrices:       roundedPrices,
 	}
-	_ = testAuction.doCache(ctx, cache, targData, &specData.BidRequest, 60, &specData.DefaultTTLs, bidCategory, &specData.DebugLog)
+	evTracking := &eventTracking{
+		accountID:          "TEST_ACC_ID",
+		enabledForAccount:  specData.EventsDataEnabledForAccount,
+		enabledForRequest:  specData.EventsDataEnabledForRequest,
+		externalURL:        "http://localhost",
+		auctionTimestampMs: 1234567890,
+	}
+	_ = testAuction.doCache(ctx, cache, targData, evTracking, &specData.BidRequest, 60, &specData.DefaultTTLs, bidCategory, &specData.DebugLog)
 
 	if len(specData.ExpectedCacheables) > len(cache.items) {
 		t.Errorf("%s:  [CACHE_ERROR] Less elements were cached than expected \n", fileDisplayName)
@@ -254,27 +224,43 @@ func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
 	} else { // len(specData.ExpectedCacheables) == len(cache.items)
 		// We cached the exact number of elements we expected, now we compare them side by side in n^2
 		var matched int = 0
-		var formattedExpectedData string
-		for i := 0; i < len(specData.ExpectedCacheables); i++ {
-			if specData.ExpectedCacheables[i].Type == prebid_cache_client.TypeJSON {
-				ExpectedData := strings.Replace(string(specData.ExpectedCacheables[i].Data), "\\", "", -1)
-				ExpectedData = strings.Replace(ExpectedData, " ", "", -1)
-				formattedExpectedData = ExpectedData[1 : len(ExpectedData)-1]
-			} else {
-				formattedExpectedData = string(specData.ExpectedCacheables[i].Data)
+		for i, expectedCacheable := range specData.ExpectedCacheables {
+			found := false
+			var expectedData interface{}
+			if err := json.Unmarshal(expectedCacheable.Data, &expectedData); err != nil {
+				t.Fatalf("Failed to decode expectedCacheables[%d].value: %v", i, err)
 			}
-			for j := 0; j < len(cache.items); j++ {
-				if formattedExpectedData == string(cache.items[j].Data) &&
-					specData.ExpectedCacheables[i].TTLSeconds == cache.items[j].TTLSeconds &&
-					specData.ExpectedCacheables[i].Type == cache.items[j].Type &&
-					len(specData.ExpectedCacheables[i].Key) <= len(cache.items[j].Key) &&
-					specData.ExpectedCacheables[i].Key == cache.items[j].Key[:len(specData.ExpectedCacheables[i].Key)] {
-					matched++
+			if s, ok := expectedData.(string); ok && expectedCacheable.Type == prebid_cache_client.TypeJSON {
+				// decode again if we have pre-encoded json string values
+				if err := json.Unmarshal([]byte(s), &expectedData); err != nil {
+					t.Fatalf("Failed to re-decode expectedCacheables[%d].value :%v", i, err)
 				}
 			}
+			for j, cachedItem := range cache.items {
+				var actualData interface{}
+				if err := json.Unmarshal(cachedItem.Data, &actualData); err != nil {
+					t.Fatalf("Failed to decode actual cache[%d].value: %s", j, err)
+				}
+				if assert.ObjectsAreEqual(expectedData, actualData) &&
+					expectedCacheable.TTLSeconds == cachedItem.TTLSeconds &&
+					expectedCacheable.Type == cachedItem.Type &&
+					len(expectedCacheable.Key) <= len(cachedItem.Key) &&
+					expectedCacheable.Key == cachedItem.Key[:len(expectedCacheable.Key)] {
+					found = true
+					cache.items = append(cache.items[:j], cache.items[j+1:]...) // remove matched item
+					break
+				}
+			}
+			if found {
+				matched++
+			} else {
+				t.Errorf("%s: [CACHE_ERROR] Did not see expected cacheable #%d: type=%s, ttl=%d, value=%s", fileDisplayName, i, expectedCacheable.Type, expectedCacheable.TTLSeconds, string(expectedCacheable.Data))
+			}
 		}
 		if matched != len(specData.ExpectedCacheables) {
-			t.Errorf("%s: [CACHE_ERROR] One or more keys were not cached as we expected \n", fileDisplayName)
+			for i, item := range cache.items {
+				t.Errorf("%s: [CACHE_ERROR] Got unexpected cached item #%d: type=%s, ttl=%d, value=%s", fileDisplayName, i, item.Type, item.TTLSeconds, string(item.Data))
+			}
 			t.FailNow()
 		}
 	}
@@ -508,6 +494,8 @@ type cacheSpec struct {
 	TargetDataIncludeBidderKeys bool                            `json:"targetDataIncludeBidderKeys"`
 	TargetDataIncludeCacheBids  bool                            `json:"targetDataIncludeCacheBids"`
 	TargetDataIncludeCacheVast  bool                            `json:"targetDataIncludeCacheVast"`
+	EventsDataEnabledForAccount bool                            `json:"eventsDataEnabledForAccount"`
+	EventsDataEnabledForRequest bool                            `json:"eventsDataEnabledForRequest"`
 	DebugLog                    DebugLog                        `json:"debugLog,omitempty"`
 }
 

exchange/bidder.go

@@ -58,13 +58,15 @@ type adaptedBidder interface {
 // pbsOrtbBid.bidType will become "response.seatbid[i].bid.ext.prebid.type" in the final OpenRTB response.
 // pbsOrtbBid.bidTargets does not need to be filled out by the Bidder. It will be set later by the exchange.
 // pbsOrtbBid.bidVideo is optional but should be filled out by the Bidder if bidType is video.
+// pbsOrtbBid.bidEvents is set by exchange when event tracking is enabled
 // pbsOrtbBid.dealPriority is optionally provided by adapters and used internally by the exchange to support deal targeted campaigns.
 // pbsOrtbBid.dealTierSatisfied is set to true by exchange.updateHbPbCatDur if deal tier satisfied otherwise it will be set to false
 type pbsOrtbBid struct {
 	bid               *openrtb.Bid
 	bidType           openrtb_ext.BidType
 	bidTargets        map[string]string
 	bidVideo          *openrtb_ext.ExtBidPrebidVideo
+	bidEvents         *openrtb_ext.ExtBidPrebidEvents
 	dealPriority      int
 	dealTierSatisfied bool
 }