Skip to content

Commit

Permalink
Adding Events support in bid responses (#1597)
Browse files Browse the repository at this point in the history
  • Loading branch information
@laurb9
laurb9 authored Jan 20, 2021
1 parent 6e32168 commit 994d0f0
Show file tree
Hide file tree
Showing 23 changed files with 1,422 additions and 162 deletions.
48 changes: 26 additions & 22 deletions endpoints/events/vtrack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
"time"

"github.com/golang/glog"
"github.com/julienschmidt/httprouter"
accountService "github.com/prebid/prebid-server/account"
Expand All @@ -13,12 +19,6 @@ import (
"github.com/prebid/prebid-server/errortypes"
"github.com/prebid/prebid-server/prebid_cache_client"
"github.com/prebid/prebid-server/stored_requests"
"io"
"io/ioutil"
"net/http"
"sort"
"strings"
"time"
)

const (
Expand Down Expand Up @@ -230,7 +230,7 @@ func (v *vtrackEndpoint) cachePutObjects(ctx context.Context, req *BidCacheReque
}

if _, ok := biddersAllowingVastUpdate[c.Bidder]; ok && nc.Data != nil {
nc.Data = modifyVastXml(v.Cfg.ExternalURL, nc.Data, c.BidID, c.Bidder, accountId, c.Timestamp)
nc.Data = ModifyVastXmlJSON(v.Cfg.ExternalURL, nc.Data, c.BidID, c.Bidder, accountId, c.Timestamp)
}

cacheables = append(cacheables, *nc)
Expand Down Expand Up @@ -269,32 +269,36 @@ func getAccountId(httpRequest *http.Request) string {
return httpRequest.URL.Query().Get(AccountParameter)
}

// modifyVastXml modifies BidCacheRequest element Vast XML data
func modifyVastXml(externalUrl string, data json.RawMessage, bidid string, bidder string, accountId string, timestamp int64) json.RawMessage {
c := string(data)
ci := strings.Index(c, ImpressionCloseTag)
// ModifyVastXmlString rewrites and returns the string vastXML and a flag indicating if it was modified
func ModifyVastXmlString(externalUrl, vast, bidid, bidder, accountID string, timestamp int64) (string, bool) {
ci := strings.Index(vast, ImpressionCloseTag)

// no impression tag - pass it as it is
if ci == -1 {
return data
return vast, false
}

vastUrlTracking := GetVastUrlTracking(externalUrl, bidid, bidder, accountId, timestamp)
vastUrlTracking := GetVastUrlTracking(externalUrl, bidid, bidder, accountID, timestamp)
impressionUrl := "<![CDATA[" + vastUrlTracking + "]]>"
oi := strings.Index(c, ImpressionOpenTag)
oi := strings.Index(vast, ImpressionOpenTag)

if ci-oi == len(ImpressionOpenTag) {
return json.RawMessage(strings.Replace(c, ImpressionOpenTag, ImpressionOpenTag+impressionUrl, 1))
return strings.Replace(vast, ImpressionOpenTag, ImpressionOpenTag+impressionUrl, 1), true
}

return json.RawMessage(strings.Replace(c, ImpressionCloseTag, ImpressionCloseTag+ImpressionOpenTag+impressionUrl+ImpressionCloseTag, 1))
return strings.Replace(vast, ImpressionCloseTag, ImpressionCloseTag+ImpressionOpenTag+impressionUrl+ImpressionCloseTag, 1), true
}

func contains(s []string, e string) bool {
if len(s) == 0 {
return false
// ModifyVastXmlJSON modifies BidCacheRequest element Vast XML data
func ModifyVastXmlJSON(externalUrl string, data json.RawMessage, bidid, bidder, accountId string, timestamp int64) json.RawMessage {
var vast string
if err := json.Unmarshal(data, &vast); err != nil {
// failed to decode json, fall back to string
vast = string(data)
}

i := sort.SearchStrings(s, e)
return i < len(s) && s[i] == e
vast, ok := ModifyVastXmlString(externalUrl, vast, bidid, bidder, accountId, timestamp)
if !ok {
return data
}
return json.RawMessage(vast)
}
1 change: 1 addition & 0 deletions endpoints/openrtb2/auction_benchmark_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) {
nil,
&config.Configuration{},
newTestMetrics(),
infos,
gdpr.AlwaysAllow{},
currency.NewRateConverter(&http.Client{}, "", time.Duration(0)),
empty_fetcher.EmptyFetcher{},
Expand Down
12 changes: 8 additions & 4 deletions exchange/auction.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func (a *auction) setRoundedPrices(priceGranularity openrtb_ext.PriceGranularity
a.roundedPrices = roundedPrices
}

func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string, debugLog *DebugLog) []error {
func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, evTracking *eventTracking, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string, debugLog *DebugLog) []error {
var bids, vast, includeBidderKeys, includeWinners bool = targData.includeCacheBids, targData.includeCacheVast, targData.includeBidderKeys, targData.includeWinners
if !((bids || vast) && (includeBidderKeys || includeWinners)) {
return nil
Expand Down Expand Up @@ -176,7 +176,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
expByImp[imp.ID] = imp.Exp
}
for _, topBidsPerImp := range a.winningBidsByBidder {
for _, topBidPerBidder := range topBidsPerImp {
for bidderName, topBidPerBidder := range topBidsPerImp {
impID := topBidPerBidder.bid.ImpID
isOverallWinner := a.winningBids[impID] == topBidPerBidder
if !includeBidderKeys && !isOverallWinner {
Expand All @@ -195,6 +195,10 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
}
if bids {
if jsonBytes, err := json.Marshal(topBidPerBidder.bid); err == nil {
jsonBytes, err = evTracking.modifyBidJSON(topBidPerBidder, bidderName, jsonBytes)
if err != nil {
errs = append(errs, err)
}
if useCustomCacheKey {
// not allowed if bids is true; log error and cache normally
errs = append(errs, errors.New("cannot use custom cache key for non-vast bids"))
Expand All @@ -210,8 +214,8 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
}
}
if vast && topBidPerBidder.bidType == openrtb_ext.BidTypeVideo {
vast := makeVAST(topBidPerBidder.bid)
if jsonBytes, err := json.Marshal(vast); err == nil {
vastXML := makeVAST(topBidPerBidder.bid)
if jsonBytes, err := json.Marshal(vastXML); err == nil {
if useCustomCacheKey {
toCache = append(toCache, prebid_cache_client.Cacheable{
Type: prebid_cache_client.TypeXML,
Expand Down
128 changes: 58 additions & 70 deletions exchange/auction_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import (
"encoding/xml"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"

"github.com/prebid/prebid-server/config"
Expand Down Expand Up @@ -100,57 +100,21 @@ func TestBuildCacheString(t *testing.T) {
// TestCacheJSON executes tests for all the *.json files in cachetest.
// customcachekey.json test here verifies custom cache key not used for non-vast video
func TestCacheJSON(t *testing.T) {
if specFiles, err := ioutil.ReadDir("./cachetest"); err == nil {
for _, specFile := range specFiles {
fileName := "./cachetest/" + specFile.Name()
fileDisplayName := "exchange/cachetest/" + specFile.Name()
specData, err := loadCacheSpec(fileName)
if err != nil {
t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
for _, dir := range []string{"cachetest", "customcachekeytest", "impcustomcachekeytest", "eventscachetest"} {
if specFiles, err := ioutil.ReadDir(dir); err == nil {
for _, specFile := range specFiles {
fileName := filepath.Join(dir, specFile.Name())
fileDisplayName := "exchange/" + fileName
t.Run(fileDisplayName, func(t *testing.T) {
specData, err := loadCacheSpec(fileName)
if assert.NoError(t, err, "Failed to load contents of file %s: %v", fileDisplayName, err) {
runCacheSpec(t, fileDisplayName, specData)
}
})
}

runCacheSpec(t, fileDisplayName, specData)
}
} else {
t.Fatalf("Failed to read contents of directory exchange/cachetest/: %v", err)
}
}

// TestCacheJSON executes tests for all the *.json files in customcachekeytest.
// customcachekey.json test here verifies custom cache key is used for vast video
func TestCustomCacheKeyJSON(t *testing.T) {
if specFiles, err := ioutil.ReadDir("./customcachekeytest"); err == nil {
for _, specFile := range specFiles {
fileName := "./customcachekeytest/" + specFile.Name()
fileDisplayName := "exchange/customcachekeytest/" + specFile.Name()
specData, err := loadCacheSpec(fileName)
if err != nil {
t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
}

runCacheSpec(t, fileDisplayName, specData)
}
} else {
t.Fatalf("Failed to read contents of directory exchange/customcachekeytest/: %v", err)
}
}

// TestMultiImpCache executes multi-Imp test cases found in *.json files in
// impcustomcachekeytest.
func TestCustomCacheKeyMultiImp(t *testing.T) {
if specFiles, err := ioutil.ReadDir("./impcustomcachekeytest"); err == nil {
for _, specFile := range specFiles {
fileName := "./impcustomcachekeytest/" + specFile.Name()
fileDisplayName := "exchange/impcustomcachekeytest/" + specFile.Name()
multiImpSpecData, err := loadCacheSpec(fileName)
if err != nil {
t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
}

runCacheSpec(t, fileDisplayName, multiImpSpecData)
} else {
t.Fatalf("Failed to read contents of directory exchange/%s: %v", dir, err)
}
} else {
t.Fatalf("Failed to read contents of directory exchange/customcachekeytest/: %v", err)
}
}

Expand All @@ -169,9 +133,8 @@ func loadCacheSpec(filename string) (*cacheSpec, error) {
return &spec, nil
}

// runCacheSpec has been modified to handle multi-Imp and multi-bid Json test files,
// it cycles through the bids found in the test cases hardcoded in json files and
// finds the highest bid of every Imp.
// runCacheSpec cycles through the bids found in the json test cases and
// finds the highest bid of every Imp, then tests doCache() with resulting auction object
func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
var bid *pbsOrtbBid
winningBidsByImp := make(map[string]*pbsOrtbBid)
Expand Down Expand Up @@ -245,7 +208,14 @@ func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
winningBidsByBidder: winningBidsByBidder,
roundedPrices: roundedPrices,
}
_ = testAuction.doCache(ctx, cache, targData, &specData.BidRequest, 60, &specData.DefaultTTLs, bidCategory, &specData.DebugLog)
evTracking := &eventTracking{
accountID: "TEST_ACC_ID",
enabledForAccount: specData.EventsDataEnabledForAccount,
enabledForRequest: specData.EventsDataEnabledForRequest,
externalURL: "http://localhost",
auctionTimestampMs: 1234567890,
}
_ = testAuction.doCache(ctx, cache, targData, evTracking, &specData.BidRequest, 60, &specData.DefaultTTLs, bidCategory, &specData.DebugLog)

if len(specData.ExpectedCacheables) > len(cache.items) {
t.Errorf("%s: [CACHE_ERROR] Less elements were cached than expected \n", fileDisplayName)
Expand All @@ -254,27 +224,43 @@ func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
} else { // len(specData.ExpectedCacheables) == len(cache.items)
// We cached the exact number of elements we expected, now we compare them side by side in n^2
var matched int = 0
var formattedExpectedData string
for i := 0; i < len(specData.ExpectedCacheables); i++ {
if specData.ExpectedCacheables[i].Type == prebid_cache_client.TypeJSON {
ExpectedData := strings.Replace(string(specData.ExpectedCacheables[i].Data), "\\", "", -1)
ExpectedData = strings.Replace(ExpectedData, " ", "", -1)
formattedExpectedData = ExpectedData[1 : len(ExpectedData)-1]
} else {
formattedExpectedData = string(specData.ExpectedCacheables[i].Data)
for i, expectedCacheable := range specData.ExpectedCacheables {
found := false
var expectedData interface{}
if err := json.Unmarshal(expectedCacheable.Data, &expectedData); err != nil {
t.Fatalf("Failed to decode expectedCacheables[%d].value: %v", i, err)
}
for j := 0; j < len(cache.items); j++ {
if formattedExpectedData == string(cache.items[j].Data) &&
specData.ExpectedCacheables[i].TTLSeconds == cache.items[j].TTLSeconds &&
specData.ExpectedCacheables[i].Type == cache.items[j].Type &&
len(specData.ExpectedCacheables[i].Key) <= len(cache.items[j].Key) &&
specData.ExpectedCacheables[i].Key == cache.items[j].Key[:len(specData.ExpectedCacheables[i].Key)] {
matched++
if s, ok := expectedData.(string); ok && expectedCacheable.Type == prebid_cache_client.TypeJSON {
// decode again if we have pre-encoded json string values
if err := json.Unmarshal([]byte(s), &expectedData); err != nil {
t.Fatalf("Failed to re-decode expectedCacheables[%d].value :%v", i, err)
}
}
for j, cachedItem := range cache.items {
var actualData interface{}
if err := json.Unmarshal(cachedItem.Data, &actualData); err != nil {
t.Fatalf("Failed to decode actual cache[%d].value: %s", j, err)
}
if assert.ObjectsAreEqual(expectedData, actualData) &&
expectedCacheable.TTLSeconds == cachedItem.TTLSeconds &&
expectedCacheable.Type == cachedItem.Type &&
len(expectedCacheable.Key) <= len(cachedItem.Key) &&
expectedCacheable.Key == cachedItem.Key[:len(expectedCacheable.Key)] {
found = true
cache.items = append(cache.items[:j], cache.items[j+1:]...) // remove matched item
break
}
}
if found {
matched++
} else {
t.Errorf("%s: [CACHE_ERROR] Did not see expected cacheable #%d: type=%s, ttl=%d, value=%s", fileDisplayName, i, expectedCacheable.Type, expectedCacheable.TTLSeconds, string(expectedCacheable.Data))
}
}
if matched != len(specData.ExpectedCacheables) {
t.Errorf("%s: [CACHE_ERROR] One or more keys were not cached as we expected \n", fileDisplayName)
for i, item := range cache.items {
t.Errorf("%s: [CACHE_ERROR] Got unexpected cached item #%d: type=%s, ttl=%d, value=%s", fileDisplayName, i, item.Type, item.TTLSeconds, string(item.Data))
}
t.FailNow()
}
}
Expand Down Expand Up @@ -508,6 +494,8 @@ type cacheSpec struct {
TargetDataIncludeBidderKeys bool `json:"targetDataIncludeBidderKeys"`
TargetDataIncludeCacheBids bool `json:"targetDataIncludeCacheBids"`
TargetDataIncludeCacheVast bool `json:"targetDataIncludeCacheVast"`
EventsDataEnabledForAccount bool `json:"eventsDataEnabledForAccount"`
EventsDataEnabledForRequest bool `json:"eventsDataEnabledForRequest"`
DebugLog DebugLog `json:"debugLog,omitempty"`
}

Expand Down
2 changes: 2 additions & 0 deletions exchange/bidder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,15 @@ type adaptedBidder interface {
// pbsOrtbBid.bidType will become "response.seatbid[i].bid.ext.prebid.type" in the final OpenRTB response.
// pbsOrtbBid.bidTargets does not need to be filled out by the Bidder. It will be set later by the exchange.
// pbsOrtbBid.bidVideo is optional but should be filled out by the Bidder if bidType is video.
// pbsOrtbBid.bidEvents is set by exchange when event tracking is enabled
// pbsOrtbBid.dealPriority is optionally provided by adapters and used internally by the exchange to support deal targeted campaigns.
// pbsOrtbBid.dealTierSatisfied is set to true by exchange.updateHbPbCatDur if deal tier satisfied otherwise it will be set to false
type pbsOrtbBid struct {
bid *openrtb.Bid
bidType openrtb_ext.BidType
bidTargets map[string]string
bidVideo *openrtb_ext.ExtBidPrebidVideo
bidEvents *openrtb_ext.ExtBidPrebidEvents
dealPriority int
dealTierSatisfied bool
}
Expand Down
Loading

0 comments on commit 994d0f0

Please sign in to comment.