From 93900732dcd058f2d1216de15f5441e599e480f2 Mon Sep 17 00:00:00 2001 From: hhhjort <31041505+hhhjort@users.noreply.github.com> Date: Tue, 27 Aug 2019 20:44:01 -0400 Subject: [PATCH] Blacklist requests by account (#1014) * initial pass at blacklisted accounts * Adds support for amp and video --- config/config.go | 11 +++ endpoints/openrtb2/amp_auction.go | 12 +++ endpoints/openrtb2/auction.go | 9 +- endpoints/openrtb2/auction_test.go | 28 +++++- .../blacklisted/blacklisted-acct.json | 88 +++++++++++++++++++ endpoints/openrtb2/video_auction.go | 9 +- errortypes/errortypes.go | 17 ++++ 7 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 endpoints/openrtb2/sample-requests/blacklisted/blacklisted-acct.json diff --git a/config/config.go b/config/config.go index 836b34a8e5a..e5d11ec636a 100644 --- a/config/config.go +++ b/config/config.go @@ -54,6 +54,9 @@ type Configuration struct { // Array of blacklisted apps that is used to create the hash table BlacklistedAppMap so App.ID's can be instantly accessed. BlacklistedApps []string `mapstructure:"blacklisted_apps,flow"` BlacklistedAppMap map[string]bool + // Array of blacklisted accounts that is used to create the hash table BlacklistedAcctMap so Account.ID's can be instantly accessed. + BlacklistedAccts []string `mapstructure:"blacklisted_accts,flow"` + BlacklistedAcctMap map[string]bool } type HTTPClient struct { @@ -404,6 +407,13 @@ func New(v *viper.Viper) (*Configuration, error) { for i := 0; i < len(c.BlacklistedApps); i++ { c.BlacklistedAppMap[c.BlacklistedApps[i]] = true } + + // To look for a request's account id in O(1) time, we fill this hash table located in the + // the BlacklistedAccts field of the Configuration struct defined in this file + c.BlacklistedAcctMap = make(map[string]bool) + for i := 0; i < len(c.BlacklistedAccts); i++ { + c.BlacklistedAcctMap[c.BlacklistedAccts[i]] = true + } return &c, nil } @@ -649,6 +659,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("default_request.file.name", "") v.SetDefault("default_request.alias_info", false) v.SetDefault("blacklisted_apps", []string{""}) + v.SetDefault("blacklisted_accts", []string{""}) // Set environment variable support: v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 98029b3ce18..9208ceaf9b7 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -18,6 +18,7 @@ import ( "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" "github.com/prebid/prebid-server/exchange" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" @@ -148,6 +149,17 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h labels.CookieFlag = pbsmetrics.CookieFlagYes } labels.PubID = effectivePubID(req.Site.Publisher) + // Blacklist account now that we have resolved the value + if _, found := deps.cfg.BlacklistedAcctMap[labels.PubID]; found { + errL = append(errL, &errortypes.BlacklistedAcct{Message: fmt.Sprintf("Prebid-server has blacklisted Account ID: %s, pleaase reach out to the prebid server host.", labels.PubID)}) + w.WriteHeader(http.StatusBadRequest) + for _, err := range errL { + w.Write([]byte(fmt.Sprintf("Invalid request format: %s\n", err.Error()))) + } + ao.Errors = append(ao.Errors, errL...) + labels.RequestStatus = pbsmetrics.RequestStatusBadInput + return + } response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories) ao.AuctionResponse = response diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 355bab584e2..8cbcb9c37c8 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -135,6 +135,13 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http } labels.PubID = effectivePubID(req.Site.Publisher) } + // Blacklist account now that we have resolved the value + if _, found := deps.cfg.BlacklistedAcctMap[labels.PubID]; found { + errL = append(errL, &errortypes.BlacklistedAcct{Message: fmt.Sprintf("Prebid-server has blacklisted Account ID: %s, pleaase reach out to the prebid server host.", labels.PubID)}) + writeError(errL, w) + labels.RequestStatus = pbsmetrics.RequestStatusBadInput + return + } response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories) ao.Request = req @@ -1149,7 +1156,7 @@ func writeError(errs []error, w http.ResponseWriter) bool { func fatalError(errL []error) bool { for _, err := range errL { errCode := errortypes.DecodeError(err) - if errCode != errortypes.BidderTemporarilyDisabledCode || errCode == errortypes.BlacklistedAppCode { + if errCode != errortypes.BidderTemporarilyDisabledCode || errCode == errortypes.BlacklistedAppCode || errCode == errortypes.BlacklistedAcctCode { return true } } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 7a5ce9b9bac..3a5d96d2155 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "net/http" @@ -213,6 +214,19 @@ func TestDisabledBidders(t *testing.T) { goodTests.assert(t) } +// TestBlacklistRequests makes sure we return 400s on blacklisted requests. +func TestBlacklistRequests(t *testing.T) { + // Need to turn off aliases for bad requests as applying the alias can fail on a bad request before the expected error is reached. + tests := &getResponseFromDirectory{ + dir: "sample-requests/blacklisted", + payloadGetter: getRequestPayload, + messageGetter: getMessage, + expectedCode: http.StatusBadRequest, + aliased: false, + } + tests.assert(t) +} + // assertResponseFromDirectory makes sure that the payload from each file in dir gets the expected response status code // from the /openrtb2/auction endpoint. func (gr *getResponseFromDirectory) assert(t *testing.T) { @@ -222,6 +236,7 @@ func (gr *getResponseFromDirectory) assert(t *testing.T) { filename := gr.dir + "/" + fileInfo.Name() fileData := readFile(t, filename) code, msg := gr.doRequest(t, gr.payloadGetter(t, fileData)) + fmt.Printf("Processing %s\n", filename) assertResponseCode(t, filename, code, gr.expectedCode, msg) expectMsg := gr.messageGetter(t, fileData) @@ -263,7 +278,18 @@ func (gr *getResponseFromDirectory) doRequest(t *testing.T, requestData []byte) // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList()) - endpoint, _ := NewEndpoint(&nobidExchange{}, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize, BlacklistedApps: []string{"spam_app"}, BlacklistedAppMap: map[string]bool{"spam_app": true}}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, aliasJSON, bidderMap) + endpoint, _ := NewEndpoint( + &nobidExchange{}, + newParamsValidator(t), + &mockStoredReqFetcher{}, + empty_fetcher.EmptyFetcher{}, + &config.Configuration{MaxRequestSize: maxSize, BlacklistedApps: []string{"spam_app"}, BlacklistedAppMap: map[string]bool{"spam_app": true}, BlacklistedAccts: []string{"bad_acct"}, BlacklistedAcctMap: map[string]bool{"bad_acct": true}}, + theMetrics, + analyticsConf.NewPBSAnalytics(&config.Analytics{}), + disabledBidders, + aliasJSON, + bidderMap, + ) request := httptest.NewRequest("POST", "/openrtb2/auction", bytes.NewReader(requestData)) recorder := httptest.NewRecorder() diff --git a/endpoints/openrtb2/sample-requests/blacklisted/blacklisted-acct.json b/endpoints/openrtb2/sample-requests/blacklisted/blacklisted-acct.json new file mode 100644 index 00000000000..4ceff6351df --- /dev/null +++ b/endpoints/openrtb2/sample-requests/blacklisted/blacklisted-acct.json @@ -0,0 +1,88 @@ +{ + "description": "This is a perfectly valid request except that it comes from a blacklisted Account", + "message": "Invalid request: Prebid-server has blacklisted Account ID: bad_acct, pleaase reach out to the prebid server host.\n", + + "requestPayload": { + "id": "some-request-id", + "user": { + "ext": { + "consent": "gdpr-consent-string", + "prebid": { + "buyeruids": { + "appnexus": "override-appnexus-id-in-cookie" + } + } + } + }, + "app": { + "id": "cool_app", + "publisher": { + "id": "bad_acct" + } + }, + "regs": { + "ext": { + "gdpr": 1 + } + }, + "imp": [ + { + "id": "some-impression-id", + "banner": { + "format": [ + { + "w": 300, + "h": 250 + }, + { + "w": 300, + "h": 600 + } + ] + }, + "ext": { + "appnexus": { + "placementId": 10433394 + }, + "districtm": { + "placementId": 105 + }, + "rubicon": { + "accountId": 1001, + "siteId": 113932, + "zoneId": 535510 + } + } + } + ], + "tmax": 500, + "ext": { + "prebid": { + "aliases": { + "districtm": "appnexus" + }, + "bidadjustmentfactors": { + "appnexus": 1.01, + "districtm": 0.98, + "rubicon": 0.99 + }, + "cache": { + "bids": {} + }, + "targeting": { + "includewinners": false, + "pricegranularity": { + "precision": 2, + "ranges": [ + { + "max": 20, + "increment": 0.10 + } + ] + } + } + } + } + } + } + \ No newline at end of file diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index 1e4ece807b4..1b07f738e37 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -198,7 +198,11 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re } labels.PubID = effectivePubID(bidReq.Site.Publisher) } - + // Blacklist account now that we have resolved the value + if _, found := deps.cfg.BlacklistedAcctMap[labels.PubID]; found { + errL := []error{&errortypes.BlacklistedAcct{Message: fmt.Sprintf("Prebid-server has blacklisted Account ID: %s, pleaase reach out to the prebid server host.", labels.PubID)}} + handleError(labels, w, errL, ao) + } //execute auction logic response, err := deps.ex.HoldAuction(ctx, bidReq, usersyncs, labels, &deps.categories) ao.Request = bidReq @@ -245,7 +249,8 @@ func handleError(labels pbsmetrics.Labels, w http.ResponseWriter, errL []error, var errors string var foundBlacklisted bool = false for _, er := range errL { - if errortypes.DecodeError(er) == errortypes.BlacklistedAppCode { + erVal := errortypes.DecodeError(er) + if erVal == errortypes.BlacklistedAppCode || erVal == errortypes.BlacklistedAcctCode { foundBlacklisted = true } errors = fmt.Sprintf("%s %s", errors, er.Error()) diff --git a/errortypes/errortypes.go b/errortypes/errortypes.go index 865c885d5d9..fcf2265d915 100644 --- a/errortypes/errortypes.go +++ b/errortypes/errortypes.go @@ -10,6 +10,7 @@ const ( BadServerResponseCode FailedToRequestBidsCode BidderTemporarilyDisabledCode + BlacklistedAcctCode ) // We should use this code for any Error interface that is not in this package @@ -68,6 +69,22 @@ func (err *BlacklistedApp) Code() int { return BlacklistedAppCode } +// BlacklistedAcct should be used when a request account ID matches an entry in the BlacklistedAccts +// environment variable array +// +// These errors will be written to http.ResponseWriter before canceling execution +type BlacklistedAcct struct { + Message string +} + +func (err *BlacklistedAcct) Error() string { + return err.Message +} + +func (err *BlacklistedAcct) Code() int { + return BlacklistedAcctCode +} + // BadServerResponse should be used when returning errors which are caused by bad/unexpected behavior on the remote server. // // For example: