CCPA Phase 2: Enforcement (prebid#1138)

adapters/syncer.go

@@ -46,7 +46,7 @@ func (s *Syncer) GetUsersyncInfo(privacyPolicies privacy.Policies) (*usersync.Us
 	syncURL, err := macros.ResolveMacros(*s.urlTemplate, macros.UserSyncTemplateParams{
 		GDPR:        privacyPolicies.GDPR.Signal,
 		GDPRConsent: privacyPolicies.GDPR.Consent,
-		USPrivacy:   privacyPolicies.CCPA.Signal,
+		USPrivacy:   privacyPolicies.CCPA.Value,
 	})
 	if err != nil {
 		return nil, err

adapters/syncer_test.go

@@ -17,7 +17,7 @@ func TestGetUsersyncInfo(t *testing.T) {
 			Consent: "B",
 		},
 		CCPA: ccpa.Policy{
-			Signal: "C",
+			Value: "C",
 		},
 	}
 

adapters/triplelift_native/triplelift_native.go

@@ -28,8 +28,6 @@ type TripleliftNativeExtInfo struct {
 	// Array is used for deserialization.
 	PublisherWhitelist []string `json:"publisher_whitelist"`
 
-	Endpoint string `json:"endpoint"`
-
 	// Map is used for optimized memory access and should be constructed after deserialization.
 	PublisherWhitelistMap map[string]bool
 }
@@ -198,5 +196,5 @@ func NewTripleliftNativeBidder(client *http.Client, endpoint string, extraInfo s
 
 	return &TripleliftNativeAdapter{
 		extInfo:  extInfo,
-		endpoint: extInfo.Endpoint}
+		endpoint: endpoint}
 }

adapters/triplelift_native/triplelift_native_test.go

@@ -19,5 +19,5 @@ func TestBadConfig(t *testing.T) {
 }
 
 func TestJsonSamples(t *testing.T) {
-	adapterstest.RunJSONBidderTest(t, "triplelift_nativetest", NewTripleliftNativeBidder(nil, "http://tlx.3lift.net/s2s/auction?supplier_id=19", "{\"publisher_whitelist\":[\"foo\",\"bar\",\"baz\"], \"endpoint\":\"http://tlx.3lift.net/s2sn/auction?supplier_id=20\"}"))
+	adapterstest.RunJSONBidderTest(t, "triplelift_nativetest", NewTripleliftNativeBidder(nil, "http://tlx.3lift.net/s2sn/auction?supplier_id=20", "{\"publisher_whitelist\":[\"foo\",\"bar\",\"baz\"]}"))
 }

config/config.go

@@ -47,6 +47,7 @@ type Configuration struct {
 	Analytics            Analytics          `mapstructure:"analytics"`
 	AMPTimeoutAdjustment int64              `mapstructure:"amp_timeout_adjustment_ms"`
 	GDPR                 GDPR               `mapstructure:"gdpr"`
+	CCPA                 CCPA               `mapstructure:"ccpa"`
 	CurrencyConverter    CurrencyConverter  `mapstructure:"currency_converter"`
 	DefReqConfig         DefReqConfig       `mapstructure:"default_request"`
 
@@ -160,6 +161,10 @@ func (t *GDPRTimeouts) ActiveTimeout() time.Duration {
 	return time.Duration(t.ActiveVendorlistFetch) * time.Millisecond
 }
 
+type CCPA struct {
+	Enforce bool `mapstructure:"enforce"`
+}
+
 type Analytics struct {
 	File FileLogs `mapstructure:"file"`
 }
@@ -202,6 +207,7 @@ const (
 	dummyPublisherID string = "12"
 	dummyGDPR        string = "0"
 	dummyGDPRConsent string = "someGDPRConsentString"
+	dummyCCPA        string = "1NYN"
 )
 
 type Adapter struct {
@@ -216,8 +222,9 @@ type Adapter struct {
 	//
 	// This value will be interpreted as a Golang Template. At runtime, the following Template variables will be replaced.
 	//
-	//   {{.GDPR}} -- This will be replaced with the "gdpr" property sent to /cookie_sync
-	//   {{.Consent}} -- This will be replaced with the "consent" property sent to /cookie_sync
+	//   {{.GDPR}}      -- This will be replaced with the "gdpr" property sent to /cookie_sync
+	//   {{.Consent}}   -- This will be replaced with the "consent" property sent to /cookie_sync
+	//   {{.USPrivacy}} -- This will be replaced with the "us_privacy" property sent to /cookie_sync
 	//
 	// For more info on templates, see: https://golang.org/pkg/text/template/
 	UserSyncURL string `mapstructure:"usersync_url"`
@@ -275,7 +282,12 @@ func validateAdapterUserSyncURL(userSyncURL string, adapterName string, errs con
 			return append(errs, fmt.Errorf("Invalid user sync URL template: %s for adapter: %s. %v", userSyncURL, adapterName, err))
 		}
 		// Resolve macros (if any) in the user_sync URL
-		resolvedUserSyncURL, err := macros.ResolveMacros(*userSyncTemplate, macros.UserSyncTemplateParams{GDPR: dummyGDPR, GDPRConsent: dummyGDPRConsent})
+		dummyMacroValues := macros.UserSyncTemplateParams{
+			GDPR:        dummyGDPR,
+			GDPRConsent: dummyGDPRConsent,
+			USPrivacy:   dummyCCPA,
+		}
+		resolvedUserSyncURL, err := macros.ResolveMacros(*userSyncTemplate, dummyMacroValues)
 		if err != nil {
 			return append(errs, fmt.Errorf("Unable to resolve user sync URL: %s for adapter: %s. %v", userSyncURL, adapterName, err))
 		}
@@ -707,6 +719,7 @@ func SetupViper(v *viper.Viper, filename string) {
 	v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0)
 	v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0)
 	v.SetDefault("gdpr.non_standard_publishers", []string{""})
+	v.SetDefault("ccpa.enforce", false)
 	v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json")
 	v.SetDefault("currency_converter.fetch_interval_seconds", 1800) // fetch currency rates every 30 minutes
 	v.SetDefault("default_request.type", "")

config/config_test.go

@@ -41,6 +41,8 @@ gdpr:
   host_vendor_id: 15
   usersync_if_ambiguous: true
   non_standard_publishers: ["siteID","fake-site-id","appID","agltb3B1Yi1pbmNyDAsSA0FwcBiJkfIUDA"]
+ccpa:
+  enforce: true
 host_cookie:
   cookie_name: userid
   family: prebid
@@ -102,7 +104,7 @@ adapters:
       username: rubiuser
       password: rubipw23
   brightroll:
-    usersync_url: http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&url=%s
+    usersync_url: http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=%s
     endpoint: http://test-bid.ybp.yahoo.com/bid/appnexuspbs
   adkerneladn:
      usersync_url: https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r=
@@ -230,6 +232,8 @@ func TestFullConfig(t *testing.T) {
 	_, found = cfg.GDPR.NonStandardPublisherMap["appnexus"]
 	cmpBools(t, "cfg.GDPR.NonStandardPublisherMap", found, false)
 
+	cmpBools(t, "ccpa.enforce", cfg.CCPA.Enforce, true)
+
 	//Assert the NonStandardPublishers was correctly unmarshalled
 	cmpStrings(t, "blacklisted_apps", cfg.BlacklistedApps[0], "spamAppID")
 	cmpStrings(t, "blacklisted_apps", cfg.BlacklistedApps[1], "sketchy-app-id")
@@ -265,7 +269,7 @@ func TestFullConfig(t *testing.T) {
 	cmpStrings(t, "adapters.rubicon.xapi.username", cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Username, "rubiuser")
 	cmpStrings(t, "adapters.rubicon.xapi.password", cfg.Adapters[string(openrtb_ext.BidderRubicon)].XAPI.Password, "rubipw23")
 	cmpStrings(t, "adapters.brightroll.endpoint", cfg.Adapters[string(openrtb_ext.BidderBrightroll)].Endpoint, "http://test-bid.ybp.yahoo.com/bid/appnexuspbs")
-	cmpStrings(t, "adapters.brightroll.usersync_url", cfg.Adapters[string(openrtb_ext.BidderBrightroll)].UserSyncURL, "http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&url=%s")
+	cmpStrings(t, "adapters.brightroll.usersync_url", cfg.Adapters[string(openrtb_ext.BidderBrightroll)].UserSyncURL, "http://test-bh.ybp.yahoo.com/sync/appnexuspbs?gdpr={{.GDPR}}&euconsent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&url=%s")
 	cmpStrings(t, "adapters.adkerneladn.usersync_url", cfg.Adapters[strings.ToLower(string(openrtb_ext.BidderAdkernelAdn))].UserSyncURL, "https://tag.adkernel.com/syncr?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&r=")
 	cmpStrings(t, "adapters.rhythmone.endpoint", cfg.Adapters[string(openrtb_ext.BidderRhythmone)].Endpoint, "http://tag.1rx.io/rmp")
 	cmpStrings(t, "adapters.rhythmone.usersync_url", cfg.Adapters[string(openrtb_ext.BidderRhythmone)].UserSyncURL, "https://sync.1rx.io/usersync2/rmphb?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&redir=http%3A%2F%2Fprebid-server.prebid.org%2F%2Fsetuid%3Fbidder%3Drhythmone%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%5BRX_UUID%5D")

endpoints/cookie_sync.go

@@ -32,6 +32,7 @@ func NewCookieSyncEndpoint(syncers map[openrtb_ext.BidderName]usersync.Usersynce
 		syncPermissions: syncPermissions,
 		metrics:         metrics,
 		pbsAnalytics:    pbsAnalytics,
+		enforceCCPA:     cfg.CCPA.Enforce,
 	}
 	return deps.Endpoint
 }
@@ -43,6 +44,7 @@ type cookieSyncDeps struct {
 	syncPermissions gdpr.Permissions
 	metrics         pbsmetrics.MetricsEngine
 	pbsAnalytics    analytics.PBSAnalyticsModule
+	enforceCCPA     bool
 }
 
 func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
@@ -103,15 +105,26 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h
 		}
 	}
 
+	privacyPolicy := privacy.Policies{
+		GDPR: gdprPolicy.Policy{
+			Signal:  gdprToString(parsedReq.GDPR),
+			Consent: parsedReq.Consent,
+		},
+		CCPA: ccpa.Policy{
+			Value: parsedReq.USPrivacy,
+		},
+	}
+
 	parsedReq.filterExistingSyncs(deps.syncers, userSyncCookie, needSyncupForSameSite)
+
 	adapterSyncs := make(map[openrtb_ext.BidderName]bool)
+	// assume all bidders will be privacy blocked
 	for _, b := range parsedReq.Bidders {
-		// assume all bidders will be GDPR blocked
 		adapterSyncs[openrtb_ext.BidderName(b)] = true
 	}
-	parsedReq.filterForGDPR(deps.syncPermissions)
+	parsedReq.filterForPrivacy(deps.syncPermissions, privacyPolicy, deps.enforceCCPA)
+	// surviving bidders are not privacy blocked
 	for _, b := range parsedReq.Bidders {
-		// surviving bidders are not GDPR blocked
 		adapterSyncs[openrtb_ext.BidderName(b)] = false
 	}
 	for b, g := range adapterSyncs {
@@ -125,15 +138,6 @@ func (deps *cookieSyncDeps) Endpoint(w http.ResponseWriter, r *http.Request, _ h
 	}
 	for i := 0; i < len(parsedReq.Bidders); i++ {
 		bidder := parsedReq.Bidders[i]
-		privacyPolicy := privacy.Policies{
-			GDPR: gdprPolicy.Policy{
-				Signal:  gdprToString(parsedReq.GDPR),
-				Consent: parsedReq.Consent,
-			},
-			CCPA: ccpa.Policy{
-				Signal: parsedReq.USPrivacy,
-			},
-		}
 		syncInfo, err := deps.syncers[openrtb_ext.BidderName(bidder)].GetUsersyncInfo(privacyPolicy)
 		if err == nil {
 			newSync := &usersync.CookieSyncBidders{
@@ -219,7 +223,12 @@ func (req *cookieSyncRequest) filterExistingSyncs(valid map[openrtb_ext.BidderNa
 	}
 }
 
-func (req *cookieSyncRequest) filterForGDPR(permissions gdpr.Permissions) {
+func (req *cookieSyncRequest) filterForPrivacy(permissions gdpr.Permissions, privacyPolicies privacy.Policies, enforceCCPA bool) {
+	if enforceCCPA && privacyPolicies.CCPA.ShouldEnforce() {
+		req.Bidders = nil
+		return
+	}
+
 	if req.GDPR != nil && *req.GDPR == 0 {
 		return
 	}

endpoints/cookie_sync_test.go

@@ -65,6 +65,61 @@ func TestGDPRConsentRequired(t *testing.T) {
 	assert.Equal(t, "gdpr_consent is required if gdpr=1\n", rr.Body.String())
 }
 
+func TestCCPA(t *testing.T) {
+	testCases := []struct {
+		description   string
+		requestBody   string
+		enforceCCPA   bool
+		expectedSyncs []string
+	}{
+		{
+			description:   "Feature Flag On & Opt-Out Yes",
+			requestBody:   `{"bidders":["appnexus"], "us_privacy":"1-Y-"}`,
+			enforceCCPA:   true,
+			expectedSyncs: []string{},
+		},
+		{
+			description:   "Feature Flag Off & Opt-Out Yes",
+			requestBody:   `{"bidders":["appnexus"], "us_privacy":"1-Y-"}`,
+			enforceCCPA:   false,
+			expectedSyncs: []string{"appnexus"},
+		},
+		{
+			description:   "Feature Flag On & Opt-Out No",
+			requestBody:   `{"bidders":["appnexus"], "us_privacy":"1-N-"}`,
+			enforceCCPA:   false,
+			expectedSyncs: []string{"appnexus"},
+		},
+		{
+			description:   "Feature Flag On & Opt-Out Unknown",
+			requestBody:   `{"bidders":["appnexus"], "us_privacy":"1---"}`,
+			enforceCCPA:   false,
+			expectedSyncs: []string{"appnexus"},
+		},
+		{
+			description:   "Feature Flag On & Opt-Out Invalid",
+			requestBody:   `{"bidders":["appnexus"], "us_privacy":"invalid"}`,
+			enforceCCPA:   false,
+			expectedSyncs: []string{"appnexus"},
+		},
+		{
+			description:   "Feature Flag On & Opt-Out Not Provided",
+			requestBody:   `{"bidders":["appnexus"]}`,
+			enforceCCPA:   false,
+			expectedSyncs: []string{"appnexus"},
+		},
+	}
+
+	for _, test := range testCases {
+		gdpr := config.GDPR{UsersyncIfAmbiguous: true}
+		ccpa := config.CCPA{Enforce: test.enforceCCPA}
+		rr := doConfigurablePost(test.requestBody, nil, true, syncersForTest(), gdpr, ccpa)
+		assert.Equal(t, http.StatusOK, rr.Code, test.description+":httpResponseCode")
+		assert.ElementsMatch(t, test.expectedSyncs, parseSyncs(t, rr.Body.Bytes()), test.description+":syncs")
+		assert.Equal(t, "no_cookie", parseStatus(t, rr.Body.Bytes()), test.description+":status")
+	}
+}
+
 func TestCookieSyncHasCookies(t *testing.T) {
 	rr := doPost(`{"bidders":["appnexus", "audienceNetwork", "random"]}`, map[string]string{
 		"adnxs":           "1234",
@@ -95,7 +150,7 @@ func TestCookieSyncNoBidders(t *testing.T) {
 }
 
 func TestCookieSyncNoCookiesBrokenGDPR(t *testing.T) {
-	rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true})
+	rr := doConfigurablePost(`{"bidders":["appnexus", "audienceNetwork", "random"],"gdpr_consent":"GLKHGKGKKGK"}`, nil, true, map[openrtb_ext.BidderName]usersync.Usersyncer{}, config.GDPR{UsersyncIfAmbiguous: true}, config.CCPA{})
 	assert.Equal(t, rr.Header().Get("Content-Type"), "application/json; charset=utf-8")
 	assert.Equal(t, http.StatusOK, rr.Code)
 	assert.ElementsMatch(t, []string{"appnexus", "audienceNetwork"}, parseSyncs(t, rr.Body.Bytes()))
@@ -118,15 +173,16 @@ func TestCookieSyncWithLargeLimit(t *testing.T) {
 }
 
 func doPost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer) *httptest.ResponseRecorder {
-	return doConfigurablePost(body, existingSyncs, gdprHostConsent, gdprBidders, config.GDPR{})
+	return doConfigurablePost(body, existingSyncs, gdprHostConsent, gdprBidders, config.GDPR{}, config.CCPA{})
 }
 
-func doConfigurablePost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer, cfgGDPR config.GDPR) *httptest.ResponseRecorder {
-	endpoint := testableEndpoint(mockPermissions(gdprHostConsent, gdprBidders), cfgGDPR)
+func doConfigurablePost(body string, existingSyncs map[string]string, gdprHostConsent bool, gdprBidders map[openrtb_ext.BidderName]usersync.Usersyncer, cfgGDPR config.GDPR, cfgCCPA config.CCPA) *httptest.ResponseRecorder {
+	endpoint := testableEndpoint(mockPermissions(gdprHostConsent, gdprBidders), cfgGDPR, cfgCCPA)
 	router := httprouter.New()
 	router.POST("/cookie_sync", endpoint)
 	req, _ := http.NewRequest("POST", "/cookie_sync", strings.NewReader(body))
 	if len(existingSyncs) > 0 {
+
 		pcs := usersync.NewPBSCookie()
 		for bidder, uid := range existingSyncs {
 			pcs.TrySync(bidder, uid)
@@ -139,8 +195,8 @@ func doConfigurablePost(body string, existingSyncs map[string]string, gdprHostCo
 	return rr
 }
 
-func testableEndpoint(perms gdpr.Permissions, cfgGDPR config.GDPR) httprouter.Handle {
-	return NewCookieSyncEndpoint(syncersForTest(), &config.Configuration{GDPR: cfgGDPR}, perms, &metricsConf.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}))
+func testableEndpoint(perms gdpr.Permissions, cfgGDPR config.GDPR, cfgCCPA config.CCPA) httprouter.Handle {
+	return NewCookieSyncEndpoint(syncersForTest(), &config.Configuration{GDPR: cfgGDPR, CCPA: cfgCCPA}, perms, &metricsConf.DummyMetricsEngine{}, analyticsConf.NewPBSAnalytics(&config.Analytics{}))
 }
 
 func syncersForTest() map[openrtb_ext.BidderName]usersync.Usersyncer {

endpoints/openrtb2/amp_auction.go

@@ -384,13 +384,12 @@ func (deps *endpointDeps) overrideWithParams(httpRequest *http.Request, req *ope
 
 	privacyPolicies := privacy.Policies{
 		GDPR: gdpr.Policy{
-			Consent: getQueryParam(httpRequest, "gdpr_consent"),
+			Consent: httpRequest.URL.Query().Get("gdpr_consent"),
 		},
 		CCPA: ccpa.Policy{
-			Signal: getQueryParam(httpRequest, "us_privacy"),
+			Value: httpRequest.URL.Query().Get("us_privacy"),
 		},
 	}
-
 	if err := privacyPolicies.Write(req); err != nil {
 		return err
 	}
@@ -402,17 +401,6 @@ func (deps *endpointDeps) overrideWithParams(httpRequest *http.Request, req *ope
 	return nil
 }
 
-func getQueryParam(httpRequest *http.Request, name string) string {
-	values, ok := httpRequest.URL.Query()[name]
-
-	if !ok || len(values) == 0 {
-		return ""
-	}
-
-	// return first value of the query param, matching the behavior of httpRequest.FormValue
-	return values[0]
-}
-
 func makeFormatReplacement(overrideWidth uint64, overrideHeight uint64, width uint64, height uint64, multisize string) []openrtb.Format {
 	if overrideWidth != 0 && overrideHeight != 0 {
 		return []openrtb.Format{{

endpoints/openrtb2/auction.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/pbsmetrics"
 	"github.com/prebid/prebid-server/prebid"
+	"github.com/prebid/prebid-server/privacy/ccpa"
 	"github.com/prebid/prebid-server/stored_requests"
 	"github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher"
 	"github.com/prebid/prebid-server/usersync"
@@ -301,6 +302,16 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) []error {
 		return errL
 	}
 
+	ccpaPolicy, ccpaPolicyErr := ccpa.ReadPolicy(req)
+	if ccpaPolicyErr != nil {
+		errL = append(errL, ccpaPolicyErr)
+		return errL
+	}
+
+	if err := ccpaPolicy.Validate(); err != nil {
+		errL = append(errL, &errortypes.Warning{Message: fmt.Sprintf("CCPA value is invalid and will be ignored. (%s)", err.Error())})
+	}
+
 	impIDs := make(map[string]int, len(req.Imp))
 	for index := range req.Imp {
 		imp := &req.Imp[index]