diff --git a/config/config.go b/config/config.go index 4c1d6983bb8..beb6e0a6844 100755 --- a/config/config.go +++ b/config/config.go @@ -142,6 +142,7 @@ type GDPR struct { Timeouts GDPRTimeouts `mapstructure:"timeouts_ms"` NonStandardPublishers []string `mapstructure:"non_standard_publishers,flow"` NonStandardPublisherMap map[string]int + AMPException bool `mapstructure:"amp_exception"` } func (cfg *GDPR) validate(errs configErrors) configErrors { @@ -768,6 +769,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0) v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0) v.SetDefault("gdpr.non_standard_publishers", []string{""}) + v.SetDefault("gdpr.amp_exception", false) v.SetDefault("ccpa.enforce", false) v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json") v.SetDefault("currency_converter.fetch_interval_seconds", 1800) // fetch currency rates every 30 minutes diff --git a/endpoints/auction_test.go b/endpoints/auction_test.go index df91f8ca004..3035a6d45fb 100644 --- a/endpoints/auction_test.go +++ b/endpoints/auction_test.go @@ -421,6 +421,10 @@ func (m *auctionMockPermissions) PersonalInfoAllowed(ctx context.Context, bidder return m.allowPI, nil } +func (m *auctionMockPermissions) AMPException() bool { + return false +} + func TestBidSizeValidate(t *testing.T) { bids := make(pbs.PBSBidSlice, 0) // bid1 will be rejected due to undefined size when adunit has multiple sizes diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 011cecf3341..bb766aa92e7 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -257,3 +257,7 @@ func (g *gdprPerms) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.Bi func (g *gdprPerms) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error) { return true, nil } + +func (g *gdprPerms) AMPException() bool { + return false +} diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 555f6173b0f..8499ac1ca5d 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -441,6 +441,10 @@ func (g *mockPermsSetUID) PersonalInfoAllowed(ctx context.Context, bidder openrt return g.allowPI, nil } +func (g *mockPermsSetUID) AMPException() bool { + return false +} + func newFakeSyncer(familyName string) usersync.Usersyncer { return fakeSyncer{ familyName: familyName, diff --git a/exchange/utils.go b/exchange/utils.go index f09b11513f1..d961089c4cb 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -43,6 +43,7 @@ func cleanOpenRTBRequests(ctx context.Context, gdpr := extractGDPR(orig, usersyncIfAmbiguous) consent := extractConsent(orig) + ampGDPRException := (labels.RType == pbsmetrics.ReqTypeAMP) && gDPR.AMPException() privacyEnforcement := privacy.Enforcement{ COPPA: orig.Regs != nil && orig.Regs.COPPA == 1, @@ -65,7 +66,7 @@ func cleanOpenRTBRequests(ctx context.Context, privacyEnforcement.GDPR = false } - privacyEnforcement.Apply(bidReq) + privacyEnforcement.Apply(bidReq, ampGDPRException) } return diff --git a/exchange/utils_test.go b/exchange/utils_test.go index edbe04a0d0f..53d6b85c243 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -31,6 +31,10 @@ func (p *permissionsMock) PersonalInfoAllowed(ctx context.Context, bidder openrt return false, nil } +func (p *permissionsMock) AMPException() bool { + return false +} + func assertReq(t *testing.T, reqByBidders map[openrtb_ext.BidderName]*openrtb.BidRequest, applyCOPPA bool, consentedVendors map[string]bool) { // assert individual bidder requests diff --git a/gdpr/gdpr.go b/gdpr/gdpr.go index 4e36e22fdb9..9390d942f80 100644 --- a/gdpr/gdpr.go +++ b/gdpr/gdpr.go @@ -24,6 +24,9 @@ type Permissions interface { // // If the consent string was nonsensical, the returned error will be an ErrorMalformedConsent. PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error) + + // Exposes the AMP execption flag + AMPException() bool } const ( diff --git a/gdpr/impl.go b/gdpr/impl.go index 615c3a090c9..8743d7f2778 100644 --- a/gdpr/impl.go +++ b/gdpr/impl.go @@ -58,6 +58,10 @@ func (p *permissionsImpl) PersonalInfoAllowed(ctx context.Context, bidder openrt return false, nil } +func (p *permissionsImpl) AMPException() bool { + return p.cfg.AMPException +} + func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, consent string) (bool, error) { // If we're not given a consent string, respect the preferences in the app config. if consent == "" { @@ -145,3 +149,7 @@ func (a AlwaysAllow) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.B func (a AlwaysAllow) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error) { return true, nil } + +func (a AlwaysAllow) AMPException() bool { + return false +} diff --git a/privacy/enforcement.go b/privacy/enforcement.go index 0230ca6b9af..96d03ef4433 100644 --- a/privacy/enforcement.go +++ b/privacy/enforcement.go @@ -17,14 +17,14 @@ func (e Enforcement) Any() bool { } // Apply cleans personally identifiable information from an OpenRTB bid request. -func (e Enforcement) Apply(bidRequest *openrtb.BidRequest) { - e.apply(bidRequest, NewScrubber()) +func (e Enforcement) Apply(bidRequest *openrtb.BidRequest, ampGDPRException bool) { + e.apply(bidRequest, ampGDPRException, NewScrubber()) } -func (e Enforcement) apply(bidRequest *openrtb.BidRequest, scrubber Scrubber) { +func (e Enforcement) apply(bidRequest *openrtb.BidRequest, ampGDPRException bool, scrubber Scrubber) { if bidRequest != nil && e.Any() { bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy()) - bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getDemographicScrubStrategy(), e.getGeoScrubStrategy()) + bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(ampGDPRException), e.getGeoScrubStrategy()) } } @@ -52,10 +52,13 @@ func (e Enforcement) getGeoScrubStrategy() ScrubStrategyGeo { return ScrubStrategyGeoNone } -func (e Enforcement) getDemographicScrubStrategy() ScrubStrategyDemographic { +func (e Enforcement) getUserScrubStrategy(ampGDPRException bool) ScrubStrategyUser { if e.COPPA { - return ScrubStrategyDemographicAgeAndGender + return ScrubStrategyUserIDAndDemographic } - return ScrubStrategyDemographicNone + if e.GDPR && ampGDPRException { + return ScrubStrategyUserNone + } + return ScrubStrategyUserID } diff --git a/privacy/enforcement_test.go b/privacy/enforcement_test.go index c7433f8b271..25e08b5e80d 100644 --- a/privacy/enforcement_test.go +++ b/privacy/enforcement_test.go @@ -51,12 +51,13 @@ func TestAny(t *testing.T) { func TestApply(t *testing.T) { testCases := []struct { - description string - enforcement Enforcement - expectedDeviceIPv6 ScrubStrategyIPV6 - expectedDeviceGeo ScrubStrategyGeo - expectedUserDemographic ScrubStrategyDemographic - expectedUserGeo ScrubStrategyGeo + description string + enforcement Enforcement + ampGDPRException bool + expectedDeviceIPv6 ScrubStrategyIPV6 + expectedDeviceGeo ScrubStrategyGeo + expectedUser ScrubStrategyUser + expectedUserGeo ScrubStrategyGeo }{ { description: "All Enforced", @@ -65,10 +66,11 @@ func TestApply(t *testing.T) { COPPA: true, GDPR: true, }, - expectedDeviceIPv6: ScrubStrategyIPV6Lowest32, - expectedDeviceGeo: ScrubStrategyGeoFull, - expectedUserDemographic: ScrubStrategyDemographicAgeAndGender, - expectedUserGeo: ScrubStrategyGeoFull, + ampGDPRException: false, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest32, + expectedDeviceGeo: ScrubStrategyGeoFull, + expectedUser: ScrubStrategyUserIDAndDemographic, + expectedUserGeo: ScrubStrategyGeoFull, }, { description: "CCPA Only", @@ -77,10 +79,11 @@ func TestApply(t *testing.T) { COPPA: false, GDPR: false, }, - expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, - expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, - expectedUserDemographic: ScrubStrategyDemographicNone, - expectedUserGeo: ScrubStrategyGeoReducedPrecision, + ampGDPRException: false, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, + expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, + expectedUser: ScrubStrategyUserID, + expectedUserGeo: ScrubStrategyGeoReducedPrecision, }, { description: "COPPA Only", @@ -89,10 +92,11 @@ func TestApply(t *testing.T) { COPPA: true, GDPR: false, }, - expectedDeviceIPv6: ScrubStrategyIPV6Lowest32, - expectedDeviceGeo: ScrubStrategyGeoFull, - expectedUserDemographic: ScrubStrategyDemographicAgeAndGender, - expectedUserGeo: ScrubStrategyGeoFull, + ampGDPRException: false, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest32, + expectedDeviceGeo: ScrubStrategyGeoFull, + expectedUser: ScrubStrategyUserIDAndDemographic, + expectedUserGeo: ScrubStrategyGeoFull, }, { description: "GDPR Only", @@ -101,10 +105,50 @@ func TestApply(t *testing.T) { COPPA: false, GDPR: true, }, - expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, - expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, - expectedUserDemographic: ScrubStrategyDemographicNone, - expectedUserGeo: ScrubStrategyGeoReducedPrecision, + ampGDPRException: false, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, + expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, + expectedUser: ScrubStrategyUserID, + expectedUserGeo: ScrubStrategyGeoReducedPrecision, + }, + { + description: "GDPR Only, ampGDPRException", + enforcement: Enforcement{ + CCPA: false, + COPPA: false, + GDPR: true, + }, + ampGDPRException: true, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, + expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, + expectedUser: ScrubStrategyUserNone, + expectedUserGeo: ScrubStrategyGeoReducedPrecision, + }, + { + description: "CCPA Only, ampGDPRException", + enforcement: Enforcement{ + CCPA: true, + COPPA: false, + GDPR: false, + }, + ampGDPRException: true, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest16, + expectedDeviceGeo: ScrubStrategyGeoReducedPrecision, + expectedUser: ScrubStrategyUserID, + expectedUserGeo: ScrubStrategyGeoReducedPrecision, + }, + { + description: "COPPA and GDPR, ampGDPRException", + enforcement: Enforcement{ + CCPA: false, + COPPA: true, + GDPR: true, + }, + ampGDPRException: true, + expectedDeviceIPv6: ScrubStrategyIPV6Lowest32, + expectedDeviceGeo: ScrubStrategyGeoFull, + expectedUser: ScrubStrategyUserIDAndDemographic, + expectedUserGeo: ScrubStrategyGeoFull, }, } @@ -118,9 +162,9 @@ func TestApply(t *testing.T) { m := &mockScrubber{} m.On("ScrubDevice", req.Device, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(replacedDevice).Once() - m.On("ScrubUser", req.User, test.expectedUserDemographic, test.expectedUserGeo).Return(replacedUser).Once() + m.On("ScrubUser", req.User, test.expectedUser, test.expectedUserGeo).Return(replacedUser).Once() - test.enforcement.apply(req, m) + test.enforcement.apply(req, test.ampGDPRException, m) m.AssertExpectations(t) assert.Same(t, replacedDevice, req.Device, "Device") @@ -138,7 +182,7 @@ func TestApplyNoneApplicable(t *testing.T) { COPPA: false, GDPR: false, } - enforcement.apply(req, m) + enforcement.apply(req, false, m) m.AssertNotCalled(t, "ScrubDevice") m.AssertNotCalled(t, "ScrubUser") @@ -148,7 +192,7 @@ func TestApplyNil(t *testing.T) { m := &mockScrubber{} enforcement := Enforcement{} - enforcement.apply(nil, m) + enforcement.apply(nil, false, m) m.AssertNotCalled(t, "ScrubDevice") m.AssertNotCalled(t, "ScrubUser") @@ -163,7 +207,7 @@ func (m *mockScrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV return args.Get(0).(*openrtb.Device) } -func (m *mockScrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User { - args := m.Called(user, demographic, geo) +func (m *mockScrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User { + args := m.Called(user, strategy, geo) return args.Get(0).(*openrtb.User) } diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 45b79c20a4e..ec313e91a80 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -34,21 +34,24 @@ const ( ScrubStrategyGeoReducedPrecision ) -// ScrubStrategyDemographic defines the approach to non-location demographic data. -type ScrubStrategyDemographic int +// ScrubStrategyUser defines the approach to scrub PII from user data. +type ScrubStrategyUser int const ( - // ScrubStrategyDemographicNone does not remove non-location demographic data. - ScrubStrategyDemographicNone ScrubStrategyDemographic = iota + // ScrubStrategyUserNone does not remove non-location data. + ScrubStrategyUserNone ScrubStrategyUser = iota - // ScrubStrategyDemographicAgeAndGender removes age and gender data. - ScrubStrategyDemographicAgeAndGender + // ScrubStrategyUserIDAndDemographic removes the user's buyer id, exchange id year of birth, and gender. + ScrubStrategyUserIDAndDemographic + + // ScrubStrategyUserID removes the user's buyer id. + ScrubStrategyUserID ) // Scrubber removes PII from parts of an OpenRTB request. type Scrubber interface { ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device - ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User + ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User } type scrubber struct{} @@ -90,19 +93,22 @@ func (scrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo return &deviceCopy } -func (scrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User { +func (scrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User { if user == nil { return nil } userCopy := *user - userCopy.BuyerUID = "" - userCopy.ID = "" - switch demographic { - case ScrubStrategyDemographicAgeAndGender: + switch strategy { + case ScrubStrategyUserIDAndDemographic: + userCopy.BuyerUID = "" + userCopy.ID = "" userCopy.Yob = 0 userCopy.Gender = "" + case ScrubStrategyUserID: + userCopy.BuyerUID = "" + userCopy.ID = "" } switch geo { diff --git a/privacy/scrubber_test.go b/privacy/scrubber_test.go index 2d5ee667538..a6c4bde3742 100644 --- a/privacy/scrubber_test.go +++ b/privacy/scrubber_test.go @@ -252,7 +252,7 @@ func TestScrubUser(t *testing.T) { testCases := []struct { expected *openrtb.User - demographic ScrubStrategyDemographic + user ScrubStrategyUser geo ScrubStrategyGeo description string }{ @@ -265,8 +265,8 @@ func TestScrubUser(t *testing.T) { Gender: "", Geo: &openrtb.Geo{}, }, - demographic: ScrubStrategyDemographicAgeAndGender, - geo: ScrubStrategyGeoFull, + user: ScrubStrategyUserIDAndDemographic, + geo: ScrubStrategyGeoFull, }, { description: "Demographic Age And Gender & Geo Reduced", @@ -283,11 +283,11 @@ func TestScrubUser(t *testing.T) { ZIP: "some zip", }, }, - demographic: ScrubStrategyDemographicAgeAndGender, - geo: ScrubStrategyGeoReducedPrecision, + user: ScrubStrategyUserIDAndDemographic, + geo: ScrubStrategyGeoReducedPrecision, }, { - description: "Demographic Age And Gender & Geo None", + description: "Demographic Full & Geo None", expected: &openrtb.User{ ID: "", BuyerUID: "", @@ -301,11 +301,11 @@ func TestScrubUser(t *testing.T) { ZIP: "some zip", }, }, - demographic: ScrubStrategyDemographicAgeAndGender, - geo: ScrubStrategyGeoNone, + user: ScrubStrategyUserIDAndDemographic, + geo: ScrubStrategyGeoNone, }, { - description: "Demographic None & Geo Full", + description: "Demographic Buyer ID & Geo Full", expected: &openrtb.User{ ID: "", BuyerUID: "", @@ -313,11 +313,11 @@ func TestScrubUser(t *testing.T) { Gender: "anyGender", Geo: &openrtb.Geo{}, }, - demographic: ScrubStrategyDemographicNone, - geo: ScrubStrategyGeoFull, + user: ScrubStrategyUserID, + geo: ScrubStrategyGeoFull, }, { - description: "Demographic None & Geo Reduced", + description: "Demographic Buyer ID & Geo Reduced", expected: &openrtb.User{ ID: "", BuyerUID: "", @@ -331,11 +331,29 @@ func TestScrubUser(t *testing.T) { ZIP: "some zip", }, }, - demographic: ScrubStrategyDemographicNone, - geo: ScrubStrategyGeoReducedPrecision, + user: ScrubStrategyUserID, + geo: ScrubStrategyGeoReducedPrecision, }, { description: "Demographic None & Geo None", + expected: &openrtb.User{ + ID: "anyID", + BuyerUID: "anyBuyerUID", + Yob: 42, + Gender: "anyGender", + Geo: &openrtb.Geo{ + Lat: 123.456, + Lon: 678.89, + Metro: "some metro", + City: "some city", + ZIP: "some zip", + }, + }, + user: ScrubStrategyUserNone, + geo: ScrubStrategyGeoNone, + }, + { + description: "Demographic BuyerIDOnly & Geo None", expected: &openrtb.User{ ID: "", BuyerUID: "", @@ -349,19 +367,19 @@ func TestScrubUser(t *testing.T) { ZIP: "some zip", }, }, - demographic: ScrubStrategyDemographicNone, - geo: ScrubStrategyGeoNone, + user: ScrubStrategyUserID, + geo: ScrubStrategyGeoNone, }, } for _, test := range testCases { - result := NewScrubber().ScrubUser(user, test.demographic, test.geo) + result := NewScrubber().ScrubUser(user, test.user, test.geo) assert.Equal(t, test.expected, result, test.description) } } func TestScrubUserNil(t *testing.T) { - result := NewScrubber().ScrubUser(nil, ScrubStrategyDemographicNone, ScrubStrategyGeoNone) + result := NewScrubber().ScrubUser(nil, ScrubStrategyUserNone, ScrubStrategyGeoNone) assert.Nil(t, result) }