Stricter Privacy Scrubbing

exchange/utils.go

@@ -43,7 +43,6 @@ func cleanOpenRTBRequests(ctx context.Context,
 
 	gdpr := extractGDPR(orig, usersyncIfAmbiguous)
 	consent := extractConsent(orig)
-	isAMP := labels.RType == pbsmetrics.ReqTypeAMP
 
 	privacyEnforcement := privacy.Enforcement{
 		COPPA: orig.Regs != nil && orig.Regs.COPPA == 1,
@@ -66,7 +65,7 @@ func cleanOpenRTBRequests(ctx context.Context,
 			privacyEnforcement.GDPR = false
 		}
 
-		privacyEnforcement.Apply(bidReq, isAMP)
+		privacyEnforcement.Apply(bidReq)
 	}
 
 	return

go.mod

@@ -55,7 +55,7 @@ require (
 	github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834 // indirect
 	github.com/spf13/pflag v1.0.2 // indirect
 	github.com/spf13/viper v1.1.0
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.5.1
 	github.com/valyala/fasthttp v1.9.0
 	github.com/vrischmann/go-metrics-influxdb v0.0.0-20160917065939-43af8332c303
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
@@ -70,5 +70,5 @@ require (
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	golang.org/x/text v0.3.0
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -143,6 +143,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
@@ -196,3 +198,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

privacy/enforcement.go

@@ -17,21 +17,17 @@ func (e Enforcement) Any() bool {
 }
 
 // Apply cleans personally identifiable information from an OpenRTB bid request.
-func (e Enforcement) Apply(bidRequest *openrtb.BidRequest, isAMP bool) {
-	e.apply(bidRequest, isAMP, NewScrubber())
+func (e Enforcement) Apply(bidRequest *openrtb.BidRequest) {
+	e.apply(bidRequest, NewScrubber())
 }
 
-func (e Enforcement) apply(bidRequest *openrtb.BidRequest, isAMP bool, scrubber Scrubber) {
+func (e Enforcement) apply(bidRequest *openrtb.BidRequest, scrubber Scrubber) {
 	if bidRequest != nil && e.Any() {
-		bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceMacAndIFA(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
-		bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(isAMP), e.getGeoScrubStrategy())
+		bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
+		bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getDemographicScrubStrategy(), e.getGeoScrubStrategy())
 	}
 }
 
-func (e Enforcement) getDeviceMacAndIFA() bool {
-	return e.COPPA
-}
-
 func (e Enforcement) getIPv6ScrubStrategy() ScrubStrategyIPV6 {
 	if e.COPPA {
 		return ScrubStrategyIPV6Lowest32
@@ -56,21 +52,10 @@ func (e Enforcement) getGeoScrubStrategy() ScrubStrategyGeo {
 	return ScrubStrategyGeoNone
 }
 
-func (e Enforcement) getUserScrubStrategy(isAMP bool) ScrubStrategyUser {
+func (e Enforcement) getDemographicScrubStrategy() ScrubStrategyDemographic {
 	if e.COPPA {
-		return ScrubStrategyUserFull
-	}
-
-	// There's no way for AMP to send a GDPR consent string yet so it's hard
-	// to know if the vendor is consented or not and therefore for AMP requests
-	// we keep the BuyerUID as is for GDPR.
-	if e.GDPR && isAMP {
-		return ScrubStrategyUserNone
-	}
-
-	if e.GDPR || e.CCPA {
-		return ScrubStrategyUserBuyerIDOnly
+		return ScrubStrategyDemographicAgeAndGender
 	}
 
-	return ScrubStrategyUserNone
+	return ScrubStrategyDemographicNone
 }

privacy/enforcement_test.go

@@ -51,164 +51,115 @@ func TestAny(t *testing.T) {
 
 func TestApply(t *testing.T) {
 	testCases := []struct {
+		description             string
 		enforcement             Enforcement
-		isAMP                   bool
-		expectedDeviceMacAndIFA bool
 		expectedDeviceIPv6      ScrubStrategyIPV6
 		expectedDeviceGeo       ScrubStrategyGeo
-		expectedUser            ScrubStrategyUser
+		expectedUserDemographic ScrubStrategyDemographic
 		expectedUserGeo         ScrubStrategyGeo
-		description             string
 	}{
 		{
+			description: "All Enforced",
 			enforcement: Enforcement{
 				CCPA:  true,
 				COPPA: true,
 				GDPR:  true,
 			},
-			isAMP:                   true,
-			expectedDeviceMacAndIFA: true,
-			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest32,
-			expectedDeviceGeo:       ScrubStrategyGeoFull,
-			expectedUser:            ScrubStrategyUserFull,
-			expectedUserGeo:         ScrubStrategyGeoFull,
-			description:             "All Enforced - Most Strict",
-		},
-		{
-			enforcement: Enforcement{
-				CCPA:  false,
-				COPPA: true,
-				GDPR:  false,
-			},
-			isAMP:                   false,
-			expectedDeviceMacAndIFA: true,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest32,
 			expectedDeviceGeo:       ScrubStrategyGeoFull,
-			expectedUser:            ScrubStrategyUserFull,
+			expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
 			expectedUserGeo:         ScrubStrategyGeoFull,
-			description:             "COPPA",
-		},
-		{
-			enforcement: Enforcement{
-				CCPA:  false,
-				COPPA: false,
-				GDPR:  true,
-			},
-			isAMP:                   false,
-			expectedDeviceMacAndIFA: false,
-			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
-			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserBuyerIDOnly,
-			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "GDPR",
-		},
-		{
-			enforcement: Enforcement{
-				CCPA:  false,
-				COPPA: false,
-				GDPR:  true,
-			},
-			isAMP:                   true,
-			expectedDeviceMacAndIFA: false,
-			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
-			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserNone,
-			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "GDPR For AMP",
 		},
 		{
+			description: "CCPA Only",
 			enforcement: Enforcement{
 				CCPA:  true,
 				COPPA: false,
 				GDPR:  false,
 			},
-			isAMP:                   false,
-			expectedDeviceMacAndIFA: false,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
 			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserBuyerIDOnly,
+			expectedUserDemographic: ScrubStrategyDemographicNone,
 			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "CCPA",
 		},
 		{
+			description: "COPPA Only",
 			enforcement: Enforcement{
-				CCPA:  true,
-				COPPA: false,
+				CCPA:  false,
+				COPPA: true,
 				GDPR:  false,
 			},
-			isAMP:                   true,
-			expectedDeviceMacAndIFA: false,
-			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
-			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserBuyerIDOnly,
-			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "CCPA For AMP",
+			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest32,
+			expectedDeviceGeo:       ScrubStrategyGeoFull,
+			expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
+			expectedUserGeo:         ScrubStrategyGeoFull,
 		},
 		{
+			description: "GDPR Only",
 			enforcement: Enforcement{
-				CCPA:  true,
+				CCPA:  false,
 				COPPA: false,
 				GDPR:  true,
 			},
-			isAMP:                   true,
-			expectedDeviceMacAndIFA: false,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
 			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserNone,
+			expectedUserDemographic: ScrubStrategyDemographicNone,
 			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "GDPR And CCPA For AMP",
 		},
 	}
 
 	for _, test := range testCases {
 		req := &openrtb.BidRequest{
-			Device: &openrtb.Device{DIDSHA1: "before"},
-			User:   &openrtb.User{ID: "before"},
+			Device: &openrtb.Device{},
+			User:   &openrtb.User{},
 		}
-		device := &openrtb.Device{DIDSHA1: "after"}
-		user := &openrtb.User{ID: "after"}
+		replacedDevice := &openrtb.Device{}
+		replacedUser := &openrtb.User{}
 
 		m := &mockScrubber{}
-		m.On("ScrubDevice", req.Device, test.expectedDeviceMacAndIFA, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(device).Once()
-		m.On("ScrubUser", req.User, test.expectedUser, test.expectedUserGeo).Return(user).Once()
+		m.On("ScrubDevice", req.Device, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(replacedDevice).Once()
+		m.On("ScrubUser", req.User, test.expectedUserDemographic, test.expectedUserGeo).Return(replacedUser).Once()
 
-		test.enforcement.apply(req, test.isAMP, m)
+		test.enforcement.apply(req, m)
 
 		m.AssertExpectations(t)
-		assert.Equal(t, device, req.Device, "Device Set Correctly")
-		assert.Equal(t, user, req.User, "User Set Correctly")
+		assert.Same(t, replacedDevice, req.Device, "Device")
+		assert.Same(t, replacedUser, req.User, "User")
 	}
 }
 
 func TestApplyNoneApplicable(t *testing.T) {
+	req := &openrtb.BidRequest{}
+
+	m := &mockScrubber{}
+
 	enforcement := Enforcement{}
-	device := &openrtb.Device{DIDSHA1: "original"}
-	user := &openrtb.User{ID: "original"}
-	req := &openrtb.BidRequest{
-		Device: device,
-		User:   user,
-	}
+	enforcement.apply(req, m)
+
+	m.AssertNotCalled(t, "ScrubDevice")
+	m.AssertNotCalled(t, "ScrubUser")
+}
 
+func TestApplyNil(t *testing.T) {
 	m := &mockScrubber{}
 
-	enforcement.apply(req, true, m)
+	enforcement := Enforcement{}
+	enforcement.apply(nil, m)
 
 	m.AssertNotCalled(t, "ScrubDevice")
 	m.AssertNotCalled(t, "ScrubUser")
-	assert.Equal(t, device, req.Device, "Device Set Correctly")
-	assert.Equal(t, user, req.User, "User Set Correctly")
 }
 
 type mockScrubber struct {
 	mock.Mock
 }
 
-func (m *mockScrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
-	args := m.Called(device, macAndIFA, ipv6, geo)
+func (m *mockScrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
+	args := m.Called(device, ipv6, geo)
 	return args.Get(0).(*openrtb.Device)
 }
 
-func (m *mockScrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User {
-	args := m.Called(user, strategy, geo)
+func (m *mockScrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User {
+	args := m.Called(user, demographic, geo)
 	return args.Get(0).(*openrtb.User)
 }