Stricter Privacy Scrubbing (prebid#1286)

* Stricter Privacy Scrubbing

* Update Unit Test Style

* Fixed Whitespace

go.mod

@@ -55,7 +55,7 @@ require (
 	github.com/spf13/jwalterweatherman v0.0.0-20180814060501-14d3d4c51834 // indirect
 	github.com/spf13/pflag v1.0.2 // indirect
 	github.com/spf13/viper v1.1.0
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.5.1
 	github.com/valyala/fasthttp v1.9.0
 	github.com/vrischmann/go-metrics-influxdb v0.0.0-20160917065939-43af8332c303
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
@@ -70,5 +70,5 @@ require (
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	golang.org/x/text v0.3.0
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -143,6 +143,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
@@ -196,3 +198,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

privacy/enforcement.go

@@ -23,15 +23,11 @@ func (e Enforcement) Apply(bidRequest *openrtb.BidRequest) {
 
 func (e Enforcement) apply(bidRequest *openrtb.BidRequest, scrubber Scrubber) {
 	if bidRequest != nil && e.Any() {
-		bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceMacAndIFA(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
-		bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(), e.getGeoScrubStrategy())
+		bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
+		bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getDemographicScrubStrategy(), e.getGeoScrubStrategy())
 	}
 }
 
-func (e Enforcement) getDeviceMacAndIFA() bool {
-	return e.COPPA
-}
-
 func (e Enforcement) getIPv6ScrubStrategy() ScrubStrategyIPV6 {
 	if e.COPPA {
 		return ScrubStrategyIPV6Lowest32
@@ -56,14 +52,10 @@ func (e Enforcement) getGeoScrubStrategy() ScrubStrategyGeo {
 	return ScrubStrategyGeoNone
 }
 
-func (e Enforcement) getUserScrubStrategy() ScrubStrategyUser {
+func (e Enforcement) getDemographicScrubStrategy() ScrubStrategyDemographic {
 	if e.COPPA {
-		return ScrubStrategyUserFull
-	}
-
-	if e.GDPR || e.CCPA {
-		return ScrubStrategyUserBuyerIDOnly
+		return ScrubStrategyDemographicAgeAndGender
 	}
 
-	return ScrubStrategyUserNone
+	return ScrubStrategyDemographicNone
 }

privacy/enforcement_test.go

@@ -15,31 +15,31 @@ func TestAny(t *testing.T) {
 		description string
 	}{
 		{
+			description: "All False",
 			enforcement: Enforcement{
 				CCPA:  false,
 				COPPA: false,
 				GDPR:  false,
 			},
-			expected:    false,
-			description: "All False",
+			expected: false,
 		},
 		{
+			description: "All True",
 			enforcement: Enforcement{
 				CCPA:  true,
 				COPPA: true,
 				GDPR:  true,
 			},
-			expected:    true,
-			description: "All True",
+			expected: true,
 		},
 		{
+			description: "Mixed",
 			enforcement: Enforcement{
 				CCPA:  false,
 				COPPA: true,
 				GDPR:  false,
 			},
-			expected:    true,
-			description: "Mixed",
+			expected: true,
 		},
 	}
 
@@ -51,117 +51,119 @@ func TestAny(t *testing.T) {
 
 func TestApply(t *testing.T) {
 	testCases := []struct {
+		description             string
 		enforcement             Enforcement
-		expectedDeviceMacAndIFA bool
 		expectedDeviceIPv6      ScrubStrategyIPV6
 		expectedDeviceGeo       ScrubStrategyGeo
-		expectedUser            ScrubStrategyUser
+		expectedUserDemographic ScrubStrategyDemographic
 		expectedUserGeo         ScrubStrategyGeo
-		description             string
 	}{
 		{
+			description: "All Enforced",
 			enforcement: Enforcement{
 				CCPA:  true,
 				COPPA: true,
 				GDPR:  true,
 			},
-			expectedDeviceMacAndIFA: true,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest32,
 			expectedDeviceGeo:       ScrubStrategyGeoFull,
-			expectedUser:            ScrubStrategyUserFull,
+			expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
 			expectedUserGeo:         ScrubStrategyGeoFull,
-			description:             "All Enforced - Most Strict",
 		},
 		{
+			description: "CCPA Only",
+			enforcement: Enforcement{
+				CCPA:  true,
+				COPPA: false,
+				GDPR:  false,
+			},
+			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
+			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
+			expectedUserDemographic: ScrubStrategyDemographicNone,
+			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
+		},
+		{
+			description: "COPPA Only",
 			enforcement: Enforcement{
 				CCPA:  false,
 				COPPA: true,
 				GDPR:  false,
 			},
-			expectedDeviceMacAndIFA: true,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest32,
 			expectedDeviceGeo:       ScrubStrategyGeoFull,
-			expectedUser:            ScrubStrategyUserFull,
+			expectedUserDemographic: ScrubStrategyDemographicAgeAndGender,
 			expectedUserGeo:         ScrubStrategyGeoFull,
-			description:             "COPPA",
 		},
 		{
+			description: "GDPR Only",
 			enforcement: Enforcement{
 				CCPA:  false,
 				COPPA: false,
 				GDPR:  true,
 			},
-			expectedDeviceMacAndIFA: false,
 			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
 			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserBuyerIDOnly,
+			expectedUserDemographic: ScrubStrategyDemographicNone,
 			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "GDPR",
-		},
-		{
-			enforcement: Enforcement{
-				CCPA:  true,
-				COPPA: false,
-				GDPR:  false,
-			},
-			expectedDeviceMacAndIFA: false,
-			expectedDeviceIPv6:      ScrubStrategyIPV6Lowest16,
-			expectedDeviceGeo:       ScrubStrategyGeoReducedPrecision,
-			expectedUser:            ScrubStrategyUserBuyerIDOnly,
-			expectedUserGeo:         ScrubStrategyGeoReducedPrecision,
-			description:             "CCPA",
 		},
 	}
 
 	for _, test := range testCases {
 		req := &openrtb.BidRequest{
-			Device: &openrtb.Device{DIDSHA1: "before"},
-			User:   &openrtb.User{ID: "before"},
+			Device: &openrtb.Device{},
+			User:   &openrtb.User{},
 		}
-		device := &openrtb.Device{DIDSHA1: "after"}
-		user := &openrtb.User{ID: "after"}
+		replacedDevice := &openrtb.Device{}
+		replacedUser := &openrtb.User{}
 
 		m := &mockScrubber{}
-		m.On("ScrubDevice", req.Device, test.expectedDeviceMacAndIFA, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(device).Once()
-		m.On("ScrubUser", req.User, test.expectedUser, test.expectedUserGeo).Return(user).Once()
+		m.On("ScrubDevice", req.Device, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(replacedDevice).Once()
+		m.On("ScrubUser", req.User, test.expectedUserDemographic, test.expectedUserGeo).Return(replacedUser).Once()
 
 		test.enforcement.apply(req, m)
 
 		m.AssertExpectations(t)
-		assert.Equal(t, device, req.Device, "Device Set Correctly")
-		assert.Equal(t, user, req.User, "User Set Correctly")
+		assert.Same(t, replacedDevice, req.Device, "Device")
+		assert.Same(t, replacedUser, req.User, "User")
 	}
 }
 
 func TestApplyNoneApplicable(t *testing.T) {
-	enforcement := Enforcement{}
-	device := &openrtb.Device{DIDSHA1: "original"}
-	user := &openrtb.User{ID: "original"}
-	req := &openrtb.BidRequest{
-		Device: device,
-		User:   user,
-	}
+	req := &openrtb.BidRequest{}
 
 	m := &mockScrubber{}
 
+	enforcement := Enforcement{
+		CCPA:  false,
+		COPPA: false,
+		GDPR:  false,
+	}
 	enforcement.apply(req, m)
 
 	m.AssertNotCalled(t, "ScrubDevice")
 	m.AssertNotCalled(t, "ScrubUser")
-	assert.Equal(t, device, req.Device, "Device Set Correctly")
-	assert.Equal(t, user, req.User, "User Set Correctly")
+}
+
+func TestApplyNil(t *testing.T) {
+	m := &mockScrubber{}
+
+	enforcement := Enforcement{}
+	enforcement.apply(nil, m)
+
+	m.AssertNotCalled(t, "ScrubDevice")
+	m.AssertNotCalled(t, "ScrubUser")
 }
 
 type mockScrubber struct {
 	mock.Mock
 }
 
-func (m *mockScrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
-	args := m.Called(device, macAndIFA, ipv6, geo)
+func (m *mockScrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
+	args := m.Called(device, ipv6, geo)
 	return args.Get(0).(*openrtb.Device)
 }
 
-func (m *mockScrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User {
-	args := m.Called(user, strategy, geo)
+func (m *mockScrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User {
+	args := m.Called(user, demographic, geo)
 	return args.Get(0).(*openrtb.User)
 }

privacy/scrubber.go

@@ -34,24 +34,21 @@ const (
 	ScrubStrategyGeoReducedPrecision
 )
 
-// ScrubStrategyUser defines the approach to scrub PII from user data.
-type ScrubStrategyUser int
+// ScrubStrategyDemographic defines the approach to non-location demographic data.
+type ScrubStrategyDemographic int
 
 const (
-	// ScrubStrategyUserNone does not remove user data.
-	ScrubStrategyUserNone ScrubStrategyUser = iota
+	// ScrubStrategyDemographicNone does not remove non-location demographic data.
+	ScrubStrategyDemographicNone ScrubStrategyDemographic = iota
 
-	// ScrubStrategyUserFull removes the user's buyer id, exchange id year of birth, and gender.
-	ScrubStrategyUserFull
-
-	// ScrubStrategyUserBuyerIDOnly removes the user's buyer id.
-	ScrubStrategyUserBuyerIDOnly
+	// ScrubStrategyDemographicAgeAndGender removes age and gender data.
+	ScrubStrategyDemographicAgeAndGender
 )
 
 // Scrubber removes PII from parts of an OpenRTB request.
 type Scrubber interface {
-	ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device
-	ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User
+	ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device
+	ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User
 }
 
 type scrubber struct{}
@@ -61,25 +58,21 @@ func NewScrubber() Scrubber {
 	return scrubber{}
 }
 
-func (scrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
+func (scrubber) ScrubDevice(device *openrtb.Device, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb.Device {
 	if device == nil {
 		return nil
 	}
 
 	deviceCopy := *device
-
 	deviceCopy.DIDMD5 = ""
 	deviceCopy.DIDSHA1 = ""
 	deviceCopy.DPIDMD5 = ""
 	deviceCopy.DPIDSHA1 = ""
+	deviceCopy.IFA = ""
+	deviceCopy.MACMD5 = ""
+	deviceCopy.MACSHA1 = ""
 	deviceCopy.IP = scrubIPV4(device.IP)
 
-	if macAndIFA {
-		deviceCopy.MACSHA1 = ""
-		deviceCopy.MACMD5 = ""
-		deviceCopy.IFA = ""
-	}
-
 	switch ipv6 {
 	case ScrubStrategyIPV6Lowest16:
 		deviceCopy.IPv6 = scrubIPV6Lowest16Bits(device.IPv6)
@@ -97,21 +90,19 @@ func (scrubber) ScrubDevice(device *openrtb.Device, macAndIFA bool, ipv6 ScrubSt
 	return &deviceCopy
 }
 
-func (scrubber) ScrubUser(user *openrtb.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb.User {
+func (scrubber) ScrubUser(user *openrtb.User, demographic ScrubStrategyDemographic, geo ScrubStrategyGeo) *openrtb.User {
 	if user == nil {
 		return nil
 	}
 
 	userCopy := *user
+	userCopy.BuyerUID = ""
+	userCopy.ID = ""
 
-	switch strategy {
-	case ScrubStrategyUserFull:
-		userCopy.BuyerUID = ""
-		userCopy.ID = ""
+	switch demographic {
+	case ScrubStrategyDemographicAgeAndGender:
 		userCopy.Yob = 0
 		userCopy.Gender = ""
-	case ScrubStrategyUserBuyerIDOnly:
-		userCopy.BuyerUID = ""
 	}
 
 	switch geo {
@@ -169,13 +160,7 @@ func scrubGeoFull(geo *openrtb.Geo) *openrtb.Geo {
 		return nil
 	}
 
-	geoCopy := *geo
-	geoCopy.Lat = 0
-	geoCopy.Lon = 0
-	geoCopy.Metro = ""
-	geoCopy.City = ""
-	geoCopy.ZIP = ""
-	return &geoCopy
+	return &openrtb.Geo{}
 }
 
 func scrubGeoPrecision(geo *openrtb.Geo) *openrtb.Geo {