pkg/trace: add support for meta_struct span proto field

.github/CODEOWNERS

@@ -209,6 +209,7 @@
 /pkg/snmp/                              @DataDog/infrastructure-integrations
 /pkg/appsec/                            @DataDog/agent-appsec
 /pkg/config/appsec.go                   @DataDog/agent-appsec
+/pkg/trace/pb/appsec.proto              @DataDog/agent-appsec
 /pkg/workloadmeta/                      @DataDog/container-integrations
 /pkg/workloadmeta/collectors/cloudfoundry @DataDog/integrations-tools-and-libraries
 

pkg/trace/agent/obfuscate.go

@@ -125,12 +125,12 @@ func newCreditCardsObfuscator(cfg config.CreditCardsConfig) *ccObfuscator {
 	cco := &ccObfuscator{luhn: cfg.Luhn}
 	if cfg.Enabled {
 		// obfuscator disabled
-		pb.SetMetaHook(cco.MetaHook)
+		pb.SetMetaHooks(cco.MetaHook, cco.MetaStructHook)
 	}
 	return cco
 }
 
-func (cco *ccObfuscator) Stop() { pb.SetMetaHook(nil) }
+func (cco *ccObfuscator) Stop() { pb.SetMetaHooks(nil, nil) }
 
 // MetaHook checks the tag with the given key and val and returns the final
 // value to be assigned to this tag.
@@ -176,3 +176,50 @@ func (cco *ccObfuscator) MetaHook(k, v string) (newval string) {
 	}
 	return v
 }
+
+// MetaStructHook checks the message inside `v` for credit card information and obfuscates it.
+func (cco *ccObfuscator) MetaStructHook(k string, v []byte) (newval []byte) {
+	if k != "appsec" {
+		// Do not obfuscate unknown structures
+		log.Debugf("Obfuscating unknown meta struct is not supported for key: %v", k)
+		return v
+	}
+	var (
+		changed      bool
+		appsecstruct pb.AppSecStruct
+	)
+	_, err := appsecstruct.UnmarshalMsg(v)
+	if err != nil {
+		// Not an appsec struct, ignore the value and log an error
+		log.Errorf("Error obfuscating appsec struct: %v", err)
+		return v
+	}
+	for _, trigger := range appsecstruct.GetTriggers() {
+		for _, rulematch := range trigger.GetRuleMatches() {
+			for _, parameter := range rulematch.GetParameters() {
+				if obfuscate.IsCardNumber(parameter.Value, cco.luhn) {
+					parameter.Value = "?"
+					changed = true
+				}
+				if parameter.Highlight == nil {
+					continue
+				}
+				for j, highlight := range parameter.Highlight {
+					if obfuscate.IsCardNumber(highlight, cco.luhn) {
+						parameter.Highlight[j] = "?"
+						changed = true
+					}
+				}
+			}
+		}
+	}
+	if changed {
+		newval, err := appsecstruct.MarshalMsg(nil)
+		if err != nil {
+			log.Errorf("Error replacing obfuscated appsec struct: %v", err)
+			return v
+		}
+		return newval
+	}
+	return v
+}

pkg/trace/agent/obfuscate_test.go

@@ -27,6 +27,8 @@ func TestNewCreditCardsObfuscator(t *testing.T) {
 	NewAgent(ctx, cfg)
 	_, ok = pb.MetaHook()
 	assert.True(t, ok)
+	_, ok = pb.MetaStructHook()
+	assert.True(t, ok)
 }
 
 func TestMetaHook(t *testing.T) {
@@ -50,6 +52,76 @@ func TestMetaHook(t *testing.T) {
 	}
 }
 
+func TestAppSecMetaStructHook(t *testing.T) {
+	cco := newCreditCardsObfuscator(config.CreditCardsConfig{Enabled: true})
+	defer cco.Stop()
+
+	t.Run("normal", func(t *testing.T) {
+		appsecstruct := pb.AppSecStruct{Triggers: []*pb.AppSecTrigger{{
+			Rule: &pb.AppSecRuleTrigger{Id: "ua-000-01", Name: "Arachni"},
+			RuleMatches: []*pb.AppSecRuleMatch{{
+				Operator:      "regex_match",
+				OperatorValue: "Arachni",
+				Parameters: []*pb.AppSecRuleParameter{
+					{
+						Address:   "http.request.headers",
+						Value:     "Arachni/v1",
+						Highlight: []string{"Arachni"},
+					},
+				},
+			}},
+		}}}
+		appsecb := []byte{}
+		appsecb, err := appsecstruct.MarshalMsg(appsecb)
+		if err != nil {
+			t.Fatalf("couldn't marshal appsec struct: %v", err)
+		}
+		assert.Equal(t, appsecb, cco.MetaStructHook("appsec", appsecb))
+	})
+
+	t.Run("creditcard", func(t *testing.T) {
+		appsecstruct := pb.AppSecStruct{Triggers: []*pb.AppSecTrigger{{
+			Rule: &pb.AppSecRuleTrigger{Id: "ua-000-01", Name: "5105-1051-0510-5100"},
+			RuleMatches: []*pb.AppSecRuleMatch{{
+				Operator:      "regex_match",
+				OperatorValue: "Arachni",
+				Parameters: []*pb.AppSecRuleParameter{
+					{
+						Address:   "http.request.headers",
+						Value:     "5105-1051-0510-5100",
+						Highlight: []string{"5105-1051-0510-5100"},
+					},
+				},
+			}},
+		}}}
+		appsecb := []byte{}
+		appsecb, err := appsecstruct.MarshalMsg(appsecb)
+		if err != nil {
+			t.Fatalf("couldn't marshal appsec struct: %v", err)
+		}
+		v := cco.MetaStructHook("appsec", appsecb)
+		_, err = appsecstruct.UnmarshalMsg(v)
+		assert.Nil(t, err)
+		// Rule name does not contain user data
+		assert.Equal(t, "5105-1051-0510-5100", appsecstruct.Triggers[0].Rule.Name)
+		// Rule match value & highlight can contain user data
+		assert.Equal(t, "?", appsecstruct.Triggers[0].RuleMatches[0].Parameters[0].Value)
+		assert.Equal(t, []string{"?"}, appsecstruct.Triggers[0].RuleMatches[0].Parameters[0].Highlight)
+	})
+
+	t.Run("unknown", func(t *testing.T) {
+		data := []byte{0x80}
+		v := cco.MetaStructHook("unknown", data)
+		assert.Equal(t, data, v)
+	})
+
+	t.Run("invalid", func(t *testing.T) {
+		data := []byte{0x80}
+		v := cco.MetaStructHook("appsec", data)
+		assert.Equal(t, data, v)
+	})
+}
+
 func TestObfuscateStatsGroup(t *testing.T) {
 	statsGroup := func(typ, resource string) *pb.ClientGroupedStats {
 		return &pb.ClientGroupedStats{

pkg/trace/api/api.go

@@ -606,9 +606,9 @@ func (r *HTTPReceiver) handleTraces(v Version, w http.ResponseWriter, req *http.
 		// a deprecated endpoint or Content-Type, or, a new decoder was implemented and the
 		// the hook was not added.
 		log.Debug("Decoded the request without running pb.MetaHook. If this is a newly implemented endpoint, please make sure to run it!")
-		if _, ok := pb.MetaHook(); ok {
+		if pb.HasMetaHooks() {
 			log.Warn("Received request on deprecated API endpoint or Content-Type. Performance is degraded. If you think this is an error, please contact support with this message.")
-			runMetaHook(tp.Chunks)
+			runMetaHooks(tp.Chunks)
 		}
 	}
 	if n, ok := r.replyOK(req, v, w); ok {
@@ -652,17 +652,25 @@ func (r *HTTPReceiver) handleTraces(v Version, w http.ResponseWriter, req *http.
 	}
 }
 
-// runMetaHook runs the pb.MetaHook on all spans from traces.
-func runMetaHook(chunks []*pb.TraceChunk) {
-	hook, ok := pb.MetaHook()
-	if !ok {
+// runMetaHooks runs pb.MetaHook and pb.MetaStructHook on all spans from traces.
+func runMetaHooks(chunks []*pb.TraceChunk) {
+	metahook, metahookOK := pb.MetaHook()
+	metastructhook, metastructhookOK := pb.MetaStructHook()
+	if !metahookOK && !metastructhookOK {
 		return
 	}
 	for _, chunk := range chunks {
 		for _, span := range chunk.Spans {
-			for k, v := range span.Meta {
-				if newv := hook(k, v); newv != v {
-					span.Meta[k] = newv
+			if metahookOK {
+				for k, v := range span.Meta {
+					if newv := metahook(k, v); newv != v {
+						span.Meta[k] = newv
+					}
+				}
+			}
+			if metastructhookOK {
+				for k, v := range span.MetaStruct {
+					span.MetaStruct[k] = metastructhook(k, v)
 				}
 			}
 		}

pkg/trace/api/api_test.go

@@ -794,7 +794,7 @@ func TestHandleTraces(t *testing.T) {
 		ts, ok := rs.Stats[info.Tags{Lang: lang, EndpointVersion: "v0.4"}]
 		assert.True(ok)
 		assert.Equal(int64(20), ts.TracesReceived)
-		assert.Equal(int64(59222), ts.TracesBytes)
+		assert.Equal(int64(61822), ts.TracesBytes)
 	}
 	// make sure we have all our languages registered
 	assert.Equal("C#|go|java|python|ruby", receiver.Languages())