[Backport 7.57.x] [APM] Normalize X-Datadog-Container-Tags header

pkg/trace/api/api.go

@@ -853,3 +853,12 @@ func getMediaType(req *http.Request) string {
 	}
 	return mt
 }
+
+// normalizeHTTPHeader takes a raw string and normalizes the value to be compatible
+// with an HTTP header value.
+func normalizeHTTPHeader(val string) string {
+	val = strings.ReplaceAll(val, "\r\n", "_")
+	val = strings.ReplaceAll(val, "\r", "_")
+	val = strings.ReplaceAll(val, "\n", "_")
+	return val
+}

pkg/trace/api/api_test.go

@@ -1057,6 +1057,61 @@ func TestExpvar(t *testing.T) {
 	})
 }
 
+func TestNormalizeHTTPHeader(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{
+			input:    "Header: value\nAnother header: value",
+			expected: "Header: value_Another header: value",
+		},
+		{
+			input:    "Header: value\rAnother header: value",
+			expected: "Header: value_Another header: value",
+		},
+		{
+			input:    "Header: value\r\nAnother header: value",
+			expected: "Header: value_Another header: value",
+		},
+		{
+			input:    "SingleLineHeader: value",
+			expected: "SingleLineHeader: value",
+		},
+		{
+			input:    "\rLeading carriage return",
+			expected: "_Leading carriage return",
+		},
+		{
+			input:    "\nLeading line break",
+			expected: "_Leading line break",
+		},
+		{
+			input:    "Trailing carriage return\r",
+			expected: "Trailing carriage return_",
+		},
+		{
+			input:    "Trailing line break\n",
+			expected: "Trailing line break_",
+		},
+		{
+			input:    "Multiple\r\nline\r\nbreaks",
+			expected: "Multiple_line_breaks",
+		},
+		{
+			input:    "",
+			expected: "",
+		},
+	}
+
+	for _, test := range tests {
+		result := normalizeHTTPHeader(test.input)
+		if result != test.expected {
+			t.Errorf("normalizeHTTPHeader(%q) = %q; expected %q", test.input, result, test.expected)
+		}
+	}
+}
+
 func msgpTraces(t *testing.T, traces pb.Traces) []byte {
 	bts, err := traces.MarshalMsg(nil)
 	if err != nil {

pkg/trace/api/evp_proxy.go

@@ -164,8 +164,9 @@ func (t *evpProxyTransport) RoundTrip(req *http.Request) (rresp *http.Response,
 	if containerID != "" {
 		req.Header.Set(header.ContainerID, containerID)
 		if ctags := getContainerTags(t.conf.ContainerTags, containerID); ctags != "" {
-			req.Header.Set("X-Datadog-Container-Tags", ctags)
-			log.Debugf("Setting header X-Datadog-Container-Tags=%s for evp proxy", ctags)
+			ctagsHeader := normalizeHTTPHeader(ctags)
+			req.Header.Set("X-Datadog-Container-Tags", ctagsHeader)
+			log.Debugf("Setting header X-Datadog-Container-Tags=%s for evp proxy", ctagsHeader)
 		}
 	}
 	req.Header.Set("X-Datadog-Hostname", t.conf.Hostname)

pkg/trace/api/evp_proxy_test.go

@@ -161,6 +161,27 @@ func TestEVPProxyForwarder(t *testing.T) {
 		assert.Equal(t, "", logs)
 	})
 
+	t.Run("normalizedContainerTags", func(t *testing.T) {
+		conf := newTestReceiverConfig()
+		conf.Site = "us3.datadoghq.com"
+		conf.Endpoints[0].APIKey = "test_api_key"
+		conf.ContainerTags = func(cid string) ([]string, error) {
+			return []string{"container:" + cid, "key:\nval"}, nil
+		}
+
+		req := httptest.NewRequest("POST", "/mypath/mysubpath?arg=test", bytes.NewReader(randBodyBuf))
+		req.Header.Set("X-Datadog-EVP-Subdomain", "my.subdomain")
+		req.Header.Set(header.ContainerID, "myid")
+		proxyreqs, resp, logs := sendRequestThroughForwarderWithMockRoundTripper(conf, req, stats)
+
+		resp.Body.Close()
+		require.Equal(t, http.StatusOK, resp.StatusCode, "Got: ", fmt.Sprint(resp.StatusCode))
+		require.Len(t, proxyreqs, 1)
+		assert.Equal(t, "container:myid,key:_val", proxyreqs[0].Header.Get("X-Datadog-Container-Tags"))
+		assert.Equal(t, "myid", proxyreqs[0].Header.Get(header.ContainerID))
+		assert.Equal(t, "", logs)
+	})
+
 	t.Run("dual-shipping", func(t *testing.T) {
 		conf := newTestReceiverConfig()
 		conf.Site = "us3.datadoghq.com"

pkg/trace/api/pipeline_stats.go

@@ -82,8 +82,9 @@ func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []
 		}
 		containerID := cidProvider.GetContainerID(req.Context(), req.Header)
 		if ctags := getContainerTags(conf.ContainerTags, containerID); ctags != "" {
-			req.Header.Set("X-Datadog-Container-Tags", ctags)
-			log.Debugf("Setting header X-Datadog-Container-Tags=%s for pipeline stats proxy", ctags)
+			ctagsHeader := normalizeHTTPHeader(ctags)
+			req.Header.Set("X-Datadog-Container-Tags", ctagsHeader)
+			log.Debugf("Setting header X-Datadog-Container-Tags=%s for pipeline stats proxy", ctagsHeader)
 		}
 		req.Header.Set("X-Datadog-Additional-Tags", tags)
 		log.Debugf("Setting header X-Datadog-Additional-Tags=%s for pipeline stats proxy", tags)

pkg/trace/api/profiles.go

@@ -116,8 +116,9 @@ func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string
 		}
 		containerID := cidProvider.GetContainerID(req.Context(), req.Header)
 		if ctags := getContainerTags(conf.ContainerTags, containerID); ctags != "" {
-			req.Header.Set("X-Datadog-Container-Tags", ctags)
-			log.Debugf("Setting header X-Datadog-Container-Tags=%s for profiles proxy", ctags)
+			ctagsHeader := normalizeHTTPHeader(ctags)
+			req.Header.Set("X-Datadog-Container-Tags", ctagsHeader)
+			log.Debugf("Setting header X-Datadog-Container-Tags=%s for profiles proxy", ctagsHeader)
 		}
 		req.Header.Set("X-Datadog-Additional-Tags", tags)
 		log.Debugf("Setting header X-Datadog-Additional-Tags=%s for profiles proxy", tags)