Configuration option for HTTP request header redaction.

CHANGELOG.next.asciidoc

@@ -82,6 +82,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
 - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594]
 - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503]
+- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 
 *Winlogbeat*
 

packetbeat/_meta/beat.reference.yml

@@ -200,6 +200,11 @@ packetbeat.protocols:
   # all headers by setting this option to true. The default is false.
   #send_all_headers: false
 
+  # A list of headers to redact if present in the HTTP request. This will keep
+  # the header field present, but will redact it's value to show the headers
+  # presence.
+  #redact_headers: []
+
   # The list of content types for which Packetbeat includes the full HTTP
   # payload. If the request's or response's Content-Type matches any on this
   # list, the full body will be included under the request or response field.

packetbeat/docs/packetbeat-options.asciidoc

@@ -682,6 +682,12 @@ headers are placed under the `headers` dictionary in the resulting JSON.
 Instead of sending a white list of headers to Elasticsearch, you can
 send all headers by setting this option to true. The default is false.
 
+===== `redact_headers`
+
+A list of headers to redact if present in the HTTP request. This will keep
+the header field present, but will redact it's value to show the header's
+presence.
+
 ===== `include_body_for`
 
 The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under

packetbeat/packetbeat.reference.yml

@@ -200,6 +200,11 @@ packetbeat.protocols:
   # all headers by setting this option to true. The default is false.
   #send_all_headers: false
 
+  # A list of headers to redact if present in the HTTP request. This will keep
+  # the header field present, but will redact it's value to show the headers
+  # presence.
+  #redact_headers: []
+
   # The list of content types for which Packetbeat includes the full HTTP
   # payload. If the request's or response's Content-Type matches any on this
   # list, the full body will be included under the request or response field.

packetbeat/protos/http/config.go

@@ -36,6 +36,7 @@ type httpConfig struct {
 	RedactAuthorization    bool     `config:"redact_authorization"`
 	MaxMessageSize         int      `config:"max_message_size"`
 	DecodeBody             bool     `config:"decode_body"`
+	RedactHeaders          []string `config:"redact_headers"`
 }
 
 var (

packetbeat/protos/http/http.go

@@ -88,6 +88,7 @@ type httpPlugin struct {
 	splitCookie         bool
 	hideKeywords        []string
 	redactAuthorization bool
+	redactHeaders       []string
 	maxMessageSize      int
 	mustDecodeBody      bool
 
@@ -147,6 +148,11 @@ func (http *httpPlugin) setFromConfig(config *httpConfig) {
 	http.transactionTimeout = config.TransactionTimeout
 	http.mustDecodeBody = config.DecodeBody
 
+	http.redactHeaders = make([]string, len(config.RedactHeaders))
+	for i, header := range config.RedactHeaders {
+		http.redactHeaders[i] = strings.ToLower(header)
+	}
+
 	for _, list := range [][]string{config.IncludeBodyFor, config.IncludeRequestBodyFor} {
 		http.parserConfig.includeRequestBodyFor = append(http.parserConfig.includeRequestBodyFor, list...)
 	}
@@ -725,6 +731,12 @@ func extractHostHeader(header string) (host string, port int) {
 }
 
 func (http *httpPlugin) hideHeaders(m *message) {
+	for _, header := range http.redactHeaders {
+		if _, exists := m.headers[header]; exists {
+			m.headers[header] = []byte("REDACTED")
+		}
+	}
+
 	if !m.isRequest || !http.redactAuthorization {
 		return
 	}

packetbeat/protos/http/http_test.go

@@ -990,6 +990,43 @@ func TestHttpParser_RedactAuthorization_Proxy_raw(t *testing.T) {
 	}
 }
 
+func TestHttpParser_RedactHeaders(t *testing.T) {
+	logp.TestingSetup(logp.WithSelectors("http", "httpdetailed"))
+
+	http := httpModForTests(nil)
+	http.redactAuthorization = true
+	http.parserConfig.sendHeaders = true
+	http.parserConfig.sendAllHeaders = true
+	http.redactHeaders = []string{"header-to-redact", "should-not-exist"}
+
+	data := []byte("POST /services/ObjectControl?ID=client0 HTTP/1.1\r\n" +
+		"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 2.0.50727.5472)\r\n" +
+		"Content-Type: text/xml; charset=utf-8\r\n" +
+		"SOAPAction: \"\"\r\n" +
+		"Header-To-Redact: sensitive-value\r\n" +
+		"Host: production.example.com\r\n" +
+		"Content-Length: 0\r\n" +
+		"Expect: 100-continue\r\n" +
+		"Accept-Encoding: gzip\r\n" +
+		"X-Forwarded-For: 10.216.89.132\r\n" +
+		"\r\n")
+
+	st := &stream{data: data, message: new(message)}
+
+	ok, _ := testParseStream(http, st, 0)
+
+	http.hideHeaders(st.message)
+
+	assert.True(t, ok)
+	var redactedString common.NetString = []byte("REDACTED")
+	var expectedAcceptEncoding common.NetString = []byte("gzip")
+	assert.Equal(t, redactedString, st.message.headers["header-to-redact"])
+	assert.Equal(t, expectedAcceptEncoding, st.message.headers["accept-encoding"])
+
+	_, invalidHeaderExists := st.message.headers["should-not-exist"]
+	assert.False(t, invalidHeaderExists)
+}
+
 func Test_splitCookiesHeader(t *testing.T) {
 	type io struct {
 		Input  string