diff --git a/config/config.go b/config/config.go index 920418425f9..7537f51ad3c 100644 --- a/config/config.go +++ b/config/config.go @@ -71,6 +71,8 @@ type Http struct { Split_cookie bool Real_ip_header string Include_body_for []string + Exclude_body_for []string + } type Mysql struct { diff --git a/packetbeat.conf b/packetbeat.conf index 0d35056a6bb..ff8b1811e05 100644 --- a/packetbeat.conf +++ b/packetbeat.conf @@ -52,6 +52,10 @@ device = "any" # configuration. [protocols.http] ports = [80, 8080, 8000, 5000, 8002] + ## Include the actual request: For Http this will only include the Headers of the request- See the Http section for finer controler + #send_request=false + ## Include the acutal response: For Http this will only include the Heards of the response - See the Http Sections for finer controle + #send_response=false [protocols.mysql] ports = [3306] @@ -104,4 +108,22 @@ device = "any" # sensitive information. #hide_keywords = ["pass", "password", "passwd"] +[http] +###### +#Only include the HTTP body if the content-type matches +# An empty array [] will capture everything ***Warning this will include images as well +#Include_body_for=["text/"] +###### +#Exclude the HTTP body if the content-type matches (Overrides Includes_body_for) +#Exclude_body_for=["image/"] +###### +#Send The Http Headers as a specific field +#Send_all_headers=true +###### +#Remove the HTTP Authentication header +#Strip_authorization=false +###### +#Split All cookies to their own field +#Split_cookie=true + # vim: set ft=toml: diff --git a/protos/http/http.go b/protos/http/http.go index ce8798d99ce..97cc538a677 100644 --- a/protos/http/http.go +++ b/protos/http/http.go @@ -110,7 +110,8 @@ type Http struct { Real_ip_header string Hide_keywords []string Strip_authorization bool - + Include_body_for []string + Exclude_body_for []string transactionsMap map[common.HashableTcpTuple]*HttpTransaction results chan common.MapStr @@ -119,35 +120,70 @@ type Http struct { func (http *Http) InitDefaults() { http.Send_request = false http.Send_response = false + http.Include_body_for= nil + http.Exclude_body_for= nil +// http.Include_body_for = make(map[string]bool) +// http.Include_body_for["all"] = false; } func (http *Http) SetFromConfig(config *config.Config, meta *toml.MetaData) (err error) { if meta.IsDefined("protocols", "http", "send_request") { http.Send_request = config.Protocols["http"].Send_request + logp.Debug("http", "ConfigSetting: protocol.http.Send_request Value: '%t'\n", http.Send_request) } if meta.IsDefined("protocols", "http", "send_response") { http.Send_response = config.Protocols["http"].Send_response + logp.Debug("http", "ConfigSetting: protocol.http.Send_response Value: '%t'\n", http.Send_response) + } + if meta.IsDefined("http", "Include_body_for") { + http.Include_body_for = config.Http.Include_body_for + logp.Debug("http", "ConfigSetting: http.Include_body_for \n") + logp.Debug("http", "ConfigSetting: http.Include_body_for Length =%d \n",len(http.Include_body_for)) + for _, include := range http.Include_body_for { + logp.Debug("http", "Value: '%s'\n", include) + } } - http.Hide_keywords = config.Passwords.Hide_keywords - http.Strip_authorization = config.Passwords.Strip_authorization - - if config.Http.Send_all_headers { + if meta.IsDefined("http", "Exclude_body_for") { + http.Exclude_body_for = config.Http.Exclude_body_for + logp.Debug("http", "ConfigSetting: http.Exclude_body_for \n") + logp.Debug("http", "ConfigSetting: http.Exclude_body_for Length =%d \n",len(http.Exclude_body_for)) + for _, include := range http.Exclude_body_for { + logp.Debug("http", "Value: '%s'\n", include) + } + } + if meta.IsDefined("http", "Hide_keywords") { + http.Hide_keywords = config.Passwords.Hide_keywords + logp.Debug("http", "ConfigSetting: http.Hide_keywords Value: '%t'\n", http.Hide_keywords) + } + if meta.IsDefined("Passwords", "Strip_authorization") { + http.Strip_authorization = config.Passwords.Strip_authorization + logp.Debug("http", "ConfigSetting: Password.Strip_authorization Value: '%t'\n", http.Strip_authorization) + } + if meta.IsDefined("Http", "Send_all_headers") { http.Send_headers = true http.Send_all_headers = true - } else { - if len(config.Http.Send_headers) > 0 { - http.Send_headers = true + http.Strip_authorization = config.Passwords.Strip_authorization + logp.Debug("http", "ConfigSetting: Http.Send_all_headers Value: '%t'\n", http.Send_all_headers) + logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers) + } + if meta.IsDefined("Http", "Send_headers") { + http.Send_headers = true + logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers) http.Headers_whitelist = map[string]bool{} for _, hdr := range config.Http.Send_headers { http.Headers_whitelist[strings.ToLower(hdr)] = true + logp.Debug("http", "ConfigSetting: Http.Headers_whitelist Value: '%s'\n", hdr) } - } } - - http.Split_cookie = config.Http.Split_cookie - - http.Real_ip_header = strings.ToLower(config.Http.Real_ip_header) + if meta.IsDefined("Http", "Split_cookie") { + http.Split_cookie = config.Http.Split_cookie + logp.Debug("http", "ConfigSetting: Split_cookie Value: '%t'\n", http.Split_cookie) + } + if meta.IsDefined("Http", "Real_ip_header") { + http.Real_ip_header = config.Http.Real_ip_header + logp.Debug("http", "ConfigSetting: Real_ip_header: '%s'\n", http.Real_ip_header) + } return nil } @@ -169,7 +205,6 @@ func (http *Http) Init(test_mode bool, results chan common.MapStr) error { } http.transactionsMap = make(map[common.HashableTcpTuple]*HttpTransaction, TransactionsHashSize) - logp.Debug("http", "transactionsMap: %p http: %p", http.transactionsMap, &http) http.results = results @@ -365,12 +400,12 @@ func (http *Http) messageParser(s *HttpStream) (bool, bool) { s.parseState = BODY_CHUNKED_START continue } - if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) { - logp.Debug("http", "Empty content length, ignore body") - // Ignore body for request that contains a message body but not a Content-Length - m.end = s.parseOffset - return true, true - } + if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) { + logp.Debug("http", "Empty content length, ignore body") + // Ignore body for request that contains a message body but not a Content-Length + m.end = s.parseOffset + return true, true + } logp.Debug("http", "Read body") s.parseState = BODY } else { @@ -657,6 +692,7 @@ func (http *Http) receivedHttpRequest(msg *HttpMessage) { // save Raw message if http.Send_request { + logp.Debug("http", "HTTP Send_resquest") trans.Request_raw = string(http.cutMessageBody(msg)) } @@ -711,7 +747,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) { trans := http.transactionsMap[tuple.Hashable()] if trans == nil { - logp.Warn("Response from unknown transaction. Ignoring: %v", tuple) + logp.Warn("Response from unknown transaction. Ignoring: %v ", tuple) return } @@ -749,6 +785,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) { // save Raw message if http.Send_response { + logp.Debug("http", "HTTP Send_response") trans.Response_raw = string(http.cutMessageBody(msg)) } @@ -819,12 +856,13 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { // add headers always raw_msg_cut = m.Raw[:m.bodyOffset] - // add body + // add body + if len(m.ContentType) == 0 || http.shouldIncludeInBody(m.ContentType) { if len(m.chunked_body) > 0 { raw_msg_cut = append(raw_msg_cut, m.chunked_body...) } else { - logp.Debug("http", "Body to include: [%s]", m.Raw[m.bodyOffset:]) + logp.Debug("httpdetailed", "Body to include: [%s]", m.Raw[m.bodyOffset:]) raw_msg_cut = append(raw_msg_cut, m.Raw[m.bodyOffset:]...) } } @@ -833,14 +871,24 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { } func (http *Http) shouldIncludeInBody(contenttype string) bool { - include_body := config.ConfigSingleton.Http.Include_body_for - for _, include := range include_body { - if strings.Contains(contenttype, include) { - logp.Debug("http", "Should Include Body = true Content-Type "+contenttype+" include_body "+include) - return true + logp.Debug("http", "In function Should Include Body %s", contenttype) + for _, exclude := range http.Exclude_body_for { + if strings.Contains(contenttype, exclude) { + logp.Debug("http", "Should Exclude Body Content-Type "+contenttype+" http.Exclude_body_for "+exclude) + return false + } } - logp.Debug("http", "Should Include Body = false Content-Type"+contenttype+" include_body "+include) - } + if ( len(http.Include_body_for) == 0 ) { + logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for All ContentTypes") + return true; + } + for _, include := range http.Include_body_for { + if strings.Contains(contenttype, include) { + logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for "+include) + return true + } + } + logp.Debug("http", "Excluding Body Content-Type "+contenttype) return false }