From b49a17d8bf08a6f245e36163623cb57493bf1f45 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 12:44:09 -0400 Subject: [PATCH 1/9] Moved varable include_body to Http.Include_body_for Moved Initialization for Http.Include_body_for to Initialization function Added Varable checking before setting values for config varables Added Debugging to Initialization function logging Debug as http Changed Debug Level of shouldInclude to httpdetailed from http --- protos/http/http.go | 56 ++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/protos/http/http.go b/protos/http/http.go index ce8798d99ce..849dd4e966c 100644 --- a/protos/http/http.go +++ b/protos/http/http.go @@ -110,6 +110,7 @@ type Http struct { Real_ip_header string Hide_keywords []string Strip_authorization bool + Include_body_for []string transactionsMap map[common.HashableTcpTuple]*HttpTransaction @@ -119,35 +120,59 @@ type Http struct { func (http *Http) InitDefaults() { http.Send_request = false http.Send_response = false +// http.Include_body_for = make(map[string]bool) +// http.Include_body_for["all"] = false; } func (http *Http) SetFromConfig(config *config.Config, meta *toml.MetaData) (err error) { if meta.IsDefined("protocols", "http", "send_request") { http.Send_request = config.Protocols["http"].Send_request + logp.Debug("http", "ConfigSetting: protocol.http.Send_request Value: '%t'\n", http.Send_request) } if meta.IsDefined("protocols", "http", "send_response") { http.Send_response = config.Protocols["http"].Send_response + logp.Debug("http", "ConfigSetting: protocol.http.Send_response Value: '%t'\n", http.Send_response) } - http.Hide_keywords = config.Passwords.Hide_keywords - http.Strip_authorization = config.Passwords.Strip_authorization - - if config.Http.Send_all_headers { + if meta.IsDefined("http", "Include_body_for") { + http.Include_body_for = config.Http.Include_body_for + logp.Debug("http", "ConfigSetting: http.Include_body_for \n") + for _, include := range http.Include_body_for { + logp.Debug("http", "Value: '%s'\n", include) + } + } + if meta.IsDefined("http", "Hide_keywords") { + http.Hide_keywords = config.Passwords.Hide_keywords + logp.Debug("http", "ConfigSetting: http.Hide_keywords Value: '%t'\n", http.Hide_keywords) + } + if meta.IsDefined("Passwords", "Strip_authorization") { + http.Strip_authorization = config.Passwords.Strip_authorization + logp.Debug("http", "ConfigSetting: Password.Strip_authorization Value: '%t'\n", http.Strip_authorization) + } + if meta.IsDefined("Http", "Send_all_headers") { http.Send_headers = true http.Send_all_headers = true - } else { - if len(config.Http.Send_headers) > 0 { - http.Send_headers = true + http.Strip_authorization = config.Passwords.Strip_authorization + logp.Debug("http", "ConfigSetting: Http.Send_all_headers Value: '%t'\n", http.Send_all_headers) + logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers) + } + if meta.IsDefined("Http", "Send_headers") { + http.Send_headers = true + logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers) http.Headers_whitelist = map[string]bool{} for _, hdr := range config.Http.Send_headers { http.Headers_whitelist[strings.ToLower(hdr)] = true + logp.Debug("http", "ConfigSetting: Http.Headers_whitelist Value: '%s'\n", hdr) } - } } - - http.Split_cookie = config.Http.Split_cookie - - http.Real_ip_header = strings.ToLower(config.Http.Real_ip_header) + if meta.IsDefined("Http", "Split_cookie") { + http.Split_cookie = config.Http.Split_cookie + logp.Debug("http", "ConfigSetting: Split_cookie Value: '%t'\n", http.Split_cookie) + } + if meta.IsDefined("Http", "Real_ip_header") { + http.Real_ip_header = config.Http.Real_ip_header + logp.Debug("http", "ConfigSetting: Real_ip_header: '%s'\n", http.Real_ip_header) + } return nil } @@ -833,13 +858,12 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { } func (http *Http) shouldIncludeInBody(contenttype string) bool { - include_body := config.ConfigSingleton.Http.Include_body_for - for _, include := range include_body { + for _, include := range http.Include_body_for { if strings.Contains(contenttype, include) { - logp.Debug("http", "Should Include Body = true Content-Type "+contenttype+" include_body "+include) + logp.Debug("httpdetailed", "Should Include Body = true Content-Type "+contenttype+" http.Include_body_for "+include) return true } - logp.Debug("http", "Should Include Body = false Content-Type"+contenttype+" include_body "+include) + logp.Debug("httpdetailed", "Should Include Body = false Content-Type"+contenttype+" http.Include_body_for "+include) } return false } From 8fad855442662562257132ae1e1a2e809250a54f Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 12:49:53 -0400 Subject: [PATCH 2/9] Updated default conf to include new headers --- packetbeat.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packetbeat.conf b/packetbeat.conf index 0d35056a6bb..47d744b4630 100644 --- a/packetbeat.conf +++ b/packetbeat.conf @@ -52,6 +52,8 @@ device = "any" # configuration. [protocols.http] ports = [80, 8080, 8000, 5000, 8002] + #send_request=false + #send_response=false [protocols.mysql] ports = [3306] @@ -104,4 +106,10 @@ device = "any" # sensitive information. #hide_keywords = ["pass", "password", "passwd"] +[http] +#Include_body_for=["text/html"] +#Send_all_headers=true +#Strip_authorization=false +#Split_cookie=true + # vim: set ft=toml: From cacf155151e3261c49bca6b7b1445a417be90eb7 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 12:55:47 -0400 Subject: [PATCH 3/9] Added Comments to packetbeat.conf for new fields --- packetbeat.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packetbeat.conf b/packetbeat.conf index 47d744b4630..c19ef819ac3 100644 --- a/packetbeat.conf +++ b/packetbeat.conf @@ -52,7 +52,9 @@ device = "any" # configuration. [protocols.http] ports = [80, 8080, 8000, 5000, 8002] + ## Include the actual request: For Http this will only include the Headers of the request- See the Http section for finer controler #send_request=false + ## Include the acutal response: For Http this will only include the Heards of the response - See the Http Sections for finer controle #send_response=false [protocols.mysql] @@ -107,9 +109,17 @@ device = "any" #hide_keywords = ["pass", "password", "passwd"] [http] +###### +#Only include the HTTP body if the content-type matches #Include_body_for=["text/html"] +###### +#Send The Http Headers as a specific field #Send_all_headers=true +###### +#Remove the HTTP Authentication header #Strip_authorization=false +###### +#Split All cookies to their own field #Split_cookie=true # vim: set ft=toml: From 82430550b8478eef1f1c7af695857b422375c52c Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 13:03:28 -0400 Subject: [PATCH 4/9] Added basic wildcard for Include_body_for [] --- protos/http/http.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/protos/http/http.go b/protos/http/http.go index 849dd4e966c..260002fc725 100644 --- a/protos/http/http.go +++ b/protos/http/http.go @@ -110,7 +110,7 @@ type Http struct { Real_ip_header string Hide_keywords []string Strip_authorization bool - Include_body_for []string + Include_body_for []string transactionsMap map[common.HashableTcpTuple]*HttpTransaction @@ -120,6 +120,7 @@ type Http struct { func (http *Http) InitDefaults() { http.Send_request = false http.Send_response = false + http.Include_body_for= nil // http.Include_body_for = make(map[string]bool) // http.Include_body_for["all"] = false; } @@ -136,6 +137,7 @@ func (http *Http) SetFromConfig(config *config.Config, meta *toml.MetaData) (err if meta.IsDefined("http", "Include_body_for") { http.Include_body_for = config.Http.Include_body_for logp.Debug("http", "ConfigSetting: http.Include_body_for \n") + logp.Debug("http", "ConfigSetting: http.Include_body_for Length =%d \n",len(http.Include_body_for)) for _, include := range http.Include_body_for { logp.Debug("http", "Value: '%s'\n", include) } @@ -858,12 +860,17 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { } func (http *Http) shouldIncludeInBody(contenttype string) bool { - for _, include := range http.Include_body_for { - if strings.Contains(contenttype, include) { - logp.Debug("httpdetailed", "Should Include Body = true Content-Type "+contenttype+" http.Include_body_for "+include) - return true + if http.Include_body_for != nil { + if len(http.Include_body_for) == 0{ + return true; + } + for _, include := range http.Include_body_for { + if strings.Contains(contenttype, include) { + logp.Debug("httpdetailed", "Should Include Body = true Content-Type "+contenttype+" http.Include_body_for "+include) + return true + } + logp.Debug("httpdetailed", "Should Include Body = false Content-Type"+contenttype+" http.Include_body_for "+include) } - logp.Debug("httpdetailed", "Should Include Body = false Content-Type"+contenttype+" http.Include_body_for "+include) } return false } From 9558f03a90ff0b5286b27e1c9c7da5a22c8177e5 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 13:04:18 -0400 Subject: [PATCH 5/9] Documented how to catch all Content-Types --- packetbeat.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/packetbeat.conf b/packetbeat.conf index c19ef819ac3..f124ad50701 100644 --- a/packetbeat.conf +++ b/packetbeat.conf @@ -111,6 +111,7 @@ device = "any" [http] ###### #Only include the HTTP body if the content-type matches +# An empty array [] will capture everything #Include_body_for=["text/html"] ###### #Send The Http Headers as a specific field From 563d7b889ab4bd4f0797c08417de3af9be3ba6d8 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 13:47:05 -0400 Subject: [PATCH 6/9] Added Exclude_body_for varable to the Http Structure --- config/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/config.go b/config/config.go index 920418425f9..7537f51ad3c 100644 --- a/config/config.go +++ b/config/config.go @@ -71,6 +71,8 @@ type Http struct { Split_cookie bool Real_ip_header string Include_body_for []string + Exclude_body_for []string + } type Mysql struct { From 4a469ad97298a8da7e14b70dd6a1cad1ae2c8696 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 13:47:46 -0400 Subject: [PATCH 7/9] Added Comments for Exclude_body_for --- packetbeat.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packetbeat.conf b/packetbeat.conf index f124ad50701..ff8b1811e05 100644 --- a/packetbeat.conf +++ b/packetbeat.conf @@ -111,8 +111,11 @@ device = "any" [http] ###### #Only include the HTTP body if the content-type matches -# An empty array [] will capture everything -#Include_body_for=["text/html"] +# An empty array [] will capture everything ***Warning this will include images as well +#Include_body_for=["text/"] +###### +#Exclude the HTTP body if the content-type matches (Overrides Includes_body_for) +#Exclude_body_for=["image/"] ###### #Send The Http Headers as a specific field #Send_all_headers=true From 175685def3169d075d28d549e054f10d8f63b12a Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 13:48:15 -0400 Subject: [PATCH 8/9] Added Exclude_body_for varable and functionality --- protos/http/http.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/protos/http/http.go b/protos/http/http.go index 260002fc725..ff098316cb5 100644 --- a/protos/http/http.go +++ b/protos/http/http.go @@ -111,7 +111,7 @@ type Http struct { Hide_keywords []string Strip_authorization bool Include_body_for []string - + Exclude_body_for []string transactionsMap map[common.HashableTcpTuple]*HttpTransaction results chan common.MapStr @@ -121,6 +121,7 @@ func (http *Http) InitDefaults() { http.Send_request = false http.Send_response = false http.Include_body_for= nil + http.Exclude_body_for= nil // http.Include_body_for = make(map[string]bool) // http.Include_body_for["all"] = false; } @@ -142,6 +143,14 @@ func (http *Http) SetFromConfig(config *config.Config, meta *toml.MetaData) (err logp.Debug("http", "Value: '%s'\n", include) } } + if meta.IsDefined("http", "Exclude_body_for") { + http.Exclude_body_for = config.Http.Exclude_body_for + logp.Debug("http", "ConfigSetting: http.Exclude_body_for \n") + logp.Debug("http", "ConfigSetting: http.Exclude_body_for Length =%d \n",len(http.Exclude_body_for)) + for _, include := range http.Exclude_body_for { + logp.Debug("http", "Value: '%s'\n", include) + } + } if meta.IsDefined("http", "Hide_keywords") { http.Hide_keywords = config.Passwords.Hide_keywords logp.Debug("http", "ConfigSetting: http.Hide_keywords Value: '%t'\n", http.Hide_keywords) @@ -851,7 +860,7 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { if len(m.chunked_body) > 0 { raw_msg_cut = append(raw_msg_cut, m.chunked_body...) } else { - logp.Debug("http", "Body to include: [%s]", m.Raw[m.bodyOffset:]) + logp.Debug("httpdetailed", "Body to include: [%s]", m.Raw[m.bodyOffset:]) raw_msg_cut = append(raw_msg_cut, m.Raw[m.bodyOffset:]...) } } @@ -861,9 +870,15 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { func (http *Http) shouldIncludeInBody(contenttype string) bool { if http.Include_body_for != nil { - if len(http.Include_body_for) == 0{ + if ( len(http.Include_body_for) == 0 && ( http.Exclude_body_for == nil || len(http.Exclude_body_for) == 0)) { return true; } + for _, exclude := range http.Exclude_body_for { + if strings.Contains(contenttype, exclude) { + logp.Debug("httpdetailed", "Should Exclude Body = false Content-Type"+contenttype+" http.Exclude_body_for "+exclude) + return false + } + } for _, include := range http.Include_body_for { if strings.Contains(contenttype, include) { logp.Debug("httpdetailed", "Should Include Body = true Content-Type "+contenttype+" http.Include_body_for "+include) From 99f375cd3758859a7371ff1b2889bf3757593c26 Mon Sep 17 00:00:00 2001 From: Ed Perry Date: Sun, 17 May 2015 15:47:26 -0400 Subject: [PATCH 9/9] Removed debugging lines I removed a code section at ~line 403 to 406 which should not have been. I restored those lines --- protos/http/http.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/protos/http/http.go b/protos/http/http.go index ff098316cb5..97cc538a677 100644 --- a/protos/http/http.go +++ b/protos/http/http.go @@ -205,7 +205,6 @@ func (http *Http) Init(test_mode bool, results chan common.MapStr) error { } http.transactionsMap = make(map[common.HashableTcpTuple]*HttpTransaction, TransactionsHashSize) - logp.Debug("http", "transactionsMap: %p http: %p", http.transactionsMap, &http) http.results = results @@ -401,12 +400,12 @@ func (http *Http) messageParser(s *HttpStream) (bool, bool) { s.parseState = BODY_CHUNKED_START continue } - if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) { - logp.Debug("http", "Empty content length, ignore body") - // Ignore body for request that contains a message body but not a Content-Length - m.end = s.parseOffset - return true, true - } + if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) { + logp.Debug("http", "Empty content length, ignore body") + // Ignore body for request that contains a message body but not a Content-Length + m.end = s.parseOffset + return true, true + } logp.Debug("http", "Read body") s.parseState = BODY } else { @@ -693,6 +692,7 @@ func (http *Http) receivedHttpRequest(msg *HttpMessage) { // save Raw message if http.Send_request { + logp.Debug("http", "HTTP Send_resquest") trans.Request_raw = string(http.cutMessageBody(msg)) } @@ -747,7 +747,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) { trans := http.transactionsMap[tuple.Hashable()] if trans == nil { - logp.Warn("Response from unknown transaction. Ignoring: %v", tuple) + logp.Warn("Response from unknown transaction. Ignoring: %v ", tuple) return } @@ -785,6 +785,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) { // save Raw message if http.Send_response { + logp.Debug("http", "HTTP Send_response") trans.Response_raw = string(http.cutMessageBody(msg)) } @@ -855,7 +856,8 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { // add headers always raw_msg_cut = m.Raw[:m.bodyOffset] - // add body + // add body + if len(m.ContentType) == 0 || http.shouldIncludeInBody(m.ContentType) { if len(m.chunked_body) > 0 { raw_msg_cut = append(raw_msg_cut, m.chunked_body...) @@ -869,24 +871,24 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte { } func (http *Http) shouldIncludeInBody(contenttype string) bool { - if http.Include_body_for != nil { - if ( len(http.Include_body_for) == 0 && ( http.Exclude_body_for == nil || len(http.Exclude_body_for) == 0)) { - return true; - } + logp.Debug("http", "In function Should Include Body %s", contenttype) for _, exclude := range http.Exclude_body_for { if strings.Contains(contenttype, exclude) { - logp.Debug("httpdetailed", "Should Exclude Body = false Content-Type"+contenttype+" http.Exclude_body_for "+exclude) + logp.Debug("http", "Should Exclude Body Content-Type "+contenttype+" http.Exclude_body_for "+exclude) return false } } + if ( len(http.Include_body_for) == 0 ) { + logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for All ContentTypes") + return true; + } for _, include := range http.Include_body_for { if strings.Contains(contenttype, include) { - logp.Debug("httpdetailed", "Should Include Body = true Content-Type "+contenttype+" http.Include_body_for "+include) + logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for "+include) return true } - logp.Debug("httpdetailed", "Should Include Body = false Content-Type"+contenttype+" http.Include_body_for "+include) } - } + logp.Debug("http", "Excluding Body Content-Type "+contenttype) return false }