Cleaned up Include_body_for, added additional debugging and Added Exclude_body_for option

config/config.go

@@ -71,6 +71,8 @@ type Http struct {
 	Split_cookie     bool
 	Real_ip_header   string
 	Include_body_for []string
+	Exclude_body_for []string
+
 }
 
 type Mysql struct {

packetbeat.conf

@@ -52,6 +52,10 @@ device = "any"
 # configuration.
   [protocols.http]
   ports = [80, 8080, 8000, 5000, 8002]
+  ## Include the actual request: For Http this will only include the Headers of the request- See the Http section for finer controler
+  #send_request=false
+  ## Include the acutal response: For Http this will only include the Heards of the response - See the Http Sections for finer controle
+  #send_response=false
 
   [protocols.mysql]
   ports = [3306]
@@ -104,4 +108,22 @@ device = "any"
 # sensitive information.
 #hide_keywords = ["pass", "password", "passwd"]
 
+[http]
+######
+#Only include the HTTP body if the content-type matches
+# An empty array []  will capture everything ***Warning this will include images as well 
+#Include_body_for=["text/"]
+######
+#Exclude the HTTP body if the content-type matches (Overrides Includes_body_for)
+#Exclude_body_for=["image/"]
+######
+#Send The Http Headers as a specific field
+#Send_all_headers=true
+######
+#Remove the HTTP Authentication header
+#Strip_authorization=false
+######
+#Split All cookies to their own field
+#Split_cookie=true
+
 # vim: set ft=toml:

protos/http/http.go

@@ -110,7 +110,8 @@ type Http struct {
 	Real_ip_header      string
 	Hide_keywords       []string
 	Strip_authorization bool
-
+	Include_body_for 	[]string
+	Exclude_body_for    []string
 	transactionsMap map[common.HashableTcpTuple]*HttpTransaction
 
 	results chan common.MapStr
@@ -119,35 +120,70 @@ type Http struct {
 func (http *Http) InitDefaults() {
 	http.Send_request = false
 	http.Send_response = false
+	http.Include_body_for= nil
+	http.Exclude_body_for= nil
+//	http.Include_body_for =  make(map[string]bool)
+//	http.Include_body_for["all"] = false;
 }
 
 func (http *Http) SetFromConfig(config *config.Config, meta *toml.MetaData) (err error) {
 	if meta.IsDefined("protocols", "http", "send_request") {
 		http.Send_request = config.Protocols["http"].Send_request
+		logp.Debug("http", "ConfigSetting: protocol.http.Send_request Value: '%t'\n", http.Send_request)
 	}
 	if meta.IsDefined("protocols", "http", "send_response") {
 		http.Send_response = config.Protocols["http"].Send_response
+		logp.Debug("http", "ConfigSetting: protocol.http.Send_response Value: '%t'\n", http.Send_response)
+	}
+	if meta.IsDefined("http", "Include_body_for") {
+		http.Include_body_for = config.Http.Include_body_for
+		logp.Debug("http", "ConfigSetting: http.Include_body_for \n")
+		logp.Debug("http", "ConfigSetting: http.Include_body_for Length =%d \n",len(http.Include_body_for))
+		for _, include := range http.Include_body_for {
+			logp.Debug("http", "Value: '%s'\n", include)
+		}
 	}
-	http.Hide_keywords = config.Passwords.Hide_keywords
-	http.Strip_authorization = config.Passwords.Strip_authorization
-
-	if config.Http.Send_all_headers {
+	if meta.IsDefined("http", "Exclude_body_for") {
+		http.Exclude_body_for = config.Http.Exclude_body_for
+		logp.Debug("http", "ConfigSetting: http.Exclude_body_for \n")
+		logp.Debug("http", "ConfigSetting: http.Exclude_body_for Length =%d \n",len(http.Exclude_body_for))
+		for _, include := range http.Exclude_body_for {
+			logp.Debug("http", "Value: '%s'\n", include)
+		}
+	}
+	if meta.IsDefined("http", "Hide_keywords") {	
+		http.Hide_keywords = config.Passwords.Hide_keywords
+		logp.Debug("http", "ConfigSetting: http.Hide_keywords Value: '%t'\n", http.Hide_keywords)
+	}
+	if meta.IsDefined("Passwords", "Strip_authorization") {	
+		http.Strip_authorization = config.Passwords.Strip_authorization
+		logp.Debug("http", "ConfigSetting: Password.Strip_authorization Value: '%t'\n", http.Strip_authorization)
+	}
+	if meta.IsDefined("Http", "Send_all_headers") {	
 		http.Send_headers = true
 		http.Send_all_headers = true
-	} else {
-		if len(config.Http.Send_headers) > 0 {
-			http.Send_headers = true
+		http.Strip_authorization = config.Passwords.Strip_authorization
+		logp.Debug("http", "ConfigSetting: Http.Send_all_headers Value: '%t'\n", http.Send_all_headers)
+		logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers)
 
+	} 
+	if meta.IsDefined("Http", "Send_headers") {	
+			http.Send_headers = true
+			logp.Debug("http", "ConfigSetting: Http.Send_headers Value: '%t'\n", http.Send_headers)
 			http.Headers_whitelist = map[string]bool{}
 			for _, hdr := range config.Http.Send_headers {
 				http.Headers_whitelist[strings.ToLower(hdr)] = true
+				logp.Debug("http", "ConfigSetting: Http.Headers_whitelist Value: '%s'\n", hdr)
 			}
-		}
 	}
-
-	http.Split_cookie = config.Http.Split_cookie
-
-	http.Real_ip_header = strings.ToLower(config.Http.Real_ip_header)
+	if meta.IsDefined("Http", "Split_cookie") {	
+		http.Split_cookie = config.Http.Split_cookie
+		logp.Debug("http", "ConfigSetting: Split_cookie Value: '%t'\n", http.Split_cookie)
+	}
+	if meta.IsDefined("Http", "Real_ip_header") {	
+		http.Real_ip_header = config.Http.Real_ip_header
+		logp.Debug("http", "ConfigSetting: Real_ip_header: '%s'\n", http.Real_ip_header)
+	}
 
 	return nil
 }
@@ -169,7 +205,6 @@ func (http *Http) Init(test_mode bool, results chan common.MapStr) error {
 	}
 
 	http.transactionsMap = make(map[common.HashableTcpTuple]*HttpTransaction, TransactionsHashSize)
-
 	logp.Debug("http", "transactionsMap: %p http: %p", http.transactionsMap, &http)
 
 	http.results = results
@@ -365,12 +400,12 @@ func (http *Http) messageParser(s *HttpStream) (bool, bool) {
 					s.parseState = BODY_CHUNKED_START
 					continue
 				}
-				if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) {
-					logp.Debug("http", "Empty content length, ignore body")
-					// Ignore body for request that contains a message body but not a Content-Length
-					m.end = s.parseOffset
-					return true, true
-				}
+              if m.ContentLength == 0 && (m.IsRequest || m.hasContentLength) {
+                       logp.Debug("http", "Empty content length, ignore body")
+                       // Ignore body for request that contains a message body but not a Content-Length
+                       m.end = s.parseOffset
+                       return true, true	
+               }
 				logp.Debug("http", "Read body")
 				s.parseState = BODY
 			} else {
@@ -657,6 +692,7 @@ func (http *Http) receivedHttpRequest(msg *HttpMessage) {
 
 	// save Raw message
 	if http.Send_request {
+		logp.Debug("http", "HTTP Send_resquest")
 		trans.Request_raw = string(http.cutMessageBody(msg))
 	}
 
@@ -711,7 +747,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) {
 
 	trans := http.transactionsMap[tuple.Hashable()]
 	if trans == nil {
-		logp.Warn("Response from unknown transaction. Ignoring: %v", tuple)
+		logp.Warn("Response from unknown transaction. Ignoring: %v ", tuple)
 		return
 	}
 
@@ -749,6 +785,7 @@ func (http *Http) receivedHttpResponse(msg *HttpMessage) {
 
 	// save Raw message
 	if http.Send_response {
+			logp.Debug("http", "HTTP Send_response")
 		trans.Response_raw = string(http.cutMessageBody(msg))
 	}
 
@@ -819,12 +856,13 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte {
 	// add headers always
 	raw_msg_cut = m.Raw[:m.bodyOffset]
 
-	// add body
+	// add body		
+
 	if len(m.ContentType) == 0 || http.shouldIncludeInBody(m.ContentType) {
 		if len(m.chunked_body) > 0 {
 			raw_msg_cut = append(raw_msg_cut, m.chunked_body...)
 		} else {
-			logp.Debug("http", "Body to include: [%s]", m.Raw[m.bodyOffset:])
+			logp.Debug("httpdetailed", "Body to include: [%s]", m.Raw[m.bodyOffset:])
 			raw_msg_cut = append(raw_msg_cut, m.Raw[m.bodyOffset:]...)
 		}
 	}
@@ -833,14 +871,24 @@ func (http *Http) cutMessageBody(m *HttpMessage) []byte {
 }
 
 func (http *Http) shouldIncludeInBody(contenttype string) bool {
-	include_body := config.ConfigSingleton.Http.Include_body_for
-	for _, include := range include_body {
-		if strings.Contains(contenttype, include) {
-			logp.Debug("http", "Should Include Body = true Content-Type "+contenttype+" include_body "+include)
-			return true
+		logp.Debug("http", "In function Should Include Body %s", contenttype)
+		for _, exclude := range http.Exclude_body_for {
+			if strings.Contains(contenttype, exclude) {
+				logp.Debug("http", "Should Exclude Body Content-Type "+contenttype+" http.Exclude_body_for "+exclude)
+				return false
+			}
 		}
-		logp.Debug("http", "Should Include Body = false Content-Type"+contenttype+" include_body "+include)
-	}
+		if ( len(http.Include_body_for) == 0 ) {
+				logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for All ContentTypes")
+				return true;
+		}
+		for _, include := range http.Include_body_for {
+			if strings.Contains(contenttype, include) {
+				logp.Debug("http", "Should Include Body Content-Type "+contenttype+" http.Include_body_for "+include)
+				return true
+			}
+		}
+	logp.Debug("http", "Excluding Body Content-Type "+contenttype)
 	return false
 }