Remove the parameters from the path.

This should solve #49.

protos/http/http.go

@@ -90,6 +90,7 @@ type HttpTransaction struct {
 	Method       string
 	RequestUri   string
 	Params       string
+	Path         string
 
 	Http common.MapStr
 
@@ -700,7 +701,7 @@ func (http *Http) receivedHttpRequest(msg *HttpMessage) {
 	trans.Real_ip = msg.Real_ip
 
 	var err error
-	trans.Params, err = http.paramsHideSecrets(msg, msg.Raw)
+	trans.Path, trans.Params, err = http.extractParameters(msg, msg.Raw)
 	if err != nil {
 		logp.Warn("http", "Fail to parse HTTP parameters: %v", err)
 	}
@@ -805,8 +806,8 @@ func (http *Http) PublishTransaction(t *HttpTransaction) {
 		event["real_ip"] = t.Real_ip
 	}
 	event["method"] = t.Method
-	event["path"] = t.RequestUri
-	event["query"] = fmt.Sprintf("%s %s", t.Method, t.RequestUri)
+	event["path"] = t.Path
+	event["query"] = fmt.Sprintf("%s %s", t.Method, t.Path)
 	event["params"] = t.Params
 
 	event["timestamp"] = common.Time(t.ts)
@@ -912,34 +913,36 @@ func (http *Http) hideSecrets(values url.Values) url.Values {
 	return params
 }
 
-// paramsHideSecrets parses the URL and the form parameters and replaces the secrets
+// extractParameters parses the URL and the form parameters and replaces the secrets
 // with the string xxxxx. The parameters containing secrets are defined in http.Hide_secrets.
-func (http *Http) paramsHideSecrets(m *HttpMessage, msg []byte) (string, error) {
+// Returns the Request URI path and the (ajdusted) parameters.
+func (http *Http) extractParameters(m *HttpMessage, msg []byte) (path string, params string, err error) {
 	var values url.Values
-	var err error
 
 	u, err := url.Parse(m.RequestUri)
 	if err != nil {
-		return "", err
+		return
 	}
 	values = u.Query()
+	path = u.Path
 
-	params := http.hideSecrets(values)
+	paramsMap := http.hideSecrets(values)
 
 	if m.ContentLength > 0 && strings.Contains(m.ContentType, "urlencoded") {
 		values, err = url.ParseQuery(string(msg[m.bodyOffset:]))
 		if err != nil {
-			return "", err
+			return
 		}
 
 		for key, value := range http.hideSecrets(values) {
-			params[key] = value
+			paramsMap[key] = value
 		}
 	}
+	params = paramsMap.Encode()
 
-	logp.Debug("httpdetailed", "Parameters: %s", params.Encode())
+	logp.Debug("httpdetailed", "Parameters: %s", params)
 
-	return params.Encode(), nil
+	return
 }
 
 func (http *Http) isSecretParameter(key string) bool {

protos/http/http_test.go

@@ -887,11 +887,15 @@ func TestHttpParser_censorPasswordURL(t *testing.T) {
 	}
 
 	msg := stream.data[stream.message.start:stream.message.end]
-	params, err := http.paramsHideSecrets(stream.message, msg)
+	path, params, err := http.extractParameters(stream.message, msg)
 	if err != nil {
 		t.Errorf("Fail to parse parameters")
 	}
 
+	if path != "/test" {
+		t.Errorf("Wrong path: %s", path)
+	}
+
 	if strings.Contains(params, "secret") {
 		t.Errorf("Failed to censor the password: %s", params)
 	}
@@ -929,11 +933,15 @@ func TestHttpParser_censorPasswordPOST(t *testing.T) {
 	}
 
 	msg := stream.data[stream.message.start:stream.message.end]
-	params, err := http.paramsHideSecrets(stream.message, msg)
+	path, params, err := http.extractParameters(stream.message, msg)
 	if err != nil {
 		t.Errorf("Fail to parse parameters")
 	}
 
+	if path != "/users/login" {
+		t.Errorf("Wrong path: %s", path)
+	}
+
 	if strings.Contains(params, "secret") {
 		t.Errorf("Failed to censor the password: %s", msg)
 	}
@@ -972,12 +980,16 @@ func TestHttpParser_censorPasswordGET(t *testing.T) {
 	}
 
 	msg := stream.data[stream.message.start:stream.message.end]
-	params, err := http.paramsHideSecrets(stream.message, msg)
+	path, params, err := http.extractParameters(stream.message, msg)
 	if err != nil {
 		t.Errorf("Faile to parse parameters")
 	}
 	logp.Debug("httpdetailed", "parameters %s", params)
 
+	if path != "/users/login" {
+		t.Errorf("Wrong path: %s", path)
+	}
+
 	if strings.Contains(params, "secret") {
 		t.Errorf("Failed to censor the password: %s", msg)
 	}

tests/test_0019_hide_params.py

@@ -23,6 +23,10 @@ def test_http_hide_post(self):
         o = objs[0]
         assert o["type"] == "http"
         assert o["params"] == "pass=xxxxx&user=monica"
+        assert o["path"] == "/login"
+        for _, val in o.items():
+            if isinstance(val, basestring):
+                assert "secret" not in val
 
     def test_http_hide_get(self):
         """
@@ -40,11 +44,14 @@ def test_http_hide_get(self):
         o = objs[0]
         assert o["type"] == "http"
         assert o["params"] == "pass=xxxxx&user=monica"
+        assert o["path"] == "/login"
+        for _, val in o.items():
+            if isinstance(val, basestring):
+                assert "secret" not in val
 
     def test_http_hide_post_default(self):
         """
-        Should be able to strip the password from
-        a POST request.
+        By default nothing is stripped.
         """
         self.render_config_template()
         self.run_packetbeat(pcap="hide_secret_POST.pcap",
@@ -55,3 +62,4 @@ def test_http_hide_post_default(self):
         o = objs[0]
         assert o["type"] == "http"
         assert o["params"] == "pass=secret&user=monica"
+        assert o["path"] == "/login"

tests/test_0024_http_query.py

@@ -33,5 +33,5 @@ def test_http_get(self):
         assert len(objs) == 1
         o = objs[0]
         assert o["type"] == "http"
-        assert o["query"] == "GET /dashboard/transactions?" + \
-                             "input=packetbeat&src_ip=192.35.243.1"
+        assert o["query"] == "GET /dashboard/transactions"
+        assert o["params"] == "input=packetbeat&src_ip=192.35.243.1"