allow empty replace values for replace stage (#2432)

grafana · Jul 28, 2020 · 8301745 · 8301745
1 parent 3f7bc33
commit 8301745
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 12 deletions.
diff --git a/docs/sources/clients/promtail/stages/replace.md b/docs/sources/clients/promtail/stages/replace.md
@@ -20,7 +20,8 @@ replace:
   [source: <string>]
 
   # Value to which the captured group will be replaced. The captured group or the named captured group will be
-  # replaced with this value and the log line will be replaced with new replaced values
+  # replaced with this value and the log line will be replaced with new replaced values. An empty value will
+  # remove the captured group from the log line.
   [replace: <string>]
 ```
 
@@ -220,3 +221,26 @@ The log line would become
 ```
 11.11.11.11 - FRANK [25/JAN/2000:14:00:01 -0500] "GET /1986.JS HTTP/1.1" 200 932 "-" "MOZILLA/5.0 (WINDOWS; U; WINDOWS NT 5.1; DE; RV:1.9.1.7) GECKO/20091221 FIREFOX/3.5.7 GTB6"
 ```
+
+### With empty `replace`
+
+Given the pipeline:
+
+```yaml
+- replace:
+    expression: "11.11.11.11 - (\\S+\\s)"
+    replace: ""
+```
+
+And the log line:
+
+```
+11.11.11.11 - frank [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"
+```
+
+The log line becomes
+
+```
+11.11.11.11 - [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"
+```
+
diff --git a/pkg/logentry/stages/replace.go b/pkg/logentry/stages/replace.go
@@ -41,10 +41,6 @@ func validateReplaceConfig(c *ReplaceConfig) (*regexp.Regexp, error) {
 		return nil, errors.New(ErrEmptyReplaceStageSource)
 	}
 
-	if c.Replace == "" {
-		return nil, errors.New(ErrEmptyReplaceStageConfig)
-	}
-
 	expr, err := regexp.Compile(c.Expression)
 	if err != nil {
 		return nil, errors.Wrap(err, ErrCouldNotCompileRegex)

diff --git a/pkg/logentry/stages/replace_test.go b/pkg/logentry/stages/replace_test.go
@@ -49,6 +49,15 @@ pipeline_stages:
       replace: '{{ if eq .Value "200" }}{{ Replace .Value "200" "HttpStatusOk" -1 }}{{ else }}{{ .Value | ToUpper }}{{ end }}'
 `
 
+var testReplaceYamlWithEmptyReplace = `
+---
+pipeline_stages:
+  -
+    replace:
+      expression: "11.11.11.11 - (\\S+\\s)"
+      replace: ''
+`
+
 var testReplaceLogLine = `11.11.11.11 - frank [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"`
 var testReplaceLogJSONLine = `{"time":"2019-01-01T01:00:00.000000001Z", "level": "info", "msg": "11.11.11.11 - \"POST /loki/api/push/ HTTP/1.1\" 200 932 \"-\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6\""}`
 
@@ -99,6 +108,12 @@ func TestPipeline_Replace(t *testing.T) {
 			map[string]interface{}{},
 			`11.11.11.11 - FRANK [25/JAN/2000:14:00:01 -0500] "GET /1986.JS HTTP/1.1" HttpStatusOk 932 "-" "MOZILLA/5.0 (WINDOWS; U; WINDOWS NT 5.1; DE; RV:1.9.1.7) GECKO/20091221 FIREFOX/3.5.7 GTB6"`,
 		},
+        "successfully run a pipeline with empty replace value": {
+            testReplaceYamlWithEmptyReplace,
+            testReplaceLogLine,
+            map[string]interface{}{},
+            `11.11.11.11 - [25/Jan/2000:14:00:01 -0500] "GET /1986.js HTTP/1.1" 200 932 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7 GTB6"`,
+        },
 	}
 
 	for testName, testData := range tests {
@@ -181,13 +196,6 @@ func TestReplaceConfig_validate(t *testing.T) {
 			},
 			errors.New(ErrEmptyReplaceStageSource),
 		},
-		"empty replace": {
-			map[string]interface{}{
-				"expression": "(?P<ts>[0-9]+).*",
-				"replace":    "",
-			},
-			errors.New(ErrEmptyReplaceStageConfig),
-		},
 		"valid without source": {
 			map[string]interface{}{
 				"expression": "(?P<ts>[0-9]+).*",