fix nested captured groups indexing in replace stage (#3381)

* fixes nested captured groups indexing in replace stage * fix typo in test * use lowercase names * fix lint error * fix lint error
grafana · Feb 26, 2021 · f14ddb8 · f14ddb8
1 parent 0d70e34
commit f14ddb8
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 4 deletions.
diff --git a/pkg/logentry/stages/replace.go b/pkg/logentry/stages/replace.go
@@ -117,7 +117,6 @@ func (r *replaceStage) Process(labels model.LabelSet, extracted map[string]inter
 
 	// Get string of matched captured groups. We will use this to extract all named captured groups
 	match := r.expression.FindStringSubmatch(*input)
-
 	matchAllIndex := r.expression.FindAllStringSubmatchIndex(*input, -1)
 
 	if matchAllIndex == nil {
@@ -167,7 +166,6 @@ func (r *replaceStage) getReplacedEntry(matchAllIndex [][]int, input string, td
 	var result string
 	previousInputEndIndex := 0
 	capturedMap := make(map[string]string)
-
 	// For a simple string like `11.11.11.11 - frank 12.12.12.12 - frank`
 	// if the regex is "(\\d{2}.\\d{2}.\\d{2}.\\d{2}) - (\\S+)"
 	// FindAllStringSubmatchIndex would return [[0 19 0 11 14 19] [20 37 20 31 34 37]].
@@ -177,6 +175,9 @@ func (r *replaceStage) getReplacedEntry(matchAllIndex [][]int, input string, td
 	// 14-19 is "frank". So, we advance by 2 index to get the next match
 	for _, matchIndex := range matchAllIndex {
 		for i := 2; i < len(matchIndex); i += 2 {
+			if matchIndex[i] == -1 {
+				continue
+			}
 			capturedString := input[matchIndex[i]:matchIndex[i+1]]
 			buf := &bytes.Buffer{}
 			td["Value"] = capturedString
@@ -185,8 +186,10 @@ func (r *replaceStage) getReplacedEntry(matchAllIndex [][]int, input string, td
 				return "", nil, err
 			}
 			st := buf.String()
-			result += input[previousInputEndIndex:matchIndex[i]] + st
-			previousInputEndIndex = matchIndex[i+1]
+			if previousInputEndIndex == 0 || previousInputEndIndex < matchIndex[i] {
+				result += input[previousInputEndIndex:matchIndex[i]] + st
+				previousInputEndIndex = matchIndex[i+1]
+			}
 			capturedMap[capturedString] = st
 		}
 	}

diff --git a/pkg/logentry/stages/replace_test.go b/pkg/logentry/stages/replace_test.go
@@ -39,6 +39,15 @@ pipeline_stages:
       replace: '{{ if eq .Value "200" }}{{ Replace .Value "200" "HttpStatusOk" -1 }}{{ else }}{{ .Value | ToUpper }}{{ end }}'
 `
 
+var testReplaceYamlWithNestedCapturedGroups = `
+---
+pipeline_stages:
+  -
+    replace:
+      expression: "(?P<ip_user>^(?P<ip>\\S+) (?P<identd>\\S+) (?P<user>\\S+)) \\[(?P<timestamp>[\\w:/]+\\s[+\\-]\\d{4})\\] \"(?P<action_path>(?P<action>\\S+)\\s?(?P<path>\\S+)?)\\s?(?P<protocol>\\S+)?\" (?P<status>\\d{3}|-) (\\d+|-)\\s?\"?(?P<referer>[^\"]*)\"?\\s?\"?(?P<useragent>[^\"]*)?\"?$"
+      replace: '{{ if eq .Value "200" }}{{ Replace .Value "200" "HttpStatusOk" -1 }}{{ else }}{{ .Value | ToUpper }}{{ end }}'
+`
+
 var testReplaceYamlWithTemplate = `
 ---
 pipeline_stages:
@@ -101,6 +110,25 @@ func TestPipeline_Replace(t *testing.T) {
 			},
 			`11.11.11.11 - FRANK [25/JAN/2000:14:00:01 -0500] "GET /1986.JS HTTP/1.1" HttpStatusOk 932 "-" "MOZILLA/5.0 (WINDOWS; U; WINDOWS NT 5.1; DE; RV:1.9.1.7) GECKO/20091221 FIREFOX/3.5.7 GTB6"`,
 		},
+		"successfully run a pipeline with 1 regex stage with nested captured groups and with template and without source": {
+			testReplaceYamlWithNestedCapturedGroups,
+			testReplaceLogLine,
+			map[string]interface{}{
+				"ip_user":     "11.11.11.11 - FRANK",
+				"action_path": "GET /1986.JS",
+				"ip":          "11.11.11.11",
+				"identd":      "-",
+				"user":        "FRANK",
+				"timestamp":   "25/JAN/2000:14:00:01 -0500",
+				"action":      "GET",
+				"path":        "/1986.JS",
+				"protocol":    "HTTP/1.1",
+				"status":      "HttpStatusOk",
+				"referer":     "-",
+				"useragent":   "MOZILLA/5.0 (WINDOWS; U; WINDOWS NT 5.1; DE; RV:1.9.1.7) GECKO/20091221 FIREFOX/3.5.7 GTB6",
+			},
+			`11.11.11.11 - FRANK [25/JAN/2000:14:00:01 -0500] "GET /1986.JS HTTP/1.1" HttpStatusOk 932 "-" "MOZILLA/5.0 (WINDOWS; U; WINDOWS NT 5.1; DE; RV:1.9.1.7) GECKO/20091221 FIREFOX/3.5.7 GTB6"`,
+		},
 		"successfully run a pipeline with 1 regex stage with template and without source": {
 			testReplaceYamlWithTemplate,
 			testReplaceLogLine,