Feature/3484 - set metric name with 'measurement' modifier for grok parser

docs/DATA_FORMATS_INPUT.md

@@ -663,6 +663,28 @@ For more information about the dropwizard json format see
 #### Grok
 Parse logstash-style "grok" patterns. Patterns can be added to patterns, or custom patterns read from custom_pattern_files.
 
+Modifiers can be appended to the end of a grok field to specify how that field should be handled. 
+There are also timestamp modifiers, which can be used to specify the format of time data.
+Available modifiers can be found below.
+
+The 'measurement' modifier has two seperate use cases, one for static measurement names and one for
+dynamic measurement names. 
+
+For setting a static measurement name, apply the 'measurement' modifier to the a single pattern in the 
+'patterns' field.  If grok matches the pattern, the measurement name will be changed to the specified name.
+So the config: `patterns = ["%{TEST:test_name:measurement}"]` would output a metric named "test_name" if grok
+matches the pattern. It is important to only specify one pattern per element in the patterns array field
+or an error will be thrown.
+So the config: `patterns = ["%{TEST:test_name:measurement}|%{TEST2:test2_name:measurement}"]` would need to be changed
+to: `patterns = ["%{TEST:test_name:measurement}","%{TEST2:test2_name:measurement}"]`
+
+For setting a dynamic measurement name, apply the 'measurement' modifier to a value in a custom pattern.
+If the pattern is matched, the measurement name will be set to the value of the field it was applied to.
+Each pattern should only have one 'measurement' modifier applied to it. The modifier should only apply to fields 
+of a single value type, not to another grok pattern.  The name of the field with the measurement modifier applied
+will be ignored, so these formats are the same: `custom_patterns = {"TEST %{WORD:ignored_name:measurement}"}`
+`custom_patterns = {"TEST %{WORD::measurement}"}`
+
 # View logstash grok pattern docs here:
 #   https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
 # All default logstash patterns are supported, these can be viewed here:
@@ -675,6 +697,7 @@ Parse logstash-style "grok" patterns. Patterns can be added to patterns, or cust
 #   duration (ie, 5.23ms gets converted to int nanoseconds)
 #   tag      (converts the field into a tag)
 #   drop     (drops the field completely)
+#   measurement (sets the metric name to designated field)
 # Timestamp modifiers:
 #   ts-ansic         ("Mon Jan _2 15:04:05 2006")
 #   ts-unix          ("Mon Jan _2 15:04:05 MST 2006")

plugins/inputs/file/README.md

@@ -14,7 +14,7 @@ use the [tail input plugin](/plugins/inputs/tail) instead.
   ## ** as a "super asterisk". ie:
   ##   /var/log/**.log     -> recursively find all .log files in /var/log
   ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
-  ##   /var/log/apache.log -> only tail the apache log file
+  ##   /var/log/apache.log -> only read the apache log file
   files = ["/var/log/apache/access.log"]
 
   ## Data format to consume.

plugins/inputs/file/dev/docker-compose.yml

@@ -6,7 +6,7 @@ services:
       volumes:
         - ./telegraf.conf:/telegraf.conf
         - ../../../../telegraf:/telegraf
-        - ./json_a.log:/var/log/test.log
+        - ./dev/json_a.log:/var/log/test.log
       entrypoint:
         - /telegraf
         - --config

plugins/inputs/file/file.go

@@ -11,9 +11,8 @@ import (
 )
 
 type File struct {
-	Files         []string `toml:"files"`
-	FromBeginning bool
-	parser        parsers.Parser
+	Files  []string `toml:"files"`
+	parser parsers.Parser
 
 	filenames []string
 }
@@ -24,7 +23,7 @@ const sampleConfig = `
   ## ** as a "super asterisk". ie:
   ##   /var/log/**.log     -> recursively find all .log files in /var/log
   ##   /var/log/*/*.log    -> find all .log files with a parent dir in /var/log
-  ##   /var/log/apache.log -> only tail the apache log file
+  ##   /var/log/apache.log -> only read the apache log file
   files = ["/var/log/apache/access.log"]
 
   ## The dataformat to be read from files
@@ -40,7 +39,7 @@ func (f *File) SampleConfig() string {
 }
 
 func (f *File) Description() string {
-	return "reload and gather from file[s] on telegraf's interval"
+	return "Reload and gather from file[s] on telegraf's interval."
 }
 
 func (f *File) Gather(acc telegraf.Accumulator) error {

plugins/inputs/file/file_test.go

@@ -14,26 +14,26 @@ import (
 func TestRefreshFilePaths(t *testing.T) {
 	wd, err := os.Getwd()
 	r := File{
-		Files: []string{filepath.Join(wd, "testfiles/**.log")},
+		Files: []string{filepath.Join(wd, "dev/testfiles/**.log")},
 	}
 
 	err = r.refreshFilePaths()
 	require.NoError(t, err)
-	assert.Equal(t, len(r.filenames), 2)
+	assert.Equal(t, 2, len(r.filenames))
 }
 func TestJSONParserCompile(t *testing.T) {
 	var acc testutil.Accumulator
 	wd, _ := os.Getwd()
 	r := File{
-		Files: []string{filepath.Join(wd, "testfiles/json_a.log")},
+		Files: []string{filepath.Join(wd, "dev/testfiles/json_a.log")},
 	}
 	parserConfig := parsers.Config{
 		DataFormat: "json",
 		TagKeys:    []string{"parent_ignored_child"},
 	}
 	nParser, err := parsers.NewParser(&parserConfig)
-	r.parser = nParser
 	assert.NoError(t, err)
+	r.parser = nParser
 
 	r.Gather(&acc)
 	assert.Equal(t, map[string]string{"parent_ignored_child": "hi"}, acc.Metrics[0].Tags)
@@ -44,7 +44,7 @@ func TestGrokParser(t *testing.T) {
 	wd, _ := os.Getwd()
 	var acc testutil.Accumulator
 	r := File{
-		Files: []string{filepath.Join(wd, "testfiles/grok_a.log")},
+		Files: []string{filepath.Join(wd, "dev/testfiles/grok_a.log")},
 	}
 
 	parserConfig := parsers.Config{
@@ -57,5 +57,5 @@ func TestGrokParser(t *testing.T) {
 	assert.NoError(t, err)
 
 	err = r.Gather(&acc)
-	assert.Equal(t, 2, len(acc.Metrics))
+	assert.Equal(t, len(acc.Metrics), 2)
 }

plugins/inputs/logparser/logparser.go

@@ -22,12 +22,12 @@ const (
 
 // LogParser in the primary interface for the plugin
 type GrokConfig struct {
-	MeasurementName    string `toml:"measurement"`
 	Patterns           []string
 	NamedPatterns      []string
 	CustomPatterns     string
 	CustomPatternFiles []string
 	TimeZone           string
+	MeasurementName    string `toml:"measurement"`
 }
 
 type logEntry struct {

plugins/inputs/logparser/logparser_test.go

@@ -44,9 +44,9 @@ func TestGrokParseLogFiles(t *testing.T) {
 
 	logparser := &LogParserPlugin{
 		GrokConfig: GrokConfig{
-			MeasurementName:    "logparser_grok",
 			Patterns:           []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"},
 			CustomPatternFiles: []string{thisdir + "grok/testdata/test-patterns"},
+			MeasurementName:    "logparser_grok",
 		},
 		FromBeginning: true,
 		Files:         []string{thisdir + "grok/testdata/*.log"},

plugins/parsers/grok/parser.go

@@ -37,6 +37,7 @@ var timeLayouts = map[string]string{
 }
 
 const (
+	MEASUREMENT       = "measurement"
 	INT               = "int"
 	TAG               = "tag"
 	FLOAT             = "float"
@@ -56,7 +57,7 @@ var (
 	//     %{IPORHOST:clientip:tag}
 	//     %{HTTPDATE:ts1:ts-http}
 	//     %{HTTPDATE:ts2:ts-"02 Jan 06 15:04"}
-	modifierRe = regexp.MustCompile(`%{\w+:(\w+):(ts-".+"|t?s?-?\w+)}`)
+	modifierRe = regexp.MustCompile(`%{\w+:(\w+|):(ts-".+"|t?s?-?\w+)}`)
 	// matches a plain pattern name. ie, %{NUMBER}
 	patternOnlyRe = regexp.MustCompile(`%{(\w+)}`)
 )
@@ -74,6 +75,9 @@ type Parser struct {
 	Measurement        string
 	DefaultTags        map[string]string
 
+	//holds any modifiers set on named user patterns
+	patternModifiers map[string][]string
+
 	// Timezone is an optional component to help render log dates to
 	// your chosen zone.
 	// Default: "" which renders UTC
@@ -126,6 +130,7 @@ func (p *Parser) Compile() error {
 	p.tsMap = make(map[string]map[string]string)
 	p.patterns = make(map[string]string)
 	p.tsModder = &tsModder{}
+	p.patternModifiers = make(map[string][]string)
 	var err error
 	p.g, err = grok.NewWithConfig(&grok.Config{NamedCapturesOnly: true})
 	if err != nil {
@@ -137,12 +142,32 @@ func (p *Parser) Compile() error {
 	p.NamedPatterns = make([]string, 0, len(p.Patterns))
 	for i, pattern := range p.Patterns {
 		pattern = strings.TrimSpace(pattern)
-		if pattern == "" {
-			continue
+
+		//checks that there is only one named field in pattern and 2 ':' indicating a modifier
+		//then extract any modifiers off pattern
+		if strings.Count(pattern, "%") == 1 && strings.Count(pattern, ":") == 2 {
+			pattern = strings.Trim(pattern, "%{}")
+			splitPattern := strings.SplitN(pattern, ":", 3)
+			if pattern == "" {
+				continue
+			}
+			name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
+
+			//map pattern modifiers by name
+			p.patternModifiers["%{"+name+"}"] = splitPattern[1:3]
+			p.CustomPatterns += "\n" + name + " " + "%{" + splitPattern[0] + "}" + "\n"
+			p.NamedPatterns = append(p.NamedPatterns, "%{"+name+"}")
+		} else {
+			if strings.Contains(pattern, ":measurement}") {
+				return fmt.Errorf("pattern with measurement modifier must have own 'pattern' field")
+			}
+			if pattern == "" {
+				continue
+			}
+			name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
+			p.CustomPatterns += "\n" + name + " " + pattern + "\n"
+			p.NamedPatterns = append(p.NamedPatterns, "%{"+name+"}")
 		}
-		name := fmt.Sprintf("GROK_INTERNAL_PATTERN_%d", i)
-		p.CustomPatterns += "\n" + name + " " + pattern + "\n"
-		p.NamedPatterns = append(p.NamedPatterns, "%{"+name+"}")
 	}
 
 	if len(p.NamedPatterns) == 0 {
@@ -213,7 +238,8 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 
 	timestamp := time.Now()
 	for k, v := range values {
-		if k == "" || v == "" {
+		if (k == "" || v == "") && p.typeMap[patternName][k] != "measurement" {
+			log.Printf("D! skipping key: %v", k)
 			continue
 		}
 
@@ -238,6 +264,8 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 		}
 
 		switch t {
+		case MEASUREMENT:
+			p.Measurement = v
 		case INT:
 			iv, err := strconv.ParseInt(v, 10, 64)
 			if err != nil {
@@ -348,8 +376,17 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 		}
 	}
 
+	//check the modifiers on the pattern
+	modifiers, ok := p.patternModifiers[patternName]
+	if ok && modifiers[1] == "measurement" {
+		if p.patternModifiers[patternName][0] == "" {
+			return nil, fmt.Errorf("pattern: %v must be named to use 'measurement' modifier", patternName)
+		}
+		p.Measurement = p.patternModifiers[patternName][0]
+	}
+
 	if len(fields) == 0 {
-		return nil, fmt.Errorf("logparser_grok: must have one or more fields")
+		return nil, fmt.Errorf("grok: must have one or more fields")
 	}
 
 	return metric.New(p.Measurement, tags, fields, p.tsModder.tsMod(timestamp))
@@ -453,6 +490,12 @@ func (p *Parser) parseTypedCaptures(name, pattern string) (string, error) {
 			}
 			hasTimestamp = true
 		} else {
+			//for handling measurement tag with no name
+			if match[1] == "" && match[2] == "measurement" {
+				match[1] = "measurement_name"
+				//add "measurement_name" to pattern so it is valid grok
+				pattern = strings.Replace(pattern, "::measurement", ":measurement_name:measurement", 1)
+			}
 			p.typeMap[patternName][match[1]] = match[2]
 		}
 

plugins/parsers/grok/parser_test.go

@@ -959,3 +959,76 @@ func TestReplaceTimestampComma(t *testing.T) {
 	//Convert Nanosecond to milisecond for compare
 	require.Equal(t, 555, m.Time().Nanosecond()/1000000)
 }
+
+func TestDynamicMeasurementModifier(t *testing.T) {
+	p := &Parser{
+		Patterns:       []string{"%{TEST}"},
+		CustomPatterns: "TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD:var3:measurement}",
+	}
+
+	require.NoError(t, p.Compile())
+	m, err := p.ParseLine("4 5 hello")
+	require.NoError(t, err)
+	require.Equal(t, m.Name(), "hello")
+}
+
+func TestStaticMeasurementModifier(t *testing.T) {
+	p := &Parser{
+		Patterns:       []string{"%{TEST:test_name:measurement}"},
+		CustomPatterns: "TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD:var3:string}",
+	}
+
+	require.NoError(t, p.Compile())
+	m, err := p.ParseLine("4 5 hello")
+	require.NoError(t, err)
+	require.Equal(t, m.Name(), "test_name")
+}
+
+func TestStaticAndDynamicMeasurementModifier(t *testing.T) {
+	p := &Parser{
+		Patterns:       []string{"%{TEST:test_name:measurement}"},
+		CustomPatterns: "TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD:var3:measurement}",
+	}
+
+	require.NoError(t, p.Compile())
+	m, err := p.ParseLine("4 5 hello")
+	require.NoError(t, err)
+	require.Equal(t, m.Name(), "test_name")
+}
+
+func TestMultipleMeasurementModifier(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{TEST:test_name:measurement}", "%{TEST2:test2_name:measurement"},
+		CustomPatterns: `TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD:var_string:string} 
+		TEST2 %{WORD:stringer1:tag} %{NUMBER:var2:float} %{NUMBER:var3:float}`,
+	}
+
+	require.NoError(t, p.Compile())
+	m, err := p.ParseLine("4 5 hello")
+	m2, err := p.ParseLine("mystr 5 9.5")
+	require.NoError(t, err)
+	require.Equal(t, m.Name(), "test_name")
+	require.Equal(t, m2.Name(), "test2_name")
+}
+
+func TestMeasurementModifierNoName(t *testing.T) {
+	p := &Parser{
+		Patterns:       []string{"%{TEST}"},
+		CustomPatterns: "TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD::measurement}",
+	}
+
+	require.NoError(t, p.Compile())
+	m, err := p.ParseLine("4 5 hello")
+	require.NoError(t, err)
+	require.Equal(t, m.Name(), "hello")
+}
+
+func TestMeasurementErrors(t *testing.T) {
+	p := &Parser{
+		Patterns: []string{"%{TEST:test_name:measurement}|%{TEST2:test2_name}"},
+		CustomPatterns: `TEST %{NUMBER:var1:tag} %{NUMBER:var2:float} %{WORD:var_string:string} 
+		TEST2 %{WORD:stringer1:tag} %{NUMBER:var2:float} %{NUMBER:var3:float}`,
+	}
+	err := p.Compile()
+	require.Error(t, err)
+}