Add support for decimal timestamps to ts-epoch modifier (influxdata#3358

)
smolse · Oct 19, 2017 · 4deb623 · 4deb623
1 parent 7088d98
commit 4deb623
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 5 deletions.
diff --git a/plugins/inputs/logparser/README.md b/plugins/inputs/logparser/README.md
@@ -100,7 +100,7 @@ current time.
   - ts-rfc3339       ("2006-01-02T15:04:05Z07:00")
   - ts-rfc3339nano   ("2006-01-02T15:04:05.999999999Z07:00")
   - ts-httpd         ("02/Jan/2006:15:04:05 -0700")
-  - ts-epoch         (seconds since unix epoch)
+  - ts-epoch         (seconds since unix epoch, may contain decimal)
   - ts-epochnano     (nanoseconds since unix epoch)
   - ts-"CUSTOM"
 
@@ -130,6 +130,19 @@ This example input and config parses a file using a custom timestamp conversion:
     patterns = ['%{TIMESTAMP_ISO8601:timestamp:ts-"2006-01-02 15:04:05"} value=%{NUMBER:value:int}']
 ```
 
+This example input and config parses a file using a timestamp in unix time:
+
+```
+1466004605 value=42
+1466004605.123456789 value=42
+```
+
+```toml
+[[inputs.logparser]]
+  [inputs.logparser.grok]
+    patterns = ['%{NUMBER:timestamp:ts-epoch} value=%{NUMBER:value:int}']
+```
+
 This example parses a file using a built-in conversion and a custom pattern:
 
 ```

diff --git a/plugins/inputs/logparser/grok/grok.go b/plugins/inputs/logparser/grok/grok.go
@@ -253,12 +253,30 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
 		case STRING:
 			fields[k] = strings.Trim(v, `"`)
 		case EPOCH:
-			iv, err := strconv.ParseInt(v, 10, 64)
+			parts := strings.SplitN(v, ".", 2)
+			if len(parts) == 0 {
+				log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+				break
+			}
+
+			sec, err := strconv.ParseInt(parts[0], 10, 64)
 			if err != nil {
-				log.Printf("E! Error parsing %s to int: %s", v, err)
-			} else {
-				timestamp = time.Unix(iv, 0)
+				log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+				break
+			}
+			ts := time.Unix(sec, 0)
+
+			if len(parts) == 2 {
+				padded := fmt.Sprintf("%-9s", parts[1])
+				nsString := strings.Replace(padded[:9], " ", "0", -1)
+				nanosec, err := strconv.ParseInt(nsString, 10, 64)
+				if err != nil {
+					log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+					break
+				}
+				ts = ts.Add(time.Duration(nanosec) * time.Nanosecond)
 			}
+			timestamp = ts
 		case EPOCH_NANO:
 			iv, err := strconv.ParseInt(v, 10, 64)
 			if err != nil {

diff --git a/plugins/inputs/logparser/grok/grok_test.go b/plugins/inputs/logparser/grok/grok_test.go
@@ -385,6 +385,77 @@ func TestParseEpoch(t *testing.T) {
 	assert.Equal(t, time.Unix(1466004605, 0), metricA.Time())
 }
 
+func TestParseEpochDecimal(t *testing.T) {
+	var tests = []struct {
+		name    string
+		line    string
+		noMatch bool
+		err     error
+		tags    map[string]string
+		fields  map[string]interface{}
+		time    time.Time
+	}{
+		{
+			name: "ns precision",
+			line: "1466004605.359052000 value=42",
+			tags: map[string]string{},
+			fields: map[string]interface{}{
+				"value": int64(42),
+			},
+			time: time.Unix(0, 1466004605359052000),
+		},
+		{
+			name: "ms precision",
+			line: "1466004605.359 value=42",
+			tags: map[string]string{},
+			fields: map[string]interface{}{
+				"value": int64(42),
+			},
+			time: time.Unix(0, 1466004605359000000),
+		},
+		{
+			name: "second precision",
+			line: "1466004605 value=42",
+			tags: map[string]string{},
+			fields: map[string]interface{}{
+				"value": int64(42),
+			},
+			time: time.Unix(0, 1466004605000000000),
+		},
+		{
+			name: "sub ns precision",
+			line: "1466004605.123456789123 value=42",
+			tags: map[string]string{},
+			fields: map[string]interface{}{
+				"value": int64(42),
+			},
+			time: time.Unix(0, 1466004605123456789),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			parser := &Parser{
+				Patterns: []string{"%{NUMBER:ts:ts-epoch} value=%{NUMBER:value:int}"},
+			}
+			assert.NoError(t, parser.Compile())
+			m, err := parser.ParseLine(tt.line)
+
+			if tt.noMatch {
+				require.Nil(t, m)
+				require.Nil(t, err)
+				return
+			}
+
+			require.Equal(t, tt.err, err)
+
+			require.NotNil(t, m)
+			require.Equal(t, tt.tags, m.Tags())
+			require.Equal(t, tt.fields, m.Fields())
+			require.Equal(t, tt.time, m.Time())
+		})
+	}
+}
+
 func TestParseEpochErrors(t *testing.T) {
 	p := &Parser{
 		Patterns: []string{"%{MYAPP}"},