feat(parsers.xpath): Add support for returning underlying data-types (#…

…11558)

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529
 	github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9
-	github.com/antchfx/jsonquery v1.2.0
+	github.com/antchfx/jsonquery v1.3.0
 	github.com/antchfx/xmlquery v1.3.9
 	github.com/antchfx/xpath v1.2.1
 	github.com/apache/thrift v0.15.0
@@ -53,7 +53,7 @@ require (
 	github.com/djherbis/times v1.5.0
 	github.com/docker/docker v20.10.17+incompatible
 	github.com/docker/go-connections v0.4.0
-	github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60
+	github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9
 	github.com/dynatrace-oss/dynatrace-metric-utils-go v0.5.0
 	github.com/eclipse/paho.golang v0.10.0
 	github.com/eclipse/paho.mqtt.golang v1.3.5

go.sum

@@ -263,8 +263,8 @@ github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RD
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
-github.com/antchfx/jsonquery v1.2.0 h1:P7hhEM/+YQA5FPbBk+11QbmAtraSIu4d3FXXLtRzmho=
-github.com/antchfx/jsonquery v1.2.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo=
+github.com/antchfx/jsonquery v1.3.0 h1:rftVBKEXpj8C9WVu+4mbqL5hd6nLz7/AbIvAQlq3D7o=
+github.com/antchfx/jsonquery v1.3.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo=
 github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ=
 github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
 github.com/antchfx/xpath v1.1.7/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
@@ -683,8 +683,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
-github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 h1:27379cxrsKlr7hAnW/xrusefspUPjqHVRW1K/bZgfGw=
-github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg=
+github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9 h1:677nbAF3nq56BEZ2R/VMl0wROQqJo4vJ/ZWuzm+vsUU=
+github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9 h1:NAvZb7gqQfLSNBPzVsvI7eZMosXtg2g2kxXrei90CtU=
 github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9/go.mod h1:glr97hP/JuXb+WMYCizc4PIFuzw1lCR97mwbe1VVXhQ=

plugins/parsers/xpath/README.md

@@ -102,6 +102,10 @@ You should use the following setting
   ## Useful when not all selected files have the exact same structure.
   # xpath_allow_empty_selection = false
 
+  ## Get native data-types for all data-format that contain type information.
+  ## Currently, protobuf, msgpack and JSON support native data-types
+  # xpath_native_types = false
+
   ## Multiple parsing sections are allowed
   [[inputs.file.xpath]]
     ## Optional: XPath-query to select a subset of nodes from the XML document.
@@ -180,6 +184,10 @@ in the metric.
   ## Useful when not all selected files have the exact same structure.
   # xpath_allow_empty_selection = false
 
+  ## Get native data-types for all data-format that contain type information.
+  ## Currently, protobuf, msgpack and JSON support native data-types
+  # xpath_native_types = false
+
   ## Multiple parsing sections are allowed
   [[inputs.file.xpath]]
     ## Optional: XPath-query to select a subset of nodes from the XML document.

plugins/parsers/xpath/parser.go

@@ -7,7 +7,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/antchfx/jsonquery"
 	path "github.com/antchfx/xpath"
+	"github.com/doclambda/protobufquery"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
@@ -34,6 +36,7 @@ type Parser struct {
 	ProtobufImportPaths []string          `toml:"xpath_protobuf_import_paths"`
 	PrintDocument       bool              `toml:"xpath_print_document"`
 	AllowEmptySelection bool              `toml:"xpath_allow_empty_selection"`
+	NativeTypes         bool              `toml:"xpath_native_types"`
 	Configs             []xpath.Config    `toml:"xpath"`
 	DefaultMetricName   string            `toml:"-"`
 	DefaultTags         map[string]string `toml:"-"`
@@ -451,17 +454,29 @@ func (p *Parser) executeQuery(doc, selected dataNode, query string) (r interface
 	// separately. Those iterators will be returned for queries directly
 	// referencing a node (value or attribute).
 	n := expr.Evaluate(p.document.CreateXPathNavigator(root))
-	if iter, ok := n.(*path.NodeIterator); ok {
-		// We got an iterator, so take the first match and get the referenced
-		// property. This will always be a string.
-		if iter.MoveNext() {
-			r = iter.Current().Value()
+	iter, ok := n.(*path.NodeIterator)
+	if !ok {
+		return n, nil
+	}
+	// We got an iterator, so take the first match and get the referenced
+	// property. This will always be a string.
+	if iter.MoveNext() {
+		current := iter.Current()
+		// If the dataformat supports native types and if support is
+		// enabled, we should return the native type of the data
+		if p.NativeTypes {
+			switch nn := current.(type) {
+			case *jsonquery.NodeNavigator:
+				return nn.GetValue(), nil
+			case *protobufquery.NodeNavigator:
+				return nn.GetValue(), nil
+			}
 		}
-	} else {
-		r = n
+		// Fallback to get the string value representation
+		return iter.Current().Value(), nil
 	}
 
-	return r, nil
+	return nil, nil
 }
 
 func splitLastPathElement(query string) []string {

plugins/parsers/xpath/parser_test.go

@@ -1,6 +1,9 @@
 package xpath
 
 import (
+	"bufio"
+	"errors"
+	"fmt"
 	"math"
 	"os"
 	"path/filepath"
@@ -9,6 +12,9 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/plugins/inputs/file"
 	"github.com/influxdata/telegraf/plugins/parsers/influx"
 	"github.com/influxdata/telegraf/plugins/parsers/temporary/xpath"
 	"github.com/influxdata/telegraf/testutil"
@@ -1388,6 +1394,81 @@ func TestProtobufImporting(t *testing.T) {
 	require.NoError(t, parser.Init())
 }
 
+func TestMultipleConfigs(t *testing.T) {
+	// Get all directories in testdata
+	folders, err := os.ReadDir("testcases")
+	require.NoError(t, err)
+
+	// Make sure the folder contains data
+	require.NotEmpty(t, folders)
+
+	for _, f := range folders {
+		if !f.IsDir() {
+			continue
+		}
+		t.Run(f.Name(), func(t *testing.T) {
+			configFilename := filepath.Join("testcases", f.Name(), "telegraf.conf")
+			expectedFilename := filepath.Join("testcases", f.Name(), "expected.out")
+
+			// Process the telegraf config file for the test
+			buf, err := os.ReadFile(configFilename)
+			if err != nil && errors.Is(err, os.ErrNotExist) {
+				return
+			}
+			require.NoError(t, err)
+			inputs.Add("file", func() telegraf.Input {
+				return &file.File{}
+			})
+			cfg := config.NewConfig()
+			require.NoError(t, cfg.LoadConfigData(buf))
+
+			// Gather the metrics from the input file configure
+			acc := testutil.Accumulator{}
+			for _, input := range cfg.Inputs {
+				require.NoError(t, input.Init())
+				require.NoError(t, input.Gather(&acc))
+			}
+
+			// Process expected metrics and compare with resulting metrics
+			expected, err := readMetricFile(expectedFilename)
+			require.NoError(t, err)
+			actual := acc.GetTelegrafMetrics()
+			testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime())
+		})
+	}
+}
+
+func readMetricFile(filename string) ([]telegraf.Metric, error) {
+	var metrics []telegraf.Metric
+
+	f, err := os.Open(filename)
+	if err != nil {
+		return metrics, err
+	}
+	defer f.Close()
+
+	parser := &influx.Parser{}
+	if err := parser.Init(); err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line != "" {
+			m, err := parser.ParseLine(line)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse metric in %q failed: %v", line, err)
+			}
+			// The timezone needs to be UTC to match the timestamp test results
+			m.SetTime(m.Time().UTC())
+			metrics = append(metrics, m)
+		}
+	}
+
+	return metrics, nil
+}
+
 func loadTestConfiguration(filename string) (*xpath.Config, []string, error) {
 	buf, err := os.ReadFile(filename)
 	if err != nil {

plugins/parsers/xpath/testcases/native_types_json/expected.out

@@ -0,0 +1 @@
+native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true

plugins/parsers/xpath/testcases/native_types_json/telegraf.conf

@@ -0,0 +1,13 @@
+[[inputs.file]]
+  files = ["./testcases/native_types_json/test.json"]
+  data_format = "xpath_json"
+  xpath_native_types = true
+
+  [[inputs.file.xpath]]
+    metric_name = "'native_types'"
+    [inputs.file.xpath.fields]
+      value_a = "//a"
+      value_b = "//b"
+      value_c = "//c"
+      value_d = "//d"
+

plugins/parsers/xpath/testcases/native_types_json/test.json

@@ -0,0 +1,6 @@
+{
+  "a": "a string",
+  "b": 3.1415,
+  "c": 42,
+  "d": true
+}

plugins/parsers/xpath/testcases/native_types_msgpack/expected.out

@@ -0,0 +1 @@
+native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true

plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out

@@ -0,0 +1 @@
+native_types value_a="a string",value_b=3.1415,value_c=true

plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf

@@ -0,0 +1,12 @@
+[[inputs.file]]
+  files = ["./testcases/native_types_json/test.json"]
+  data_format = "xpath_json"
+  xpath_native_types = true
+
+  [[inputs.file.xpath]]
+    metric_name = "'native_types'"
+    [inputs.file.xpath.fields]
+      value_a = "//a"
+      value_b = "//b"
+      value_c = "//c"
+

plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json

@@ -0,0 +1,5 @@
+{
+  "a": "a string",
+  "b": 3.1415,
+  "c": true
+}

plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf

@@ -0,0 +1,13 @@
+[[inputs.file]]
+  files = ["./testcases/native_types_msgpack/test.msg"]
+  data_format = "xpath_msgpack"
+  xpath_native_types = true
+
+  [[inputs.file.xpath]]
+    metric_name = "'native_types'"
+    [inputs.file.xpath.fields]
+      value_a = "//a"
+      value_b = "//b"
+      value_c = "//c"
+      value_d = "//d"
+

plugins/parsers/xpath/testcases/native_types_protobuf/expected.out

@@ -0,0 +1 @@
+native_types value_a="a string",value_b=3.1415,value_c=42i,value_d=true

plugins/parsers/xpath/testcases/native_types_protobuf/message.proto

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package native_type;
+
+message Message {
+    string a = 1;
+    double b = 2;
+    int32 c = 3;
+    bool d = 4;
+}

plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf

@@ -0,0 +1,17 @@
+[[inputs.file]]
+  files = ["./testcases/native_types_protobuf/test.dat"]
+  data_format = "xpath_protobuf"
+  xpath_native_types = true
+
+  xpath_protobuf_file = "message.proto"
+  xpath_protobuf_type = "native_type.Message"
+  xpath_protobuf_import_paths = [".", "./testcases/native_types_protobuf"]
+
+  [[inputs.file.xpath]]
+    metric_name = "'native_types'"
+    [inputs.file.xpath.fields]
+      value_a = "//a"
+      value_b = "//b"
+      value_c = "//c"
+      value_d = "//d"
+

plugins/parsers/xpath/testcases/native_types_protobuf/test.dat

@@ -0,0 +1,2 @@
+
+a stringo���!	@*