From ff17ede62de63b11d2c5a6c65ec994872f452b39 Mon Sep 17 00:00:00 2001 From: Sven Rebhan <36194019+srebhan@users.noreply.github.com> Date: Thu, 28 Jul 2022 14:18:55 +0200 Subject: [PATCH] feat(parsers.xpath): Add support for returning underlying data-types (#11558) --- go.mod | 4 +- go.sum | 8 +- plugins/parsers/xpath/README.md | 8 ++ plugins/parsers/xpath/parser.go | 31 +++++-- plugins/parsers/xpath/parser_test.go | 81 ++++++++++++++++++ .../testcases/native_types_json/expected.out | 1 + .../testcases/native_types_json/telegraf.conf | 13 +++ .../testcases/native_types_json/test.json | 6 ++ .../native_types_msgpack/expected.out | 1 + .../native_types_json/expected.out | 1 + .../native_types_json/telegraf.conf | 12 +++ .../native_types_json/test.json | 5 ++ .../native_types_msgpack/telegraf.conf | 13 +++ .../testcases/native_types_msgpack/test.msg | Bin 0 -> 37 bytes .../native_types_protobuf/expected.out | 1 + .../native_types_protobuf/message.proto | 10 +++ .../native_types_protobuf/telegraf.conf | 17 ++++ .../testcases/native_types_protobuf/test.dat | 2 + 18 files changed, 200 insertions(+), 14 deletions(-) create mode 100644 plugins/parsers/xpath/testcases/native_types_json/expected.out create mode 100644 plugins/parsers/xpath/testcases/native_types_json/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/native_types_json/test.json create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/expected.out create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/native_types_msgpack/test.msg create mode 100644 plugins/parsers/xpath/testcases/native_types_protobuf/expected.out create mode 100644 plugins/parsers/xpath/testcases/native_types_protobuf/message.proto create mode 100644 plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf create mode 100644 plugins/parsers/xpath/testcases/native_types_protobuf/test.dat diff --git a/go.mod b/go.mod index 67af67d5e7246..9416796916c67 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1529 github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 - github.com/antchfx/jsonquery v1.2.0 + github.com/antchfx/jsonquery v1.3.0 github.com/antchfx/xmlquery v1.3.9 github.com/antchfx/xpath v1.2.1 github.com/apache/thrift v0.15.0 @@ -53,7 +53,7 @@ require ( github.com/djherbis/times v1.5.0 github.com/docker/docker v20.10.17+incompatible github.com/docker/go-connections v0.4.0 - github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 + github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9 github.com/dynatrace-oss/dynatrace-metric-utils-go v0.5.0 github.com/eclipse/paho.golang v0.10.0 github.com/eclipse/paho.mqtt.golang v1.3.5 diff --git a/go.sum b/go.sum index 724d70ba8e0ff..850688f6072ee 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,8 @@ github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RD github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/antchfx/jsonquery v1.2.0 h1:P7hhEM/+YQA5FPbBk+11QbmAtraSIu4d3FXXLtRzmho= -github.com/antchfx/jsonquery v1.2.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo= +github.com/antchfx/jsonquery v1.3.0 h1:rftVBKEXpj8C9WVu+4mbqL5hd6nLz7/AbIvAQlq3D7o= +github.com/antchfx/jsonquery v1.3.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo= github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ= github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= github.com/antchfx/xpath v1.1.7/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= @@ -683,8 +683,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60 h1:27379cxrsKlr7hAnW/xrusefspUPjqHVRW1K/bZgfGw= -github.com/doclambda/protobufquery v0.0.0-20210317203640-88ffabe06a60/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg= +github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9 h1:677nbAF3nq56BEZ2R/VMl0wROQqJo4vJ/ZWuzm+vsUU= +github.com/doclambda/protobufquery v0.0.0-20220727165953-0da287796ee9/go.mod h1:8Ia4zp86glrUhC29AAdK9hwTYh8RB6v0WRCtpplYqEg= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9 h1:NAvZb7gqQfLSNBPzVsvI7eZMosXtg2g2kxXrei90CtU= github.com/dropbox/godropbox v0.0.0-20180512210157-31879d3884b9/go.mod h1:glr97hP/JuXb+WMYCizc4PIFuzw1lCR97mwbe1VVXhQ= diff --git a/plugins/parsers/xpath/README.md b/plugins/parsers/xpath/README.md index 2d9ff8eeec628..73a494117ee24 100644 --- a/plugins/parsers/xpath/README.md +++ b/plugins/parsers/xpath/README.md @@ -102,6 +102,10 @@ You should use the following setting ## Useful when not all selected files have the exact same structure. # xpath_allow_empty_selection = false + ## Get native data-types for all data-format that contain type information. + ## Currently, protobuf, msgpack and JSON support native data-types + # xpath_native_types = false + ## Multiple parsing sections are allowed [[inputs.file.xpath]] ## Optional: XPath-query to select a subset of nodes from the XML document. @@ -180,6 +184,10 @@ in the metric. ## Useful when not all selected files have the exact same structure. # xpath_allow_empty_selection = false + ## Get native data-types for all data-format that contain type information. + ## Currently, protobuf, msgpack and JSON support native data-types + # xpath_native_types = false + ## Multiple parsing sections are allowed [[inputs.file.xpath]] ## Optional: XPath-query to select a subset of nodes from the XML document. diff --git a/plugins/parsers/xpath/parser.go b/plugins/parsers/xpath/parser.go index 361d3e1f18660..7d3e5e9309638 100644 --- a/plugins/parsers/xpath/parser.go +++ b/plugins/parsers/xpath/parser.go @@ -7,7 +7,9 @@ import ( "strings" "time" + "github.com/antchfx/jsonquery" path "github.com/antchfx/xpath" + "github.com/doclambda/protobufquery" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" @@ -34,6 +36,7 @@ type Parser struct { ProtobufImportPaths []string `toml:"xpath_protobuf_import_paths"` PrintDocument bool `toml:"xpath_print_document"` AllowEmptySelection bool `toml:"xpath_allow_empty_selection"` + NativeTypes bool `toml:"xpath_native_types"` Configs []xpath.Config `toml:"xpath"` DefaultMetricName string `toml:"-"` DefaultTags map[string]string `toml:"-"` @@ -451,17 +454,29 @@ func (p *Parser) executeQuery(doc, selected dataNode, query string) (r interface // separately. Those iterators will be returned for queries directly // referencing a node (value or attribute). n := expr.Evaluate(p.document.CreateXPathNavigator(root)) - if iter, ok := n.(*path.NodeIterator); ok { - // We got an iterator, so take the first match and get the referenced - // property. This will always be a string. - if iter.MoveNext() { - r = iter.Current().Value() + iter, ok := n.(*path.NodeIterator) + if !ok { + return n, nil + } + // We got an iterator, so take the first match and get the referenced + // property. This will always be a string. + if iter.MoveNext() { + current := iter.Current() + // If the dataformat supports native types and if support is + // enabled, we should return the native type of the data + if p.NativeTypes { + switch nn := current.(type) { + case *jsonquery.NodeNavigator: + return nn.GetValue(), nil + case *protobufquery.NodeNavigator: + return nn.GetValue(), nil + } } - } else { - r = n + // Fallback to get the string value representation + return iter.Current().Value(), nil } - return r, nil + return nil, nil } func splitLastPathElement(query string) []string { diff --git a/plugins/parsers/xpath/parser_test.go b/plugins/parsers/xpath/parser_test.go index 9b1e7a16328dd..fd32e5dd06aed 100644 --- a/plugins/parsers/xpath/parser_test.go +++ b/plugins/parsers/xpath/parser_test.go @@ -1,6 +1,9 @@ package xpath import ( + "bufio" + "errors" + "fmt" "math" "os" "path/filepath" @@ -9,6 +12,9 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/plugins/inputs/file" "github.com/influxdata/telegraf/plugins/parsers/influx" "github.com/influxdata/telegraf/plugins/parsers/temporary/xpath" "github.com/influxdata/telegraf/testutil" @@ -1388,6 +1394,81 @@ func TestProtobufImporting(t *testing.T) { require.NoError(t, parser.Init()) } +func TestMultipleConfigs(t *testing.T) { + // Get all directories in testdata + folders, err := os.ReadDir("testcases") + require.NoError(t, err) + + // Make sure the folder contains data + require.NotEmpty(t, folders) + + for _, f := range folders { + if !f.IsDir() { + continue + } + t.Run(f.Name(), func(t *testing.T) { + configFilename := filepath.Join("testcases", f.Name(), "telegraf.conf") + expectedFilename := filepath.Join("testcases", f.Name(), "expected.out") + + // Process the telegraf config file for the test + buf, err := os.ReadFile(configFilename) + if err != nil && errors.Is(err, os.ErrNotExist) { + return + } + require.NoError(t, err) + inputs.Add("file", func() telegraf.Input { + return &file.File{} + }) + cfg := config.NewConfig() + require.NoError(t, cfg.LoadConfigData(buf)) + + // Gather the metrics from the input file configure + acc := testutil.Accumulator{} + for _, input := range cfg.Inputs { + require.NoError(t, input.Init()) + require.NoError(t, input.Gather(&acc)) + } + + // Process expected metrics and compare with resulting metrics + expected, err := readMetricFile(expectedFilename) + require.NoError(t, err) + actual := acc.GetTelegrafMetrics() + testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) + }) + } +} + +func readMetricFile(filename string) ([]telegraf.Metric, error) { + var metrics []telegraf.Metric + + f, err := os.Open(filename) + if err != nil { + return metrics, err + } + defer f.Close() + + parser := &influx.Parser{} + if err := parser.Init(); err != nil { + return nil, err + } + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + m, err := parser.ParseLine(line) + if err != nil { + return nil, fmt.Errorf("unable to parse metric in %q failed: %v", line, err) + } + // The timezone needs to be UTC to match the timestamp test results + m.SetTime(m.Time().UTC()) + metrics = append(metrics, m) + } + } + + return metrics, nil +} + func loadTestConfiguration(filename string) (*xpath.Config, []string, error) { buf, err := os.ReadFile(filename) if err != nil { diff --git a/plugins/parsers/xpath/testcases/native_types_json/expected.out b/plugins/parsers/xpath/testcases/native_types_json/expected.out new file mode 100644 index 0000000000000..44c1ad0a9d888 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_json/expected.out @@ -0,0 +1 @@ +native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true diff --git a/plugins/parsers/xpath/testcases/native_types_json/telegraf.conf b/plugins/parsers/xpath/testcases/native_types_json/telegraf.conf new file mode 100644 index 0000000000000..ccd67c03fc6e6 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_json/telegraf.conf @@ -0,0 +1,13 @@ +[[inputs.file]] + files = ["./testcases/native_types_json/test.json"] + data_format = "xpath_json" + xpath_native_types = true + + [[inputs.file.xpath]] + metric_name = "'native_types'" + [inputs.file.xpath.fields] + value_a = "//a" + value_b = "//b" + value_c = "//c" + value_d = "//d" + diff --git a/plugins/parsers/xpath/testcases/native_types_json/test.json b/plugins/parsers/xpath/testcases/native_types_json/test.json new file mode 100644 index 0000000000000..51cd0282e8437 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_json/test.json @@ -0,0 +1,6 @@ +{ + "a": "a string", + "b": 3.1415, + "c": 42, + "d": true +} \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/expected.out b/plugins/parsers/xpath/testcases/native_types_msgpack/expected.out new file mode 100644 index 0000000000000..44c1ad0a9d888 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_msgpack/expected.out @@ -0,0 +1 @@ +native_types value_a="a string",value_b=3.1415,value_c=42.0,value_d=true diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out new file mode 100644 index 0000000000000..1114c44d333ce --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/expected.out @@ -0,0 +1 @@ +native_types value_a="a string",value_b=3.1415,value_c=true diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf new file mode 100644 index 0000000000000..210c8bcb90a8c --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/telegraf.conf @@ -0,0 +1,12 @@ +[[inputs.file]] + files = ["./testcases/native_types_json/test.json"] + data_format = "xpath_json" + xpath_native_types = true + + [[inputs.file.xpath]] + metric_name = "'native_types'" + [inputs.file.xpath.fields] + value_a = "//a" + value_b = "//b" + value_c = "//c" + diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json new file mode 100644 index 0000000000000..5c25603acd440 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_msgpack/native_types_json/test.json @@ -0,0 +1,5 @@ +{ + "a": "a string", + "b": 3.1415, + "c": true +} \ No newline at end of file diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf b/plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf new file mode 100644 index 0000000000000..2aa0e5d2ccfe5 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_msgpack/telegraf.conf @@ -0,0 +1,13 @@ +[[inputs.file]] + files = ["./testcases/native_types_msgpack/test.msg"] + data_format = "xpath_msgpack" + xpath_native_types = true + + [[inputs.file.xpath]] + metric_name = "'native_types'" + [inputs.file.xpath.fields] + value_a = "//a" + value_b = "//b" + value_c = "//c" + value_d = "//d" + diff --git a/plugins/parsers/xpath/testcases/native_types_msgpack/test.msg b/plugins/parsers/xpath/testcases/native_types_msgpack/test.msg new file mode 100644 index 0000000000000000000000000000000000000000..28a4c76a956d4053029eebd37c91834cb33d75b2 GIT binary patch literal 37 pcmZo!n7ATQp}3?dGcSE%(rE`y#Zw2Gh4L3BpLTF%0E2}ohXE-44k-Wt literal 0 HcmV?d00001 diff --git a/plugins/parsers/xpath/testcases/native_types_protobuf/expected.out b/plugins/parsers/xpath/testcases/native_types_protobuf/expected.out new file mode 100644 index 0000000000000..5b22dda20477a --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_protobuf/expected.out @@ -0,0 +1 @@ +native_types value_a="a string",value_b=3.1415,value_c=42i,value_d=true diff --git a/plugins/parsers/xpath/testcases/native_types_protobuf/message.proto b/plugins/parsers/xpath/testcases/native_types_protobuf/message.proto new file mode 100644 index 0000000000000..524eb5b1696e8 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_protobuf/message.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package native_type; + +message Message { + string a = 1; + double b = 2; + int32 c = 3; + bool d = 4; +} diff --git a/plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf b/plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf new file mode 100644 index 0000000000000..af7c42f75e988 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_protobuf/telegraf.conf @@ -0,0 +1,17 @@ +[[inputs.file]] + files = ["./testcases/native_types_protobuf/test.dat"] + data_format = "xpath_protobuf" + xpath_native_types = true + + xpath_protobuf_file = "message.proto" + xpath_protobuf_type = "native_type.Message" + xpath_protobuf_import_paths = [".", "./testcases/native_types_protobuf"] + + [[inputs.file.xpath]] + metric_name = "'native_types'" + [inputs.file.xpath.fields] + value_a = "//a" + value_b = "//b" + value_c = "//c" + value_d = "//d" + diff --git a/plugins/parsers/xpath/testcases/native_types_protobuf/test.dat b/plugins/parsers/xpath/testcases/native_types_protobuf/test.dat new file mode 100644 index 0000000000000..5b86ce78878a0 --- /dev/null +++ b/plugins/parsers/xpath/testcases/native_types_protobuf/test.dat @@ -0,0 +1,2 @@ + +a stringoƒÀÊ! @*  \ No newline at end of file