From ddf77c94b0d03a9892ef6eec1bd0dc62804285a0 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 13:47:56 -0400 Subject: [PATCH 1/9] feat(outputs.dynatrace) add new configuration: additional_counters_patterns and update test --- plugins/outputs/dynatrace/dynatrace.go | 42 ++++++-- plugins/outputs/dynatrace/dynatrace_test.go | 108 ++++++++++++++++++++ 2 files changed, 140 insertions(+), 10 deletions(-) diff --git a/plugins/outputs/dynatrace/dynatrace.go b/plugins/outputs/dynatrace/dynatrace.go index ade9be8ca900e..f28d0cbbb1127 100644 --- a/plugins/outputs/dynatrace/dynatrace.go +++ b/plugins/outputs/dynatrace/dynatrace.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "regexp" "strings" "time" @@ -26,12 +27,14 @@ var sampleConfig string // Dynatrace Configuration for the Dynatrace output plugin type Dynatrace struct { - URL string `toml:"url"` - APIToken config.Secret `toml:"api_token"` - Prefix string `toml:"prefix"` - Log telegraf.Logger `toml:"-"` - Timeout config.Duration `toml:"timeout"` - AddCounterMetrics []string `toml:"additional_counters"` + URL string `toml:"url"` + APIToken config.Secret `toml:"api_token"` + Prefix string `toml:"prefix"` + Log telegraf.Logger `toml:"-"` + Timeout config.Duration `toml:"timeout"` + AddCounterMetrics []string `toml:"additional_counters"` + AddCounterMetricsPatterns []string `toml:"additional_counters_patterns"` + DefaultDimensions map[string]string `toml:"default_dimensions"` normalizedDefaultDimensions dimensions.NormalizedDimensionList @@ -229,10 +232,10 @@ func init() { func (d *Dynatrace) getTypeOption(metric telegraf.Metric, field *telegraf.Field) dtMetric.MetricOption { metricName := metric.Name() + "." + field.Key - for _, i := range d.AddCounterMetrics { - if metricName != i { - continue - } + + if d.isCounterMetricsMatch(d.AddCounterMetrics, metricName) || + d.isCounterMetricsPatternsMatch(d.AddCounterMetricsPatterns, metricName) { + switch v := field.Value.(type) { case float64: return dtMetric.WithFloatCounterValueDelta(v) @@ -261,3 +264,22 @@ func (d *Dynatrace) getTypeOption(metric telegraf.Metric, field *telegraf.Field) return nil } + +func (d *Dynatrace) isCounterMetricsMatch(counterMetrics []string, metricName string) bool { + for _, i := range counterMetrics { + if i == metricName { + return true + } + } + return false +} + +func (d *Dynatrace) isCounterMetricsPatternsMatch(counterPatterns []string, metricName string) bool { + for _, pattern := range counterPatterns { + regex, err := regexp.Compile(pattern) + if err == nil && regex.MatchString(metricName) { + return true + } + } + return false +} diff --git a/plugins/outputs/dynatrace/dynatrace_test.go b/plugins/outputs/dynatrace/dynatrace_test.go index 973524d335c47..15b4c713ca54b 100644 --- a/plugins/outputs/dynatrace/dynatrace_test.go +++ b/plugins/outputs/dynatrace/dynatrace_test.go @@ -213,6 +213,114 @@ func TestSendMetrics(t *testing.T) { require.NoError(t, err) } +func TestSendMetricsWithPatterns(t *testing.T) { + expected := []string{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // check the encoded result + bodyBytes, err := io.ReadAll(r.Body) + require.NoError(t, err) + bodyString := string(bodyBytes) + + lines := strings.Split(bodyString, "\n") + + sort.Strings(lines) + sort.Strings(expected) + + expectedString := strings.Join(expected, "\n") + foundString := strings.Join(lines, "\n") + if foundString != expectedString { + t.Errorf("Metric encoding failed. expected: %#v but got: %#v", expectedString, foundString) + } + w.WriteHeader(http.StatusOK) + err = json.NewEncoder(w).Encode(fmt.Sprintf(`{"linesOk":%d,"linesInvalid":0,"error":null}`, len(lines))) + require.NoError(t, err) + })) + defer ts.Close() + + d := &Dynatrace{ + URL: ts.URL, + APIToken: config.NewSecret([]byte("123")), + Log: testutil.Logger{}, + AddCounterMetrics: []string{}, + AddCounterMetricsPatterns: []string{}, + } + + err := d.Init() + require.NoError(t, err) + err = d.Connect() + require.NoError(t, err) + + // Init metrics + + // Simple metrics are exported as a gauge unless pattern match in additional_counters_patterns + expected = append(expected, + "simple_abc_metric.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", + "simple_abc_metric.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", + "simple_xyz_metric.value,dt.metrics.source=telegraf gauge,3.14 1289516400000", + "simple_xyz_metric.counter,dt.metrics.source=telegraf count,delta=5 1289516400000", + ) + d.AddCounterMetricsPatterns = append(d.AddCounterMetricsPatterns, "simple_[a-z]+_metric.counter") + m1 := metric.New( + "simple_abc_metric", + map[string]string{}, + map[string]interface{}{"value": float64(3.14), "counter": 5}, + time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), + ) + + m2 := metric.New( + "simple_xyz_metric", + map[string]string{}, + map[string]interface{}{"value": float64(3.14), "counter": 5}, + time.Date(2010, time.November, 11, 23, 0, 0, 0, time.UTC), + ) + + // Even if Type() returns counter, all metrics are treated as a gauge unless pattern match with additional_counters_patterns + expected = append(expected, + "counter_fan01_type.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", + "counter_fan01_type.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", + "counter_fanNaN_type.counter,dt.metrics.source=telegraf gauge,5 1289516400000", + "counter_fanNaN_type.value,dt.metrics.source=telegraf gauge,3.14 1289516400000", + ) + d.AddCounterMetricsPatterns = append(d.AddCounterMetricsPatterns, "counter_fan[0-9]+_type.counter") + m3 := metric.New( + "counter_fan01_type", + map[string]string{}, + map[string]interface{}{"value": float64(3.14), "counter": 5}, + time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), + telegraf.Counter, + ) + + m4 := metric.New( + "counter_fanNaN_type", + map[string]string{}, + map[string]interface{}{"value": float64(3.14), "counter": 5}, + time.Date(2010, time.November, 11, 23, 0, 0, 0, time.UTC), + telegraf.Counter, + ) + + expected = append(expected, + "complex_metric.int,dt.metrics.source=telegraf gauge,1 1289430000000", + "complex_metric.int64,dt.metrics.source=telegraf gauge,2 1289430000000", + "complex_metric.float,dt.metrics.source=telegraf gauge,3 1289430000000", + "complex_metric.float64,dt.metrics.source=telegraf gauge,4 1289430000000", + "complex_metric.true,dt.metrics.source=telegraf gauge,1 1289430000000", + "complex_metric.false,dt.metrics.source=telegraf gauge,0 1289430000000", + ) + + m5 := metric.New( + "complex_metric", + map[string]string{}, + map[string]interface{}{"int": 1, "int64": int64(2), "float": 3.0, "float64": float64(4.0), "true": true, "false": false}, + time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), + ) + + metrics := []telegraf.Metric{m1, m2, m3, m4, m5} + + err = d.Write(metrics) + require.NoError(t, err) +} + func TestSendSingleMetricWithUnorderedTags(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // check the encoded result From 87f5e8cf42797481c82c8cd2382d485194a2f627 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 13:48:44 -0400 Subject: [PATCH 2/9] chore(outputs.dynatrace) update docs --- plugins/outputs/dynatrace/README.md | 19 +++++++++++++++++-- plugins/outputs/dynatrace/sample.conf | 4 ++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/dynatrace/README.md b/plugins/outputs/dynatrace/README.md index 2f0fe3d48d8ee..96de279dcb7e0 100644 --- a/plugins/outputs/dynatrace/README.md +++ b/plugins/outputs/dynatrace/README.md @@ -6,8 +6,8 @@ OneAgent for automatic authentication or it may be run standalone on a host without a OneAgent by specifying a URL and API Token. More information on the plugin can be found in the [Dynatrace documentation][docs]. All metrics are reported as gauges, unless they are specified to be delta counters using the -`additional_counters` config option (see below). See the [Dynatrace Metrics -ingestion protocol documentation][proto-docs] for details on the types defined +`additional_counters` or `additional_counters_patterns` config option (see below). +See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] for details on the types defined there. [api-v2]: https://docs.dynatrace.com/docs/shortlink/api-metrics-v2 @@ -144,6 +144,10 @@ to use them. ## If you want metrics to be treated and reported as delta counters, add the metric names here additional_counters = [ ] + ## In addition or as an alternative to additional_counters, if you want metrics to be treated and reported as + ## delta counters using regular expression pattern matching + additional_counters_patterns = [ ] + ## NOTE: Due to the way TOML is parsed, tables must be at the END of the ## plugin definition, otherwise additional config options are read as part of ## the table @@ -216,6 +220,17 @@ to this list. additional_counters = [ ] ``` +### `additional_counters_patterns` + +*required*: `false` + +In addition or as an alternative to additional_counters, if you want a metric to be treated and +reported as a delta counter using regular expression, add its pattern to this list. + +```toml +additional_counters_patterns = [ ] +``` + ### `default_dimensions` *required*: `false` diff --git a/plugins/outputs/dynatrace/sample.conf b/plugins/outputs/dynatrace/sample.conf index ff1eb1b2c2a1f..b940a3b757a07 100644 --- a/plugins/outputs/dynatrace/sample.conf +++ b/plugins/outputs/dynatrace/sample.conf @@ -31,6 +31,10 @@ ## If you want metrics to be treated and reported as delta counters, add the metric names here additional_counters = [ ] + ## In addition or as an alternative to additional_counters, if you want metrics to be treated and + ## reported as delta counters using regular expression pattern matching + additional_counters_patterns = [ ] + ## NOTE: Due to the way TOML is parsed, tables must be at the END of the ## plugin definition, otherwise additional config options are read as part of ## the table From 85d2ad54104ffb5473c5540643b6551e5609eeb8 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 14:38:48 -0400 Subject: [PATCH 3/9] chore(outputs.dynatrace) update docs --- plugins/outputs/dynatrace/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/outputs/dynatrace/README.md b/plugins/outputs/dynatrace/README.md index 96de279dcb7e0..54ba8f228ba8f 100644 --- a/plugins/outputs/dynatrace/README.md +++ b/plugins/outputs/dynatrace/README.md @@ -144,8 +144,8 @@ to use them. ## If you want metrics to be treated and reported as delta counters, add the metric names here additional_counters = [ ] - ## In addition or as an alternative to additional_counters, if you want metrics to be treated and reported as - ## delta counters using regular expression pattern matching + ## In addition or as an alternative to additional_counters, if you want metrics to be treated and + ## reported as delta counters using regular expression pattern matching additional_counters_patterns = [ ] ## NOTE: Due to the way TOML is parsed, tables must be at the END of the From e7f19445193b193f36f9fc477e40a2507b1f6d3b Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 14:41:55 -0400 Subject: [PATCH 4/9] chore(outputs.dynatrace) cleanup docs --- plugins/outputs/dynatrace/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/outputs/dynatrace/README.md b/plugins/outputs/dynatrace/README.md index 54ba8f228ba8f..bd237348ad6a5 100644 --- a/plugins/outputs/dynatrace/README.md +++ b/plugins/outputs/dynatrace/README.md @@ -6,9 +6,10 @@ OneAgent for automatic authentication or it may be run standalone on a host without a OneAgent by specifying a URL and API Token. More information on the plugin can be found in the [Dynatrace documentation][docs]. All metrics are reported as gauges, unless they are specified to be delta counters using the -`additional_counters` or `additional_counters_patterns` config option (see below). -See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] for details on the types defined -there. +`additional_counters` or `additional_counters_patterns` config option +(see below). +See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] for details +on the types defined there. [api-v2]: https://docs.dynatrace.com/docs/shortlink/api-metrics-v2 @@ -224,8 +225,9 @@ additional_counters = [ ] *required*: `false` -In addition or as an alternative to additional_counters, if you want a metric to be treated and -reported as a delta counter using regular expression, add its pattern to this list. +In addition or as an alternative to additional_counters, if you want a metric to be +treated and reported as a delta counter using regular expression, add its pattern +to this list. ```toml additional_counters_patterns = [ ] From 7a2804c52067e011fd63728b5c71e9255e30ab0b Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 14:43:28 -0400 Subject: [PATCH 5/9] chore(outputs.dynatrace) cleanup docs --- plugins/outputs/dynatrace/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/outputs/dynatrace/README.md b/plugins/outputs/dynatrace/README.md index bd237348ad6a5..f4eb0d35c2392 100644 --- a/plugins/outputs/dynatrace/README.md +++ b/plugins/outputs/dynatrace/README.md @@ -8,8 +8,8 @@ plugin can be found in the [Dynatrace documentation][docs]. All metrics are reported as gauges, unless they are specified to be delta counters using the `additional_counters` or `additional_counters_patterns` config option (see below). -See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] for details -on the types defined there. +See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] +for details on the types defined there. [api-v2]: https://docs.dynatrace.com/docs/shortlink/api-metrics-v2 @@ -225,9 +225,9 @@ additional_counters = [ ] *required*: `false` -In addition or as an alternative to additional_counters, if you want a metric to be -treated and reported as a delta counter using regular expression, add its pattern -to this list. +In addition or as an alternative to additional_counters, if you want a metric +to be treated and reported as a delta counter using regular expression, +add its pattern to this list. ```toml additional_counters_patterns = [ ] From 3447395867600950c63f509cfd0dd084e308b798 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 14:50:44 -0400 Subject: [PATCH 6/9] chore(outputs.dynatrace) cleanup docs --- plugins/outputs/dynatrace/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/outputs/dynatrace/README.md b/plugins/outputs/dynatrace/README.md index f4eb0d35c2392..3bda9dab8a49a 100644 --- a/plugins/outputs/dynatrace/README.md +++ b/plugins/outputs/dynatrace/README.md @@ -8,7 +8,7 @@ plugin can be found in the [Dynatrace documentation][docs]. All metrics are reported as gauges, unless they are specified to be delta counters using the `additional_counters` or `additional_counters_patterns` config option (see below). -See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] +See the [Dynatrace Metrics ingestion protocol documentation][proto-docs] for details on the types defined there. [api-v2]: https://docs.dynatrace.com/docs/shortlink/api-metrics-v2 @@ -225,8 +225,8 @@ additional_counters = [ ] *required*: `false` -In addition or as an alternative to additional_counters, if you want a metric -to be treated and reported as a delta counter using regular expression, +In addition or as an alternative to additional_counters, if you want a metric +to be treated and reported as a delta counter using regular expression, add its pattern to this list. ```toml From 0b623266c3cded504314a8c62bc1ceea8f7c1ea0 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 15:19:35 -0400 Subject: [PATCH 7/9] chore(outputs.dynatrace) cleanup code --- plugins/outputs/dynatrace/dynatrace.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/outputs/dynatrace/dynatrace.go b/plugins/outputs/dynatrace/dynatrace.go index f28d0cbbb1127..357e9299a4e39 100644 --- a/plugins/outputs/dynatrace/dynatrace.go +++ b/plugins/outputs/dynatrace/dynatrace.go @@ -232,10 +232,8 @@ func init() { func (d *Dynatrace) getTypeOption(metric telegraf.Metric, field *telegraf.Field) dtMetric.MetricOption { metricName := metric.Name() + "." + field.Key - if d.isCounterMetricsMatch(d.AddCounterMetrics, metricName) || d.isCounterMetricsPatternsMatch(d.AddCounterMetricsPatterns, metricName) { - switch v := field.Value.(type) { case float64: return dtMetric.WithFloatCounterValueDelta(v) @@ -247,7 +245,6 @@ func (d *Dynatrace) getTypeOption(metric telegraf.Metric, field *telegraf.Field) return nil } } - switch v := field.Value.(type) { case float64: return dtMetric.WithFloatGaugeValue(v) From 44706c14db929b215101ceaf1807391706b15ed2 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 19:22:57 -0400 Subject: [PATCH 8/9] chore(outputs.dynatrace) cleanup test code --- plugins/outputs/dynatrace/dynatrace_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/outputs/dynatrace/dynatrace_test.go b/plugins/outputs/dynatrace/dynatrace_test.go index 15b4c713ca54b..0422fe8e27bd5 100644 --- a/plugins/outputs/dynatrace/dynatrace_test.go +++ b/plugins/outputs/dynatrace/dynatrace_test.go @@ -257,8 +257,8 @@ func TestSendMetricsWithPatterns(t *testing.T) { expected = append(expected, "simple_abc_metric.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", "simple_abc_metric.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", - "simple_xyz_metric.value,dt.metrics.source=telegraf gauge,3.14 1289516400000", - "simple_xyz_metric.counter,dt.metrics.source=telegraf count,delta=5 1289516400000", + "simple_xyz_metric.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", + "simple_xyz_metric.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", ) d.AddCounterMetricsPatterns = append(d.AddCounterMetricsPatterns, "simple_[a-z]+_metric.counter") m1 := metric.New( @@ -272,15 +272,15 @@ func TestSendMetricsWithPatterns(t *testing.T) { "simple_xyz_metric", map[string]string{}, map[string]interface{}{"value": float64(3.14), "counter": 5}, - time.Date(2010, time.November, 11, 23, 0, 0, 0, time.UTC), + time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), ) // Even if Type() returns counter, all metrics are treated as a gauge unless pattern match with additional_counters_patterns expected = append(expected, "counter_fan01_type.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", "counter_fan01_type.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", - "counter_fanNaN_type.counter,dt.metrics.source=telegraf gauge,5 1289516400000", - "counter_fanNaN_type.value,dt.metrics.source=telegraf gauge,3.14 1289516400000", + "counter_fanNaN_type.counter,dt.metrics.source=telegraf gauge,5 1289430000000", + "counter_fanNaN_type.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", ) d.AddCounterMetricsPatterns = append(d.AddCounterMetricsPatterns, "counter_fan[0-9]+_type.counter") m3 := metric.New( @@ -295,7 +295,7 @@ func TestSendMetricsWithPatterns(t *testing.T) { "counter_fanNaN_type", map[string]string{}, map[string]interface{}{"value": float64(3.14), "counter": 5}, - time.Date(2010, time.November, 11, 23, 0, 0, 0, time.UTC), + time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), telegraf.Counter, ) From 8f2954f9c9b505b7d89af2fddd032d9b8b873ed5 Mon Sep 17 00:00:00 2001 From: Dhruv Thakkar Date: Thu, 25 Jul 2024 20:11:02 -0400 Subject: [PATCH 9/9] chore(outputs.dynatrace) cleanup test code --- plugins/outputs/dynatrace/dynatrace_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/outputs/dynatrace/dynatrace_test.go b/plugins/outputs/dynatrace/dynatrace_test.go index 0422fe8e27bd5..ad95553d90798 100644 --- a/plugins/outputs/dynatrace/dynatrace_test.go +++ b/plugins/outputs/dynatrace/dynatrace_test.go @@ -260,7 +260,9 @@ func TestSendMetricsWithPatterns(t *testing.T) { "simple_xyz_metric.value,dt.metrics.source=telegraf gauge,3.14 1289430000000", "simple_xyz_metric.counter,dt.metrics.source=telegraf count,delta=5 1289430000000", ) + // Add pattern to match all metrics that match simple_[a-z]+_metric.counter d.AddCounterMetricsPatterns = append(d.AddCounterMetricsPatterns, "simple_[a-z]+_metric.counter") + m1 := metric.New( "simple_abc_metric", map[string]string{},