From 7126e8a18c7f9d7e45c5d60112bba154cbc5564e Mon Sep 17 00:00:00 2001 From: esther kim Date: Fri, 13 Dec 2019 16:22:43 +0900 Subject: [PATCH 01/27] [Metricbeat] Add 'query' metricset for prometheus module --- metricbeat/include/list_common.go | 1 + metricbeat/module/prometheus/_meta/config.yml | 11 +++ metricbeat/module/prometheus/fields.go | 2 +- .../module/prometheus/query/_meta/data.json | 35 +++++++ .../prometheus/query/_meta/docs.asciidoc | 21 ++++ .../module/prometheus/query/_meta/fields.yml | 6 ++ .../query/_meta/testdata/config.yml | 4 + .../prometheus/query/_meta/testdata/docs.json | 15 +++ .../_meta/testdata/docs.json-expected.json | 32 +++++++ metricbeat/module/prometheus/query/config.go | 59 ++++++++++++ metricbeat/module/prometheus/query/data.go | 69 ++++++++++++++ metricbeat/module/prometheus/query/query.go | 95 +++++++++++++++++++ .../module/prometheus/query/query_test.go | 32 +++++++ 13 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 metricbeat/module/prometheus/query/_meta/data.json create mode 100644 metricbeat/module/prometheus/query/_meta/docs.asciidoc create mode 100644 metricbeat/module/prometheus/query/_meta/fields.yml create mode 100644 metricbeat/module/prometheus/query/_meta/testdata/config.yml create mode 100644 metricbeat/module/prometheus/query/_meta/testdata/docs.json create mode 100644 metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json create mode 100644 metricbeat/module/prometheus/query/config.go create mode 100644 metricbeat/module/prometheus/query/data.go create mode 100644 metricbeat/module/prometheus/query/query.go create mode 100644 metricbeat/module/prometheus/query/query_test.go diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 26db25d5744..9f35a611349 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -119,6 +119,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/postgresql/statement" _ "github.com/elastic/beats/metricbeat/module/prometheus" _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" + _ "github.com/elastic/beats/metricbeat/module/prometheus/query" _ "github.com/elastic/beats/metricbeat/module/rabbitmq" _ "github.com/elastic/beats/metricbeat/module/rabbitmq/connection" _ "github.com/elastic/beats/metricbeat/module/rabbitmq/exchange" diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index d322be32d5c..69ce03331ae 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -1,6 +1,7 @@ - module: prometheus period: 10s hosts: ["localhost:9090"] + metricsets: ["collect"] metrics_path: /metrics #username: "user" #password: "secret" @@ -9,3 +10,13 @@ # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: 'http_request' + path: '/api/v1/query' + fields: + 'query': sum(rate(prometheus_http_requests_total[1m])) \ No newline at end of file diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index 23d94ffe4c5..9b692d5a95d 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/prometheus. func AssetPrometheus() string { - return "eJyUkc1qwzAQhO9+ikG9hSQPoENfoYUeSwmKtba30R+7G0LeviQ2aQIttLppv9FoNNrgQGePJjWTTXTUDjC2RB7u9TZ0HRBJe+FmXIvHcwcAbxZMob2ERhGD1IyA71OgElvlYtsO0KmK7fpaBh49hpCUOkAoUVDyGMNFQ2ZcRvV4d6rJreEms+Y+OmBgSlH99d4nvEgkASs4tyoWimEioTVS2FNSnDgl5GD9hIFFbQ2bCEJqCEKI9bhPdPXaoIRM9w1sZ4/t6soBOzfyqPtP6m0ZzZvdTA50PlWJC/qhpsu6ayWTCfdL0t8yzKK/h7h70QPZ5dAal3GRuZX7Z84b2Tx81lcAAAD//3hXtDU=" + return "eJyUkk1u4zAMhfc+xYNmFyQ5gBZzhRmgy6IIFJu21eivJI3Aty9iO4lbJEDLnd+j+T5R2uFEo0XhHEl7GqQC1GsgC/P/JpoKaEhq9kV9ThZ/KwB4UacCqdkVatByjnC4/wVKTck+6b4CpM+shzqn1ncWrQtCFcAUyAlZdO7SQ6o+dWLxakSC2cL0qsW8VUDrKTRip9w/+McNMbzAx5JZXVL0xLRFcEcKgrMPAdFp3aP1LLqF9gQmUTgmNHk4Bppm7ZBcpPUG9vOM/WbyAR0LWeTjO9W6SPPHYXZONJ4zN4v1YE2XWm0lkrKvF9JnDHPTzyFWJ/riHKIrxaduaTMb80vOm7P7dllXdUb/GIjHm3rl7TgPZaU+Cb3UNGDJFNKV9yj2/h4+AwAA//8bo9bO" } diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json new file mode 100644 index 00000000000..5d1695bf64d --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2019-03-01T08:05:34.853Z", + "event": { + "dataset": "prometheus.query", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "prometheus": { + "query": { + "cpu_usage": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + "1576583842.761000", + "2880242405376" + ] + } + ] + } + } + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } +} \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc new file mode 100644 index 00000000000..48a37509356 --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -0,0 +1,21 @@ +This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries. + + +[float] +=== Configuration + +This module can query using HTTP API provided by Prometheus. +https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + metricsets: ["query"] + hosts: ["node:9100"] + paths: + - name: 'http_request' + path: '/api/v1/query' + fields: + 'query': sum(rate(prometheus_http_requests_total[1m])) +------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/fields.yml b/metricbeat/module/prometheus/query/_meta/fields.yml new file mode 100644 index 00000000000..acbc35db449 --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: query + type: group + description: > + query metricset + release: beta + fields: diff --git a/metricbeat/module/prometheus/query/_meta/testdata/config.yml b/metricbeat/module/prometheus/query/_meta/testdata/config.yml new file mode 100644 index 00000000000..da4129b45e5 --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/api/v1/query?query=" +module: + namespace: test \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json new file mode 100644 index 00000000000..d98624fbe58 --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/testdata/docs.json @@ -0,0 +1,15 @@ +{ + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1576583842.761000, + "2880242405376" + ] + } + ] + } +} \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json new file mode 100644 index 00000000000..5a44bfa336a --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json @@ -0,0 +1,32 @@ +[ + { + "event": { + "dataset": "prometheus.query.test", + "duration": 115000, + "module": "prometheus" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "prometheus": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": {}, + "value": [ + 1576217053.147, + "55293501.76000571" + ] + } + ] + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "prometheus" + } + } +] \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/config.go b/metricbeat/module/prometheus/query/config.go new file mode 100644 index 00000000000..dcec798202d --- /dev/null +++ b/metricbeat/module/prometheus/query/config.go @@ -0,0 +1,59 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package query + +import ( + "errors" + + "github.com/elastic/beats/libbeat/common" +) + +// Config for "query" metricset +type Config struct { + Paths []PathConfig `config:"paths"` + DefaultPath PathConfig `config:"default_path"` +} + +// PathConfig is used to make a API request. +type PathConfig struct { + Path string `config:"path"` + Fields common.MapStr `config:"fields"` + Name string `config:"name"` +} + +func defaultConfig() Config { + return Config{ + DefaultPath: PathConfig{ + Path: "/api/v1/query", + Name: "default", + }, + } +} + +// Validate for Prometheus "query" metricset config +func (p PathConfig) Validate() error { + if p.Name == "" { + return errors.New("`namespace` can not be empty in path configuration") + } + + if p.Path == "" { + return errors.New("`path` can not be empty in path configuration") + } + + return nil +} diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go new file mode 100644 index 00000000000..667b41211ec --- /dev/null +++ b/metricbeat/module/prometheus/query/data.go @@ -0,0 +1,69 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package query + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +// PromResponseBody for Prometheus Query API Request +type PromResponseBody struct { + Status string `json:"status"` + Data prometheusData `json:"data"` +} +type prometheusData struct { + ResultType string `json:"resultType"` + Results []result `json:"result"` +} +type result struct { + Metric interface{} `json:"metric"` + Vectors []interface{} `json:"value"` +} + +func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) mb.Event { + var event common.MapStr + var res PromResponseBody + if err := json.Unmarshal(body, &res); err != nil { + m.Logger().Error("Failed to parsing api response ", err) + } + + // Check if there is vector array. + // Vector [ , "" ] is not acceptable for Elasticsearch. + // Because there are two types in one array. + // So change Vector to [ " Date: Thu, 19 Dec 2019 19:17:27 +0900 Subject: [PATCH 02/27] refactor the output structure for Prometheus query API * Remove array from response body. Original response body has an array (Vector type). It makes difficult to query with Elasticsearch QL. New data schema: "prometheus": { "query": { "mem_usage": { "status": "success", "data": { "resultType": "vector", "result": [ { "metric": {}, "reconciledValue": { "unixtimestamp": 1.576751116531e+09, "value": "2947656593408" } } ] } } } } --- metricbeat/module/prometheus/query/data.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 667b41211ec..298cc78d659 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,7 +19,6 @@ package query import ( "encoding/json" - "fmt" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" @@ -35,8 +34,9 @@ type prometheusData struct { Results []result `json:"result"` } type result struct { - Metric interface{} `json:"metric"` - Vectors []interface{} `json:"value"` + Metric interface{} `json:"metric"` + Vectors []interface{} `json:"value,omitempty"` + ReconciledValue map[string]interface{} `json:"reconciledValue"` } func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) mb.Event { @@ -49,11 +49,15 @@ func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) mb.Event { // Check if there is vector array. // Vector [ , "" ] is not acceptable for Elasticsearch. // Because there are two types in one array. - // So change Vector to [ " Date: Sat, 21 Dec 2019 18:21:51 +0900 Subject: [PATCH 03/27] [Metricbeat] Support all expression query result formats * reference url: https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats --- metricbeat/module/prometheus/_meta/config.yml | 17 ++- .../module/prometheus/query/_meta/data.json | 26 ++-- .../prometheus/query/_meta/docs.asciidoc | 7 +- .../query/_meta/testdata/config.yml | 2 +- .../prometheus/query/_meta/testdata/docs.json | 15 ++- .../_meta/testdata/docs.json-expected.json | 24 ++-- metricbeat/module/prometheus/query/data.go | 116 +++++++++++++----- metricbeat/module/prometheus/query/query.go | 12 +- 8 files changed, 144 insertions(+), 75 deletions(-) diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 69ce03331ae..d77b311684f 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -1,7 +1,7 @@ - module: prometheus period: 10s hosts: ["localhost:9090"] - metricsets: ["collect"] + metricsets: ["collector"] metrics_path: /metrics #username: "user" #password: "secret" @@ -16,7 +16,18 @@ hosts: ["localhost:9090"] metricsets: ["query"] paths: - - name: 'http_request' + - name: 'instant_vector' path: '/api/v1/query' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) \ No newline at end of file + 'query': sum(rate(prometheus_http_requests_total[1m])) + - name: 'range_vector' + path: '/api/v1/query_range' + fields: + 'query': 'up' + 'start': '2019-12-20T00:00:00.000Z' + 'end': '2019-12-21T00:00:00.000Z' + 'step': 1h + - name: 'scalar' + path: '/api/v1/query' + fields: + 'query': '100' \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 5d1695bf64d..c35955d96c4 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -11,21 +11,17 @@ }, "prometheus": { "query": { - "cpu_usage": { - "status": "success", - "data": { - "resultType": "vector", - "result": [ - { - "metric": {}, - "value": [ - "1576583842.761000", - "2880242405376" - ] - } - ] - } - } + "labels": { + "job": "kubelet", + "namespace": "kube-system", + "node": "master01", + "service": "prometheus-prometheus-oper-kubelet", + "__name__": "up", + "endpoint": "https-metrics", + "instance": "192.168.48.204:10250" + }, + "dataType": "matrix", + "matrix_test": "1" } }, "service": { diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index 48a37509356..922b295a4ae 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -15,7 +15,10 @@ https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries hosts: ["node:9100"] paths: - name: 'http_request' - path: '/api/v1/query' + path: '/api/v1/query_range' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) + 'query': 'up{node="master01"}' + 'start': '2019-12-20T23:30:00.000Z' + 'end': '2019-12-21T00:00:00.000Z' + 'step': 1h ------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/config.yml b/metricbeat/module/prometheus/query/_meta/testdata/config.yml index da4129b45e5..d9c65c70960 100644 --- a/metricbeat/module/prometheus/query/_meta/testdata/config.yml +++ b/metricbeat/module/prometheus/query/_meta/testdata/config.yml @@ -1,4 +1,4 @@ type: http -url: "/api/v1/query?query=" +url: "/api/v1/query_range?query=" module: namespace: test \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json index d98624fbe58..cef0735d6a6 100644 --- a/metricbeat/module/prometheus/query/_meta/testdata/docs.json +++ b/metricbeat/module/prometheus/query/_meta/testdata/docs.json @@ -1,13 +1,20 @@ { "status": "success", "data": { - "resultType": "vector", + "resultType": "matrix", "result": [ { - "metric": {}, + "metric": { + "job": "kubelet", + "namespace": "kube-system", + "node": "master01", + "service": "prometheus-prometheus-oper-kubelet", + "__name__": "up", + "endpoint": "https-metrics", + "instance": "192.168.48.204:10250" + }, "value": [ - 1576583842.761000, - "2880242405376" + [1576884600, "1"] ] } ] diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json index 5a44bfa336a..8bd671e241a 100644 --- a/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json +++ b/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json @@ -10,18 +10,18 @@ "period": 10000 }, "prometheus": { - "status": "success", - "data": { - "resultType": "vector", - "result": [ - { - "metric": {}, - "value": [ - 1576217053.147, - "55293501.76000571" - ] - } - ] + "query": { + "labels": { + "job": "kubelet", + "namespace": "kube-system", + "node": "master01", + "service": "prometheus-prometheus-oper-kubelet", + "__name__": "up", + "endpoint": "https-metrics", + "instance": "192.168.48.204:10250" + }, + "dataType": "matrix", + "matrix_test": "1" } }, "service": { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 298cc78d659..8414e00b3f0 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,55 +19,103 @@ package query import ( "encoding/json" + "time" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" ) -// PromResponseBody for Prometheus Query API Request -type PromResponseBody struct { - Status string `json:"status"` - Data prometheusData `json:"data"` +// ArrayResponse is for "scalar", "string" type. +type ArrayResponse struct { + Status string `json:"status"` + Data arrayData `json:"data"` } -type prometheusData struct { - ResultType string `json:"resultType"` - Results []result `json:"result"` +type arrayData struct { + ResultType string `json:"resultType"` + Results []interface{} `json:"result"` } -type result struct { - Metric interface{} `json:"metric"` - Vectors []interface{} `json:"value,omitempty"` - ReconciledValue map[string]interface{} `json:"reconciledValue"` +// MapResponse is for "vector", "matrix" type from Prometheus Query API Request +type MapResponse struct { + Status string `json:"status"` + Data mapData `json:"data"` +} +type mapData struct { + ResultType string `json:"resultType"` + Results []mapResult `json:"result"` +} +type mapResult struct { + Metric map[string]string `json:"metric"` + Vector []interface{} `json:"value"` + Vectors [][]interface{} `json:"values"` } -func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) mb.Event { - var event common.MapStr - var res PromResponseBody - if err := json.Unmarshal(body, &res); err != nil { - m.Logger().Error("Failed to parsing api response ", err) +func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { + var events []mb.Event + converted, resultType, err := convertJSONToStruct(body) + if err != nil { + return events, err } - - // Check if there is vector array. - // Vector [ , "" ] is not acceptable for Elasticsearch. - // Because there are two types in one array. - // So change Vector to Object { unixtimestamp: " Date: Wed, 5 Feb 2020 18:35:03 +0900 Subject: [PATCH 04/27] [Metricbeat] convert numeric string to integer/float type * Prometheus API returns "string" type for query result. But actual result type is a number. * Make it easy to use ES SQL --- metricbeat/module/prometheus/query/data.go | 56 +++++++++++++--------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 8414e00b3f0..5f05efdeae4 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,35 +19,37 @@ package query import ( "encoding/json" + "strconv" "time" - "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" + "github.com/pkg/errors" ) // ArrayResponse is for "scalar", "string" type. type ArrayResponse struct { - Status string `json:"status"` - Data arrayData `json:"data"` + Status string `json:"status"` + Data arrayData `json:"data"` } type arrayData struct { - ResultType string `json:"resultType"` - Results []interface{} `json:"result"` + ResultType string `json:"resultType"` + Results []interface{} `json:"result"` } + // MapResponse is for "vector", "matrix" type from Prometheus Query API Request type MapResponse struct { - Status string `json:"status"` - Data mapData `json:"data"` + Status string `json:"status"` + Data mapData `json:"data"` } type mapData struct { - ResultType string `json:"resultType"` - Results []mapResult `json:"result"` + ResultType string `json:"resultType"` + Results []mapResult `json:"result"` } type mapResult struct { - Metric map[string]string `json:"metric"` - Vector []interface{} `json:"value"` - Vectors [][]interface{} `json:"values"` + Metric map[string]string `json:"metric"` + Vector []interface{} `json:"value"` + Vectors [][]interface{} `json:"values"` } func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { @@ -62,8 +64,8 @@ func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Even events = append(events, mb.Event{ Timestamp: getTimestamp(res.Data.Results[0].(float64)), MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.Name: res.Data.Results[1], + "dataType": resultType, + pathConfig.Name: convertToNumeric(res.Data.Results[1].(string)), }, }) case "vector": @@ -72,9 +74,9 @@ func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Even events = append(events, mb.Event{ Timestamp: getTimestamp(result.Vector[0].(float64)), MetricSetFields: common.MapStr{ - "labels": result.Metric, - "dataType": resultType, - pathConfig.Name: result.Vector[1], + "labels": result.Metric, + "dataType": resultType, + pathConfig.Name: convertToNumeric(result.Vector[1].(string)), }, }) } @@ -85,9 +87,9 @@ func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Even events = append(events, mb.Event{ Timestamp: getTimestamp(vector[0].(float64)), MetricSetFields: common.MapStr{ - "labels": result.Metric, - "dataType": resultType, - pathConfig.Name: vector[1], + "labels": result.Metric, + "dataType": resultType, + pathConfig.Name: convertToNumeric(vector[1].(string)), }, }) } @@ -104,7 +106,7 @@ func convertJSONToStruct(body []byte) (interface{}, string, error) { return nil, "", errors.Wrap(err, "Failed to parse api response") } - if arrayBody.Data.ResultType == "vector" || arrayBody.Data.ResultType == "matrix" { + if arrayBody.Data.ResultType == "vector" || arrayBody.Data.ResultType == "matrix" { mapBody := MapResponse{} if err := json.Unmarshal(body, &mapBody); err != nil { return nil, arrayBody.Data.ResultType, errors.Wrap(err, "Failed to parse api response") @@ -116,6 +118,16 @@ func convertJSONToStruct(body []byte) (interface{}, string, error) { func getTimestamp(num float64) time.Time { sec := int64(num) - ns := int64((num - float64(sec))*1000) + ns := int64((num - float64(sec)) * 1000) return time.Unix(sec, ns) } + +func convertToNumeric(str string) interface{} { + if res, err := strconv.Atoi(str); err == nil { + return res + } else if res, err := strconv.ParseFloat(str, 64); err == nil { + return res + } else { + return str + } +} From 55de06268742b7caf3b79377370f5cfbf7ea33e1 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 18 Mar 2020 14:20:46 +0200 Subject: [PATCH 05/27] wip Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 6 ++++ metricbeat/docs/modules/prometheus.asciidoc | 26 ++++++++++++++ .../docs/modules/prometheus/query.asciidoc | 23 +++++++++++++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list_common.go | 3 +- metricbeat/metricbeat.reference.yml | 22 ++++++++++++ metricbeat/module/prometheus/fields.go | 2 +- metricbeat/module/prometheus/query/config.go | 2 +- metricbeat/module/prometheus/query/data.go | 13 ++++--- metricbeat/module/prometheus/query/query.go | 34 ++++++++++++++----- metricbeat/modules.d/prometheus.yml.disabled | 22 ++++++++++++ 11 files changed, 139 insertions(+), 17 deletions(-) create mode 100644 metricbeat/docs/modules/prometheus/query.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 41b905dc8e8..d6bdf693f4b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -31595,6 +31595,12 @@ type: object -- +[float] +=== query + +query metricset + + [float] === remote_write diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 13065dff7c7..59ed27c17ad 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -45,6 +45,7 @@ metricbeat.modules: #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Metrics sent by a Prometheus server using remote_write option - module: prometheus metricsets: ["remote_write"] @@ -54,6 +55,27 @@ metricbeat.modules: # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: 'instant_vector' + path: '/api/v1/query' + fields: + 'query': sum(rate(prometheus_http_requests_total[1m])) + - name: 'range_vector' + path: '/api/v1/query_range' + fields: + 'query': 'up' + 'start': '2019-12-20T00:00:00.000Z' + 'end': '2019-12-21T00:00:00.000Z' + 'step': 1h + - name: 'scalar' + path: '/api/v1/query' + fields: + 'query': '100' ---- This module supports TLS connections when using `ssl` config field, as described in <>. @@ -66,9 +88,13 @@ The following metricsets are available: * <> +* <> + * <> include::prometheus/collector.asciidoc[] +include::prometheus/query.asciidoc[] + include::prometheus/remote_write.asciidoc[] diff --git a/metricbeat/docs/modules/prometheus/query.asciidoc b/metricbeat/docs/modules/prometheus/query.asciidoc new file mode 100644 index 00000000000..72c0fde9278 --- /dev/null +++ b/metricbeat/docs/modules/prometheus/query.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-prometheus-query]] +=== Prometheus query metricset + +beta[] + +include::../../../module/prometheus/query/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/prometheus/query/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2a8c8f76bc4..bcd8d27a147 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -205,7 +205,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.3+| .3+| |<> +|<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 2e73e3f3eda..f15d6a4be16 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -125,6 +125,7 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/postgresql/statement" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus/collector" + _ "github.com/elastic/beats/v7/metricbeat/module/prometheus/query" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus/remote_write" _ "github.com/elastic/beats/v7/metricbeat/module/rabbitmq" _ "github.com/elastic/beats/v7/metricbeat/module/rabbitmq/connection" @@ -169,4 +170,4 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/connection" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/mntr" _ "github.com/elastic/beats/v7/metricbeat/module/zookeeper/server" - ) +) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 1416df1b42f..f117c47e1e3 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -717,6 +717,7 @@ metricbeat.modules: #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Metrics sent by a Prometheus server using remote_write option - module: prometheus metricsets: ["remote_write"] @@ -727,6 +728,27 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: 'instant_vector' + path: '/api/v1/query' + fields: + 'query': sum(rate(prometheus_http_requests_total[1m])) + - name: 'range_vector' + path: '/api/v1/query_range' + fields: + 'query': 'up' + 'start': '2019-12-20T00:00:00.000Z' + 'end': '2019-12-21T00:00:00.000Z' + 'step': 1h + - name: 'scalar' + path: '/api/v1/query' + fields: + 'query': '100' + #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq metricsets: ["node", "queue", "connection"] diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index 01b4f85d156..779293db3f7 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/prometheus. func AssetPrometheus() string { - return "eJyUkk1u4zAMhfc+xYNmFyQ5gBZzhRmgy6IIFJu21eivJI3Aty9iO4lbJEDLnd+j+T5R2uFEo0XhHEl7GqQC1GsgC/P/JpoKaEhq9kV9ThZ/KwB4UacCqdkVatByjnC4/wVKTck+6b4CpM+shzqn1ncWrQtCFcAUyAlZdO7SQ6o+dWLxakSC2cL0qsW8VUDrKTRip9w/+McNMbzAx5JZXVL0xLRFcEcKgrMPAdFp3aP1LLqF9gQmUTgmNHk4Bppm7ZBcpPUG9vOM/WbyAR0LWeTjO9W6SPPHYXZONJ4zN4v1YE2XWm0lkrKvF9JnDHPTzyFWJ/riHKIrxaduaTMb80vOm7P7dllXdUb/GIjHm3rl7TgPZaU+Cb3UNGDJFNKV9yj2/h4+AwAA//8bo9bO" + return "eJykk82O00AQhO9+itJwW2X3AXzgFUDiiFA0scvxsPNHd4cob48Sx2GSRUILc6yarv66x37GK089qpREm3nQDrBgkT3c55voOmCkDhKqhZJ7fOwA4It5U+ggvnLEJCXB43cVmMdaQraXDtC5iG2Hkqew7zH5qOwAYaRX9tj78x2ahbzXHl+danQbuNmsum8dMAXGUftL32dkn/hAfT52qucsKYd6Vdqy8/mATzJSEBQh1SLms2GmcIPod4yKY4gRydswYwqitoHNhFANXoixHHaRt7wVZSl+eboZK0zZfedgjbwI28V95elYZGzsP6x5Pc1mE03CcO36BmZx30/zMNudu02+1pD316vuyf0jdEN7//r3M/w4UE5v+Nun/UvjS8C6CrYz39ruaL7RH7+VFUWYinF7lGD8H6IlB5ecFWz5a5olKeUn5R20vwIAAP//lMoVVQ==" } diff --git a/metricbeat/module/prometheus/query/config.go b/metricbeat/module/prometheus/query/config.go index dcec798202d..eda750b67de 100644 --- a/metricbeat/module/prometheus/query/config.go +++ b/metricbeat/module/prometheus/query/config.go @@ -20,7 +20,7 @@ package query import ( "errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common" ) // Config for "query" metricset diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 5f05efdeae4..c075ef36fdd 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,11 +19,12 @@ package query import ( "encoding/json" + "fmt" "strconv" "time" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/pkg/errors" ) @@ -52,7 +53,7 @@ type mapResult struct { Vectors [][]interface{} `json:"values"` } -func (m *MetricSet) parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { +func parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { var events []mb.Event converted, resultType, err := convertJSONToStruct(body) if err != nil { @@ -105,7 +106,11 @@ func convertJSONToStruct(body []byte) (interface{}, string, error) { if err := json.Unmarshal(body, &arrayBody); err != nil { return nil, "", errors.Wrap(err, "Failed to parse api response") } - + if arrayBody.Status == "error" { + return nil, "", errors.Errorf("Failed to query") + } + fmt.Println("here it is:") + fmt.Println(arrayBody) if arrayBody.Data.ResultType == "vector" || arrayBody.Data.ResultType == "matrix" { mapBody := MapResponse{} if err := json.Unmarshal(body, &mapBody); err != nil { diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index 24fc1748d97..a01dbb87004 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -18,16 +18,29 @@ package query import ( + "github.com/elastic/beats/v7/metricbeat/mb/parse" "io/ioutil" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/metricbeat/helper" - "github.com/elastic/beats/metricbeat/mb" + "fmt" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/pkg/errors" ) +const ( + defaultScheme = "http" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + }.Build() +) + func init() { - mb.Registry.MustAddMetricSet("prometheus", "query", New) + mb.Registry.MustAddMetricSet("prometheus", "query", New, + mb.WithHostParser(hostParser), + ) } // MetricSet type defines all fields of the MetricSet for Prometheus Query @@ -64,10 +77,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, pathConfig := range m.paths { url := m.getURL(pathConfig.Path, pathConfig.Fields) + fmt.Println(url) m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { - return err + return errors.Wrap(err, "unable to fetch data from prometheus endpoint") } defer func() { if err := response.Body.Close(); err != nil { @@ -80,9 +94,11 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { return err } - events, parseErr := m.parseResponse(body, pathConfig) + events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { - return err + m.Logger().Debug("error parsing response for ", pathConfig.Name, ": ", parseErr) + reporter.Error(errors.Wrap(err, "error mapping channel to its schema")) + continue } for _, e := range events { if reported := reporter.Event(e); !reported { @@ -95,5 +111,5 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { func (m *MetricSet) getURL(path string, queryMap common.MapStr) string { queryStr := mb.QueryParams(queryMap).String() - return "http://" + m.BaseMetricSet.Host() + path + "?" + queryStr + return m.http.GetURI() + path + "?" + queryStr } diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 065c7df407e..a4179076ccc 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -18,6 +18,7 @@ #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Metrics sent by a Prometheus server using remote_write option - module: prometheus metricsets: ["remote_write"] @@ -27,3 +28,24 @@ # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: 'instant_vector' + path: '/api/v1/query' + fields: + 'query': sum(rate(prometheus_http_requests_total[1m])) + - name: 'range_vector' + path: '/api/v1/query_range' + fields: + 'query': 'up' + 'start': '2019-12-20T00:00:00.000Z' + 'end': '2019-12-21T00:00:00.000Z' + 'step': 1h + - name: 'scalar' + path: '/api/v1/query' + fields: + 'query': '100' From 5bddf268d5de9758c24858c42440c4eeb00df3a5 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 18 Mar 2020 16:06:30 +0200 Subject: [PATCH 06/27] Fix fields, docs, config Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 10 ++++ metricbeat/docs/modules/prometheus.asciidoc | 20 ++++---- metricbeat/metricbeat.reference.yml | 20 ++++---- metricbeat/module/prometheus/_meta/config.yml | 20 ++++---- metricbeat/module/prometheus/_meta/fields.yml | 6 +++ metricbeat/module/prometheus/fields.go | 2 +- .../prometheus/query/_meta/docs.asciidoc | 50 +++++++++++++++---- metricbeat/modules.d/prometheus.yml.disabled | 20 ++++---- 8 files changed, 97 insertions(+), 51 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d6bdf693f4b..80821196824 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -31591,6 +31591,16 @@ type: object Prometheus metric +type: object + +-- + +*`prometheus.query.*`*:: ++ +-- +Prometheus value resulted from PromQL + + type: object -- diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 59ed27c17ad..4eaff8019a3 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -64,18 +64,18 @@ metricbeat.modules: - name: 'instant_vector' path: '/api/v1/query' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) - - name: 'range_vector' - path: '/api/v1/query_range' + query: "sum(rate(prometheus_http_requests_total[1m]))" + - name: "range_vector" + path: "/api/v1/query_range" fields: - 'query': 'up' - 'start': '2019-12-20T00:00:00.000Z' - 'end': '2019-12-21T00:00:00.000Z' - 'step': 1h - - name: 'scalar' - path: '/api/v1/query' + query: "up" + start: "2019-12-20T00:00:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h + - name: "scalar" + path: "/api/v1/query" fields: - 'query': '100' + query: "100" ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f117c47e1e3..77c43f35d49 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -736,18 +736,18 @@ metricbeat.modules: - name: 'instant_vector' path: '/api/v1/query' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) - - name: 'range_vector' - path: '/api/v1/query_range' + query: "sum(rate(prometheus_http_requests_total[1m]))" + - name: "range_vector" + path: "/api/v1/query_range" fields: - 'query': 'up' - 'start': '2019-12-20T00:00:00.000Z' - 'end': '2019-12-21T00:00:00.000Z' - 'step': 1h - - name: 'scalar' - path: '/api/v1/query' + query: "up" + start: "2019-12-20T00:00:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h + - name: "scalar" + path: "/api/v1/query" fields: - 'query': '100' + query: "100" #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 43b77c02328..1dc5d2ecaf0 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -34,15 +34,15 @@ - name: 'instant_vector' path: '/api/v1/query' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) - - name: 'range_vector' - path: '/api/v1/query_range' + query: "sum(rate(prometheus_http_requests_total[1m]))" + - name: "range_vector" + path: "/api/v1/query_range" fields: - 'query': 'up' - 'start': '2019-12-20T00:00:00.000Z' - 'end': '2019-12-21T00:00:00.000Z' - 'step': 1h - - name: 'scalar' - path: '/api/v1/query' + query: "up" + start: "2019-12-20T00:00:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h + - name: "scalar" + path: "/api/v1/query" fields: - 'query': '100' + query: "100" diff --git a/metricbeat/module/prometheus/_meta/fields.yml b/metricbeat/module/prometheus/_meta/fields.yml index ceea6b49dd2..eaeb1070386 100644 --- a/metricbeat/module/prometheus/_meta/fields.yml +++ b/metricbeat/module/prometheus/_meta/fields.yml @@ -21,3 +21,9 @@ object_type_mapping_type: "*" description: > Prometheus metric + - name: query.* + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus value resulted from PromQL diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index 779293db3f7..d8b0eb230f4 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/prometheus. func AssetPrometheus() string { - return "eJykk82O00AQhO9+itJwW2X3AXzgFUDiiFA0scvxsPNHd4cob48Sx2GSRUILc6yarv66x37GK089qpREm3nQDrBgkT3c55voOmCkDhKqhZJ7fOwA4It5U+ggvnLEJCXB43cVmMdaQraXDtC5iG2Hkqew7zH5qOwAYaRX9tj78x2ahbzXHl+danQbuNmsum8dMAXGUftL32dkn/hAfT52qucsKYd6Vdqy8/mATzJSEBQh1SLms2GmcIPod4yKY4gRydswYwqitoHNhFANXoixHHaRt7wVZSl+eboZK0zZfedgjbwI28V95elYZGzsP6x5Pc1mE03CcO36BmZx30/zMNudu02+1pD316vuyf0jdEN7//r3M/w4UE5v+Nun/UvjS8C6CrYz39ruaL7RH7+VFUWYinF7lGD8H6IlB5ecFWz5a5olKeUn5R20vwIAAP//lMoVVQ==" + return "eJzMkkGO2zAMRfc+xYe7G2TmAF70BAXaosuiCBT7O1ZHllSSniC3LxzHGU0yQJF2Uy75RfLxi4945rFBljTSBk5aAeYtsEH95ZKsK6CjtuKz+RQbfKwA4Js5U2grLrNDL2mEw2sVGLucfLSnCtAhiW3bFHu/b9C7oKwAYaBTNti7+Q3NfNxrg++1aqg3qAezXP+ogN4zdNqc5j4iupFX1HPYMc+9JE35nCnL5viAz9JR4BV+zEnMRcNA4QbB7RgUBx8CRmftgN6L2gY2EEI1OCG6NO0CL/1WlKX46eEirDBp95OtFeklsV3UZx4PSbpCfsfmNQpnR5r49jz1BmZR76e52u2Nuh1dzj7uz0/rh/ovoW9of02U4//G+uLCdPr1Kdh627P89VPB//Z639nqZqfyNP8Ac2qwfiVLHy5jdzRX5K9vfUURjsm4PYg3/gvR0genPivYqzNn45TyQrmD9ncAAAD//1baTA8=" } diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index 922b295a4ae..afbbdf6b842 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -1,11 +1,41 @@ -This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries. +This is the `query` metricset to query from querring API[https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries] of Promtheus. [float] === Configuration -This module can query using HTTP API provided by Prometheus. -https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries +[float] +==== Instant queries + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: "sum_rate_http_requests_total" + path: "/api/v1/query" + fields: + query: "sum(rate(prometheus_http_requests_total[1m]))" +------------------------------------------------------------------------------------- + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + paths: + - name: 'up' + path: '/api/v1/query' + fields: + query: "up" +------------------------------------------------------------------------------------- + + +[float] +==== Range queries [source,yaml] ------------------------------------------------------------------------------------- @@ -14,11 +44,11 @@ https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries metricsets: ["query"] hosts: ["node:9100"] paths: - - name: 'http_request' - path: '/api/v1/query_range' + - name: "up_master" + path: "/api/v1/query_range" fields: - 'query': 'up{node="master01"}' - 'start': '2019-12-20T23:30:00.000Z' - 'end': '2019-12-21T00:00:00.000Z' - 'step': 1h -------------------------------------------------------------------------------------- \ No newline at end of file + query: "up{node="master01"}" + start: "2019-12-20T23:30:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h +------------------------------------------------------------------------------------- diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index a4179076ccc..1cb646c3aee 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -37,15 +37,15 @@ - name: 'instant_vector' path: '/api/v1/query' fields: - 'query': sum(rate(prometheus_http_requests_total[1m])) - - name: 'range_vector' - path: '/api/v1/query_range' + query: "sum(rate(prometheus_http_requests_total[1m]))" + - name: "range_vector" + path: "/api/v1/query_range" fields: - 'query': 'up' - 'start': '2019-12-20T00:00:00.000Z' - 'end': '2019-12-21T00:00:00.000Z' - 'step': 1h - - name: 'scalar' - path: '/api/v1/query' + query: "up" + start: "2019-12-20T00:00:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h + - name: "scalar" + path: "/api/v1/query" fields: - 'query': '100' + query: "100" From e27ba4ad7a8392366f2c7f55720a58c0f75f58c9 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 18 Mar 2020 16:28:17 +0200 Subject: [PATCH 07/27] refactor and cleanups Signed-off-by: chrismark --- .../query/_meta/testdata/config.yml | 4 --- .../prometheus/query/_meta/testdata/docs.json | 22 ------------- .../_meta/testdata/docs.json-expected.json | 32 ------------------- metricbeat/module/prometheus/query/config.go | 24 +++++++------- metricbeat/module/prometheus/query/data.go | 21 ++++++------ metricbeat/module/prometheus/query/query.go | 32 +++++++++++-------- 6 files changed, 41 insertions(+), 94 deletions(-) delete mode 100644 metricbeat/module/prometheus/query/_meta/testdata/config.yml delete mode 100644 metricbeat/module/prometheus/query/_meta/testdata/docs.json delete mode 100644 metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json diff --git a/metricbeat/module/prometheus/query/_meta/testdata/config.yml b/metricbeat/module/prometheus/query/_meta/testdata/config.yml deleted file mode 100644 index d9c65c70960..00000000000 --- a/metricbeat/module/prometheus/query/_meta/testdata/config.yml +++ /dev/null @@ -1,4 +0,0 @@ -type: http -url: "/api/v1/query_range?query=" -module: - namespace: test \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json deleted file mode 100644 index cef0735d6a6..00000000000 --- a/metricbeat/module/prometheus/query/_meta/testdata/docs.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "status": "success", - "data": { - "resultType": "matrix", - "result": [ - { - "metric": { - "job": "kubelet", - "namespace": "kube-system", - "node": "master01", - "service": "prometheus-prometheus-oper-kubelet", - "__name__": "up", - "endpoint": "https-metrics", - "instance": "192.168.48.204:10250" - }, - "value": [ - [1576884600, "1"] - ] - } - ] - } -} \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json b/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json deleted file mode 100644 index 8bd671e241a..00000000000 --- a/metricbeat/module/prometheus/query/_meta/testdata/docs.json-expected.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "event": { - "dataset": "prometheus.query.test", - "duration": 115000, - "module": "prometheus" - }, - "metricset": { - "name": "query", - "period": 10000 - }, - "prometheus": { - "query": { - "labels": { - "job": "kubelet", - "namespace": "kube-system", - "node": "master01", - "service": "prometheus-prometheus-oper-kubelet", - "__name__": "up", - "endpoint": "https-metrics", - "instance": "192.168.48.204:10250" - }, - "dataType": "matrix", - "matrix_test": "1" - } - }, - "service": { - "address": "127.0.0.1:55555", - "type": "prometheus" - } - } -] \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/config.go b/metricbeat/module/prometheus/query/config.go index eda750b67de..825990007c9 100644 --- a/metricbeat/module/prometheus/query/config.go +++ b/metricbeat/module/prometheus/query/config.go @@ -25,30 +25,30 @@ import ( // Config for "query" metricset type Config struct { - Paths []PathConfig `config:"paths"` - DefaultPath PathConfig `config:"default_path"` + Queries []QueryConfig `config:"queries"` + DefaultQuery QueryConfig `config:"default_path"` } // PathConfig is used to make a API request. -type PathConfig struct { - Path string `config:"path"` - Fields common.MapStr `config:"fields"` - Name string `config:"name"` +type QueryConfig struct { + Path string `config:"path"` + QueryParams common.MapStr `config:"query_params"` + QueryName string `config:"query_name"` } func defaultConfig() Config { return Config{ - DefaultPath: PathConfig{ - Path: "/api/v1/query", - Name: "default", + DefaultQuery: QueryConfig{ + Path: "/api/v1/query", + QueryName: "default", }, } } // Validate for Prometheus "query" metricset config -func (p PathConfig) Validate() error { - if p.Name == "" { - return errors.New("`namespace` can not be empty in path configuration") +func (p QueryConfig) Validate() error { + if p.QueryName == "" { + return errors.New("`query_name` can not be empty in path configuration") } if p.Path == "" { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index c075ef36fdd..aa0f7546972 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -23,9 +23,10 @@ import ( "strconv" "time" + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/pkg/errors" ) // ArrayResponse is for "scalar", "string" type. @@ -53,7 +54,7 @@ type mapResult struct { Vectors [][]interface{} `json:"values"` } -func parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { +func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { var events []mb.Event converted, resultType, err := convertJSONToStruct(body) if err != nil { @@ -65,8 +66,8 @@ func parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { events = append(events, mb.Event{ Timestamp: getTimestamp(res.Data.Results[0].(float64)), MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.Name: convertToNumeric(res.Data.Results[1].(string)), + "dataType": resultType, + pathConfig.QueryName: convertToNumeric(res.Data.Results[1].(string)), }, }) case "vector": @@ -75,9 +76,9 @@ func parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { events = append(events, mb.Event{ Timestamp: getTimestamp(result.Vector[0].(float64)), MetricSetFields: common.MapStr{ - "labels": result.Metric, - "dataType": resultType, - pathConfig.Name: convertToNumeric(result.Vector[1].(string)), + "labels": result.Metric, + "dataType": resultType, + pathConfig.QueryName: convertToNumeric(result.Vector[1].(string)), }, }) } @@ -88,9 +89,9 @@ func parseResponse(body []byte, pathConfig PathConfig) ([]mb.Event, error) { events = append(events, mb.Event{ Timestamp: getTimestamp(vector[0].(float64)), MetricSetFields: common.MapStr{ - "labels": result.Metric, - "dataType": resultType, - pathConfig.Name: convertToNumeric(vector[1].(string)), + "labels": result.Metric, + "dataType": resultType, + pathConfig.QueryName: convertToNumeric(vector[1].(string)), }, }) } diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index a01dbb87004..e7c4b44ecb2 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -18,13 +18,15 @@ package query import ( - "github.com/elastic/beats/v7/metricbeat/mb/parse" - "io/ioutil" "fmt" + "io/ioutil" + + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/pkg/errors" + "github.com/elastic/beats/v7/metricbeat/mb/parse" ) const ( @@ -46,8 +48,8 @@ func init() { // MetricSet type defines all fields of the MetricSet for Prometheus Query type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP - paths []PathConfig + http *helper.HTTP + queries []QueryConfig } // New create a new instance of the MetricSet @@ -67,7 +69,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, http: http, - paths: config.Paths, + queries: config.Queries, }, nil } @@ -75,13 +77,16 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { - for _, pathConfig := range m.paths { - url := m.getURL(pathConfig.Path, pathConfig.Fields) + for _, pathConfig := range m.queries { + url := m.getURL(pathConfig.Path, pathConfig.QueryParams) fmt.Println(url) m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { - return errors.Wrap(err, "unable to fetch data from prometheus endpoint") + msg := fmt.Sprintf("unable to fetch data from prometheus endpoint: %v", pathConfig.Path) + m.Logger().Debug(msg, err) + reporter.Error(errors.Wrap(err, msg)) + continue } defer func() { if err := response.Body.Close(); err != nil { @@ -96,14 +101,13 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { - m.Logger().Debug("error parsing response for ", pathConfig.Name, ": ", parseErr) - reporter.Error(errors.Wrap(err, "error mapping channel to its schema")) + msg := fmt.Sprintf("error parsing response for: %v", pathConfig.QueryName) + m.Logger().Debug(msg, err) + reporter.Error(errors.Wrap(err, msg)) continue } for _, e := range events { - if reported := reporter.Event(e); !reported { - m.Logger().Debug(errors.Errorf("error reporting event: %#v", e)) - } + reporter.Event(e) } } return nil From 8d2de92a5dc4831342669a5f1a9090e696de55a1 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 18 Mar 2020 17:51:35 +0200 Subject: [PATCH 08/27] Add unit&system tests Signed-off-by: chrismark --- metricbeat/docs/modules/prometheus.asciidoc | 16 ++-- metricbeat/metricbeat.reference.yml | 16 ++-- metricbeat/module/prometheus/_meta/config.yml | 16 ++-- .../module/prometheus/query/_meta/data.json | 16 ++-- .../prometheus/query/_meta/docs.asciidoc | 18 ++-- .../query/_meta/test/querymetrics.json | 24 ++++++ metricbeat/module/prometheus/query/data.go | 4 +- metricbeat/module/prometheus/query/query.go | 2 - .../query/query_integration_test.go | 82 +++++++++++++++++++ .../module/prometheus/query/query_test.go | 45 ++++++++-- .../module/prometheus/test_prometheus.py | 26 ++++++ metricbeat/modules.d/prometheus.yml.disabled | 16 ++-- 12 files changed, 218 insertions(+), 63 deletions(-) create mode 100644 metricbeat/module/prometheus/query/_meta/test/querymetrics.json create mode 100644 metricbeat/module/prometheus/query/query_integration_test.go diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 4eaff8019a3..d69b8b1e138 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -60,21 +60,21 @@ metricbeat.modules: period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: 'instant_vector' - path: '/api/v1/query' - fields: + queries: + - query_name: "instant_vector" + path: "/api/v1/query" + query_params: query: "sum(rate(prometheus_http_requests_total[1m]))" - - name: "range_vector" + - query_name: "range_vector" path: "/api/v1/query_range" - fields: + query_params: query: "up" start: "2019-12-20T00:00:00.000Z" end: "2019-12-21T00:00:00.000Z" step: 1h - - name: "scalar" + - query_name: "scalar" path: "/api/v1/query" - fields: + query_params: query: "100" ---- diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 77c43f35d49..23767f9fa59 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -732,21 +732,21 @@ metricbeat.modules: period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: 'instant_vector' - path: '/api/v1/query' - fields: + queries: + - query_name: "instant_vector" + path: "/api/v1/query" + query_params: query: "sum(rate(prometheus_http_requests_total[1m]))" - - name: "range_vector" + - query_name: "range_vector" path: "/api/v1/query_range" - fields: + query_params: query: "up" start: "2019-12-20T00:00:00.000Z" end: "2019-12-21T00:00:00.000Z" step: 1h - - name: "scalar" + - query_name: "scalar" path: "/api/v1/query" - fields: + query_params: query: "100" #------------------------------- RabbitMQ Module ------------------------------- diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 1dc5d2ecaf0..6d0cff76266 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -30,19 +30,19 @@ period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: 'instant_vector' - path: '/api/v1/query' - fields: + queries: + - query_name: "instant_vector" + path: "/api/v1/query" + query_params: query: "sum(rate(prometheus_http_requests_total[1m]))" - - name: "range_vector" + - query_name: "range_vector" path: "/api/v1/query_range" - fields: + query_params: query: "up" start: "2019-12-20T00:00:00.000Z" end: "2019-12-21T00:00:00.000Z" step: 1h - - name: "scalar" + - query_name: "scalar" path: "/api/v1/query" - fields: + query_params: query: "100" diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index c35955d96c4..69df85c57f4 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -1,5 +1,5 @@ { - "@timestamp": "2019-03-01T08:05:34.853Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "event": { "dataset": "prometheus.query", "duration": 115000, @@ -11,21 +11,17 @@ }, "prometheus": { "query": { + "dataType": "vector", "labels": { - "job": "kubelet", - "namespace": "kube-system", - "node": "master01", - "service": "prometheus-prometheus-oper-kubelet", "__name__": "up", - "endpoint": "https-metrics", - "instance": "192.168.48.204:10250" + "instance": "localhost:9090", + "job": "prometheus" }, - "dataType": "matrix", - "matrix_test": "1" + "up": 1 } }, "service": { - "address": "127.0.0.1:55555", + "address": "localhost:32768", "type": "prometheus" } } \ No newline at end of file diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index afbbdf6b842..b7294274c01 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -13,10 +13,10 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: "sum_rate_http_requests_total" + queries: + - query_name: "sum_rate_http_requests_total" path: "/api/v1/query" - fields: + query_params: query: "sum(rate(prometheus_http_requests_total[1m]))" ------------------------------------------------------------------------------------- @@ -26,10 +26,10 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: 'up' + queries: + - query_name: 'up' path: '/api/v1/query' - fields: + query_params: query: "up" ------------------------------------------------------------------------------------- @@ -43,10 +43,10 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d period: 10s metricsets: ["query"] hosts: ["node:9100"] - paths: - - name: "up_master" + queries: + - query_name: "up_master" path: "/api/v1/query_range" - fields: + query_params: query: "up{node="master01"}" start: "2019-12-20T23:30:00.000Z" end: "2019-12-21T00:00:00.000Z" diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics.json new file mode 100644 index 00000000000..2646c453dfe --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics.json @@ -0,0 +1,24 @@ +{ + "status" : "success", + "data" : { + "resultType" : "vector", + "result" : [ + { + "metric" : { + "__name__" : "up", + "job" : "prometheus", + "instance" : "localhost:9090" + }, + "value": [ 1435781451.781, "1" ] + }, + { + "metric" : { + "__name__" : "up", + "job" : "node", + "instance" : "localhost:9100" + }, + "value" : [ 1435781451.781, "0" ] + } + ] + } +} diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index aa0f7546972..323e3aaf85c 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,7 +19,6 @@ package query import ( "encoding/json" - "fmt" "strconv" "time" @@ -110,8 +109,7 @@ func convertJSONToStruct(body []byte) (interface{}, string, error) { if arrayBody.Status == "error" { return nil, "", errors.Errorf("Failed to query") } - fmt.Println("here it is:") - fmt.Println(arrayBody) + if arrayBody.Data.ResultType == "vector" || arrayBody.Data.ResultType == "matrix" { mapBody := MapResponse{} if err := json.Unmarshal(body, &mapBody); err != nil { diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index e7c4b44ecb2..caff2f6f7c5 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -65,7 +65,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } - return &MetricSet{ BaseMetricSet: base, http: http, @@ -79,7 +78,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, pathConfig := range m.queries { url := m.getURL(pathConfig.Path, pathConfig.QueryParams) - fmt.Println(url) m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go new file mode 100644 index 00000000000..54c4ff37928 --- /dev/null +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -0,0 +1,82 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package query + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "prometheus") + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{service.Host()}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + ms := mbtest.NewReportingMetricSetV2Error(t, config) + err := mbtest.WriteEventsReporterV2Error(ms, t, "") + if err == nil { + return + } + t.Fatal("write", err) +} + +func TestQueryFetch(t *testing.T) { + service := compose.EnsureUp(t, "prometheus") + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{service.Host()}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + f := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 errors, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + event := events[0].MetricSetFields + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} diff --git a/metricbeat/module/prometheus/query/query_test.go b/metricbeat/module/prometheus/query/query_test.go index 947f0bbf085..b9530361f7a 100644 --- a/metricbeat/module/prometheus/query/query_test.go +++ b/metricbeat/module/prometheus/query/query_test.go @@ -15,18 +15,49 @@ // specific language governing permissions and limitations // under the License. -// +build !integration - package query import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - _ "github.com/elastic/beats/metricbeat/module/http" + "github.com/elastic/beats/v7/libbeat/common" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - mbtest.TestDataFiles(t, "prometheus", "query") +func TestQueryFetchEventContent(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test/") + + response, _ := ioutil.ReadFile(absPath + "/querymetrics.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{server.URL}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } diff --git a/metricbeat/module/prometheus/test_prometheus.py b/metricbeat/module/prometheus/test_prometheus.py index 1d119b0b7b3..26a2981eb33 100644 --- a/metricbeat/module/prometheus/test_prometheus.py +++ b/metricbeat/module/prometheus/test_prometheus.py @@ -37,6 +37,32 @@ def test_stats(self): self.assert_fields_are_documented(evt) + # @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + # def test_query(self): + # """ + # prometheus query test + # """ + # self.render_config_template(modules=[{ + # "name": "prometheus", + # "metricsets": ["query"], + # "hosts": self.get_hosts(), + # "period": "5s", + # 'queries': [{'path': '/api/v1/query', + # 'query_name': 'instant_vector', + # 'query_params': {'query': 'up'}}] + # }]) + # proc = self.start_beat() + # self.wait_until(lambda: self.output_lines() > 0) + # proc.check_kill_and_wait() + # self.assert_no_logged_warnings() + # + # output = self.read_output_json() + # evt = output[0] + # + # self.assertCountEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt) + # + # self.assert_fields_are_documented(evt) + class TestRemoteWrite(metricbeat.BaseTest): diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 1cb646c3aee..7b074309913 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -33,19 +33,19 @@ period: 10s hosts: ["localhost:9090"] metricsets: ["query"] - paths: - - name: 'instant_vector' - path: '/api/v1/query' - fields: + queries: + - query_name: "instant_vector" + path: "/api/v1/query" + query_params: query: "sum(rate(prometheus_http_requests_total[1m]))" - - name: "range_vector" + - query_name: "range_vector" path: "/api/v1/query_range" - fields: + query_params: query: "up" start: "2019-12-20T00:00:00.000Z" end: "2019-12-21T00:00:00.000Z" step: 1h - - name: "scalar" + - query_name: "scalar" path: "/api/v1/query" - fields: + query_params: query: "100" From c87af0fd65a3ee11d84d71be896e84117d9a4334 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 18 Mar 2020 18:54:14 +0200 Subject: [PATCH 09/27] Add system tests Signed-off-by: chrismark --- metricbeat/docs/modules/prometheus.asciidoc | 1 + metricbeat/metricbeat.reference.yml | 1 + metricbeat/module/prometheus/_meta/config.yml | 1 + .../module/prometheus/query/_meta/data.json | 12 ++--- metricbeat/module/prometheus/query/data.go | 8 +-- metricbeat/module/prometheus/query/query.go | 8 +-- .../query/query_integration_test.go | 8 +-- .../module/prometheus/test_prometheus.py | 54 ++++++++++--------- metricbeat/modules.d/prometheus.yml.disabled | 1 + 9 files changed, 52 insertions(+), 42 deletions(-) diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index d69b8b1e138..f19e9b6572e 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -56,6 +56,7 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + - module: prometheus period: 10s hosts: ["localhost:9090"] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 23767f9fa59..651f2b00a4a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -728,6 +728,7 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + - module: prometheus period: 10s hosts: ["localhost:9090"] diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 6d0cff76266..00ceef9094c 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -26,6 +26,7 @@ #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + - module: prometheus period: 10s hosts: ["localhost:9090"] diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 69df85c57f4..3d4a31914f3 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -10,14 +10,14 @@ "period": 10000 }, "prometheus": { + "labels": { + "__name__": "go_threads", + "instance": "localhost:9090", + "job": "prometheus" + }, "query": { "dataType": "vector", - "labels": { - "__name__": "up", - "instance": "localhost:9090", - "job": "prometheus" - }, - "up": 1 + "go_threads": 24 } }, "service": { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 323e3aaf85c..4e946a49d4f 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -73,9 +73,9 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { res := converted.(MapResponse) for _, result := range res.Data.Results { events = append(events, mb.Event{ - Timestamp: getTimestamp(result.Vector[0].(float64)), + Timestamp: getTimestamp(result.Vector[0].(float64)), + ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ - "labels": result.Metric, "dataType": resultType, pathConfig.QueryName: convertToNumeric(result.Vector[1].(string)), }, @@ -86,9 +86,9 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { for _, result := range res.Data.Results { for _, vector := range result.Vectors { events = append(events, mb.Event{ - Timestamp: getTimestamp(vector[0].(float64)), + Timestamp: getTimestamp(vector[0].(float64)), + ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ - "labels": result.Metric, "dataType": resultType, pathConfig.QueryName: convertToNumeric(vector[1].(string)), }, diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index caff2f6f7c5..db05c260e11 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -50,6 +50,7 @@ type MetricSet struct { mb.BaseMetricSet http *helper.HTTP queries []QueryConfig + baseURL string } // New create a new instance of the MetricSet @@ -69,6 +70,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, http: http, queries: config.Queries, + baseURL: http.GetURI(), }, nil } @@ -82,7 +84,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { response, err := m.http.FetchResponse() if err != nil { msg := fmt.Sprintf("unable to fetch data from prometheus endpoint: %v", pathConfig.Path) - m.Logger().Debug(msg, err) + m.Logger().Debugf("%v: %v", msg, err) reporter.Error(errors.Wrap(err, msg)) continue } @@ -100,7 +102,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { msg := fmt.Sprintf("error parsing response for: %v", pathConfig.QueryName) - m.Logger().Debug(msg, err) + m.Logger().Debugf("%v: %v", msg, err) reporter.Error(errors.Wrap(err, msg)) continue } @@ -113,5 +115,5 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { func (m *MetricSet) getURL(path string, queryMap common.MapStr) string { queryStr := mb.QueryParams(queryMap).String() - return m.http.GetURI() + path + "?" + queryStr + return m.baseURL + path + "?" + queryStr } diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 54c4ff37928..0f01f016903 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -38,10 +38,10 @@ func TestData(t *testing.T) { "hosts": []string{service.Host()}, "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", + "query_name": "go_threads", "path": "/api/v1/query", "query_params": common.MapStr{ - "query": "up", + "query": "go_threads", }, }, }, @@ -63,10 +63,10 @@ func TestQueryFetch(t *testing.T) { "hosts": []string{service.Host()}, "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", + "query_name": "go_threads", "path": "/api/v1/query", "query_params": common.MapStr{ - "query": "up", + "query": "go_threads", }, }, }, diff --git a/metricbeat/module/prometheus/test_prometheus.py b/metricbeat/module/prometheus/test_prometheus.py index 26a2981eb33..8b5376a7c04 100644 --- a/metricbeat/module/prometheus/test_prometheus.py +++ b/metricbeat/module/prometheus/test_prometheus.py @@ -37,31 +37,35 @@ def test_stats(self): self.assert_fields_are_documented(evt) - # @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - # def test_query(self): - # """ - # prometheus query test - # """ - # self.render_config_template(modules=[{ - # "name": "prometheus", - # "metricsets": ["query"], - # "hosts": self.get_hosts(), - # "period": "5s", - # 'queries': [{'path': '/api/v1/query', - # 'query_name': 'instant_vector', - # 'query_params': {'query': 'up'}}] - # }]) - # proc = self.start_beat() - # self.wait_until(lambda: self.output_lines() > 0) - # proc.check_kill_and_wait() - # self.assert_no_logged_warnings() - # - # output = self.read_output_json() - # evt = output[0] - # - # self.assertCountEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt) - # - # self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_query(self): + """ + prometheus query test + """ + self.render_config_template(modules=[{ + "name": "prometheus", + "metricsets": ["query"], + "hosts": self.get_hosts(), + "period": "5s", + "extras": { + "queries": [{ + "path": "/api/v1/query", + 'query_name': 'instant_vector', + 'query_params': {'query': 'go_threads'} + }] + } + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + evt = output[0] + + self.assertCountEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) class TestRemoteWrite(metricbeat.BaseTest): diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 7b074309913..a6d414e3ffc 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -29,6 +29,7 @@ #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + - module: prometheus period: 10s hosts: ["localhost:9090"] From ccc1e54598b7767d29389f180ce14df9c4f6b335 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 11:17:01 +0200 Subject: [PATCH 10/27] Add changelog and fixes Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + .../prometheus/query/_meta/docs.asciidoc | 28 ++++++++++++------- metricbeat/module/prometheus/query/config.go | 6 ++-- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a2cf0514d0e..7c2b7bd61c9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -234,6 +234,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add PubSub metricset to Google Cloud Platform module {pull}15536[15536] - Add Prometheus remote write endpoint {pull}16609[16609] - Release STAN module as GA. {pull}16980[16980] +- Add query metricset for prometheus module. {pull}17104[17104] *Packetbeat* diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index b7294274c01..d6342c3bedc 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -1,4 +1,4 @@ -This is the `query` metricset to query from querring API[https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries] of Promtheus. +This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Promtheus]. [float] @@ -7,6 +7,7 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d [float] ==== Instant queries +The following configuration performs an instant query for `up` metric at a single point in time: [source,yaml] ------------------------------------------------------------------------------------- - module: prometheus @@ -14,12 +15,15 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "sum_rate_http_requests_total" - path: "/api/v1/query" + - query_name: 'up' + path: '/api/v1/query' query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" + query: "up" ------------------------------------------------------------------------------------- + +More complex PromQL expressions can also be used like the following one which calculates the per-second rate of HTTP +requests as measured over the last 5 minutes. [source,yaml] ------------------------------------------------------------------------------------- - module: prometheus @@ -27,16 +31,20 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: 'up' - path: '/api/v1/query' + - query_name: "rate_http_requests_total" + path: "/api/v1/query" query_params: - query: "up" + query: "rate(prometheus_http_requests_total[5m])" ------------------------------------------------------------------------------------- + + [float] ==== Range queries + +The following example evaluates the expression `up` over a 30-second range with a query resolution of 15 seconds: [source,yaml] ------------------------------------------------------------------------------------- - module: prometheus @@ -48,7 +56,7 @@ This is the `query` metricset to query from querring API[https://prometheus.io/d path: "/api/v1/query_range" query_params: query: "up{node="master01"}" - start: "2019-12-20T23:30:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h + start: "2019-12-20T23:30:30.000Z" + end: "2019-12-21T23:31:00.000Z" + step: 15s ------------------------------------------------------------------------------------- diff --git a/metricbeat/module/prometheus/query/config.go b/metricbeat/module/prometheus/query/config.go index 825990007c9..c5cfbe3828e 100644 --- a/metricbeat/module/prometheus/query/config.go +++ b/metricbeat/module/prometheus/query/config.go @@ -23,13 +23,13 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -// Config for "query" metricset +// Config defines the "query" metricset's configuration type Config struct { Queries []QueryConfig `config:"queries"` DefaultQuery QueryConfig `config:"default_path"` } -// PathConfig is used to make a API request. +// QueryConfig is used to make an API request. type QueryConfig struct { Path string `config:"path"` QueryParams common.MapStr `config:"query_params"` @@ -46,7 +46,7 @@ func defaultConfig() Config { } // Validate for Prometheus "query" metricset config -func (p QueryConfig) Validate() error { +func (p *QueryConfig) Validate() error { if p.QueryName == "" { return errors.New("`query_name` can not be empty in path configuration") } From 9e414dcecae8777c7148a96fde587a2445a82860 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 11:39:04 +0200 Subject: [PATCH 11/27] make update Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 8c9cd35d3c2..7251ef4f835 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1009,6 +1009,7 @@ metricbeat.modules: #ssl.certificate_authorities: # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + # Metrics sent by a Prometheus server using remote_write option - module: prometheus metricsets: ["remote_write"] @@ -1019,6 +1020,28 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" + +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metricsets: ["query"] + queries: + - query_name: "instant_vector" + path: "/api/v1/query" + query_params: + query: "sum(rate(prometheus_http_requests_total[1m]))" + - query_name: "range_vector" + path: "/api/v1/query_range" + query_params: + query: "up" + start: "2019-12-20T00:00:00.000Z" + end: "2019-12-21T00:00:00.000Z" + step: 1h + - query_name: "scalar" + path: "/api/v1/query" + query_params: + query: "100" + #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq metricsets: ["node", "queue", "connection"] From 5568fd4cb8a9d90b4a0c1c1d44b50fcf3bdeedbc Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 12:53:00 +0200 Subject: [PATCH 12/27] review fixes Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 92 ++++++++++++------- metricbeat/module/prometheus/query/query.go | 13 ++- .../query/query_integration_test.go | 31 +++++-- 3 files changed, 87 insertions(+), 49 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 4e946a49d4f..5fd95fb2405 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -55,44 +55,68 @@ type mapResult struct { func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { var events []mb.Event - converted, resultType, err := convertJSONToStruct(body) + var resultType string + var convertedMap MapResponse + + // try to convert to array response + convertedArray, err := convertJSONToArrayResponse(body) if err != nil { return events, err } + resultType = convertedArray.Data.ResultType + + // check if it is a vector or matrix and unmarshal more + if resultType == "vector" || resultType == "matrix" { + convertedMap, err = convertJSONToMapResponse(body) + if err != nil { + return events, err + } + resultType = convertedMap.Data.ResultType + } + switch resultType { case "scalar", "string": - res := converted.(ArrayResponse) - events = append(events, mb.Event{ - Timestamp: getTimestamp(res.Data.Results[0].(float64)), - MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.QueryName: convertToNumeric(res.Data.Results[1].(string)), - }, - }) - case "vector": - res := converted.(MapResponse) - for _, result := range res.Data.Results { + if convertedArray.Data.Results != nil { events = append(events, mb.Event{ - Timestamp: getTimestamp(result.Vector[0].(float64)), - ModuleFields: common.MapStr{"labels": result.Metric}, + Timestamp: getTimestamp(convertedArray.Data.Results[0].(float64)), MetricSetFields: common.MapStr{ "dataType": resultType, - pathConfig.QueryName: convertToNumeric(result.Vector[1].(string)), + pathConfig.QueryName: attemptConvertToNumeric(convertedArray.Data.Results[1].(string)), }, }) + } else { + return events, errors.New("Could not retrieve results") } - case "matrix": - res := converted.(MapResponse) - for _, result := range res.Data.Results { - for _, vector := range result.Vectors { + case "vector": + for _, result := range convertedMap.Data.Results { + if result.Vector != nil { events = append(events, mb.Event{ - Timestamp: getTimestamp(vector[0].(float64)), + Timestamp: getTimestamp(result.Vector[0].(float64)), ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ "dataType": resultType, - pathConfig.QueryName: convertToNumeric(vector[1].(string)), + pathConfig.QueryName: attemptConvertToNumeric(result.Vector[1].(string)), }, }) + } else { + return events, errors.New("Could not retrieve results") + } + } + case "matrix": + for _, result := range convertedMap.Data.Results { + for _, vector := range result.Vectors { + if vector != nil { + events = append(events, mb.Event{ + Timestamp: getTimestamp(vector[0].(float64)), + ModuleFields: common.MapStr{"labels": result.Metric}, + MetricSetFields: common.MapStr{ + "dataType": resultType, + pathConfig.QueryName: attemptConvertToNumeric(vector[1].(string)), + }, + }) + } else { + return events, errors.New("Could not retrieve results") + } } } default: @@ -101,23 +125,23 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { return events, nil } -func convertJSONToStruct(body []byte) (interface{}, string, error) { +func convertJSONToMapResponse(body []byte) (MapResponse, error) { + mapBody := MapResponse{} + if err := json.Unmarshal(body, &mapBody); err != nil { + return MapResponse{}, errors.Wrap(err, "Failed to parse api response") + } + return mapBody, nil +} + +func convertJSONToArrayResponse(body []byte) (ArrayResponse, error) { arrayBody := ArrayResponse{} if err := json.Unmarshal(body, &arrayBody); err != nil { - return nil, "", errors.Wrap(err, "Failed to parse api response") + return arrayBody, errors.Wrap(err, "Failed to parse api response") } if arrayBody.Status == "error" { - return nil, "", errors.Errorf("Failed to query") - } - - if arrayBody.Data.ResultType == "vector" || arrayBody.Data.ResultType == "matrix" { - mapBody := MapResponse{} - if err := json.Unmarshal(body, &mapBody); err != nil { - return nil, arrayBody.Data.ResultType, errors.Wrap(err, "Failed to parse api response") - } - return mapBody, mapBody.Data.ResultType, nil + return arrayBody, errors.Errorf("Failed to query") } - return arrayBody, arrayBody.Data.ResultType, nil + return arrayBody, nil } func getTimestamp(num float64) time.Time { @@ -126,7 +150,7 @@ func getTimestamp(num float64) time.Time { return time.Unix(sec, ns) } -func convertToNumeric(str string) interface{} { +func attemptConvertToNumeric(str string) interface{} { if res, err := strconv.Atoi(str); err == nil { return res } else if res, err := strconv.ParseFloat(str, 64); err == nil { diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index db05c260e11..390db235afa 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -18,7 +18,6 @@ package query import ( - "fmt" "io/ioutil" "github.com/pkg/errors" @@ -83,9 +82,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { - msg := fmt.Sprintf("unable to fetch data from prometheus endpoint: %v", pathConfig.Path) - m.Logger().Debugf("%v: %v", msg, err) - reporter.Error(errors.Wrap(err, msg)) + err = errors.Wrapf(err, "unable to fetch data from prometheus endpoint: %v", pathConfig.Path) + m.Logger().Debug(err) + reporter.Error(err) continue } defer func() { @@ -101,9 +100,9 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { - msg := fmt.Sprintf("error parsing response for: %v", pathConfig.QueryName) - m.Logger().Debugf("%v: %v", msg, err) - reporter.Error(errors.Wrap(err, msg)) + parseErr = errors.Wrapf(parseErr, "error parsing response for: %v", pathConfig.QueryName) + m.Logger().Debug(parseErr) + reporter.Error(parseErr) continue } for _, e := range events { diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 0f01f016903..997bb0ac9b0 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -21,12 +21,14 @@ package query import ( "testing" + "time" + + "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/tests/compose" + "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) func TestData(t *testing.T) { @@ -47,9 +49,13 @@ func TestData(t *testing.T) { }, } ms := mbtest.NewReportingMetricSetV2Error(t, config) - err := mbtest.WriteEventsReporterV2Error(ms, t, "") - if err == nil { - return + var err error + for retries := 0; retries < 3; retries++ { + err = mbtest.WriteEventsReporterV2Error(ms, t, "") + if err == nil { + return + } + time.Sleep(300 * time.Millisecond) } t.Fatal("write", err) } @@ -72,9 +78,18 @@ func TestQueryFetch(t *testing.T) { }, } f := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 errors, had %d. %v\n", len(errs), errs) + + var events []mb.Event + var errors []error + for retries := 0; retries < 3; retries++ { + events, errors = mbtest.ReportingFetchV2Error(f) + if len(events) > 0 { + break + } + time.Sleep(300 * time.Millisecond) + } + if len(errors) > 0 { + t.Fatalf("Expected 0 errors, had %d. %v\n", len(errors), errors) } assert.NotEmpty(t, events) event := events[0].MetricSetFields From 532baaedb33a0afb669cb4e98e507c9784b512ad Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 16:26:08 +0200 Subject: [PATCH 13/27] refactor code Signed-off-by: chrismark --- .../_meta/test/querymetrics_range_vector.json | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json new file mode 100644 index 00000000000..2646c453dfe --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json @@ -0,0 +1,24 @@ +{ + "status" : "success", + "data" : { + "resultType" : "vector", + "result" : [ + { + "metric" : { + "__name__" : "up", + "job" : "prometheus", + "instance" : "localhost:9090" + }, + "value": [ 1435781451.781, "1" ] + }, + { + "metric" : { + "__name__" : "up", + "job" : "node", + "instance" : "localhost:9100" + }, + "value" : [ 1435781451.781, "0" ] + } + ] + } +} From a007fa74a3f2682dcabfa3b5a85c1db8d49051b6 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 16:31:51 +0200 Subject: [PATCH 14/27] Add unit tests Signed-off-by: chrismark --- ....json => querymetrics_instant_vector.json} | 0 .../_meta/test/querymetrics_range_vector.json | 16 +- metricbeat/module/prometheus/query/data.go | 165 +++++++++++++----- metricbeat/module/prometheus/query/query.go | 8 +- .../module/prometheus/query/query_test.go | 70 +++++++- 5 files changed, 198 insertions(+), 61 deletions(-) rename metricbeat/module/prometheus/query/_meta/test/{querymetrics.json => querymetrics_instant_vector.json} (100%) diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json similarity index 100% rename from metricbeat/module/prometheus/query/_meta/test/querymetrics.json rename to metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json index 2646c453dfe..7256a92c5f7 100644 --- a/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json @@ -1,7 +1,7 @@ { "status" : "success", "data" : { - "resultType" : "vector", + "resultType" : "matrix", "result" : [ { "metric" : { @@ -9,15 +9,23 @@ "job" : "prometheus", "instance" : "localhost:9090" }, - "value": [ 1435781451.781, "1" ] + "values" : [ + [ 1435781430.781, "1" ], + [ 1435781445.781, "1" ], + [ 1435781460.781, "1" ] + ] }, { "metric" : { "__name__" : "up", "job" : "node", - "instance" : "localhost:9100" + "instance" : "localhost:9091" }, - "value" : [ 1435781451.781, "0" ] + "values" : [ + [ 1435781430.781, "0" ], + [ 1435781445.781, "0" ], + [ 1435781460.781, "1" ] + ] } ] } diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 5fd95fb2405..2a402ad4fb5 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -19,6 +19,7 @@ package query import ( "encoding/json" + "fmt" "strconv" "time" @@ -28,6 +29,14 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) +// Response stores the very basic response information to only keep the Status and the ResultType. +type Response struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + } `json:"data"` +} + // ArrayResponse is for "scalar", "string" type. type ArrayResponse struct { Status string `json:"status"` @@ -55,81 +64,143 @@ type mapResult struct { func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { var events []mb.Event - var resultType string - var convertedMap MapResponse - // try to convert to array response - convertedArray, err := convertJSONToArrayResponse(body) + resultType, err := getResultType(body) if err != nil { return events, err } - resultType = convertedArray.Data.ResultType - // check if it is a vector or matrix and unmarshal more - if resultType == "vector" || resultType == "matrix" { - convertedMap, err = convertJSONToMapResponse(body) + switch resultType { + case "scalar", "string": + event, err := getEventFromScalarOrString(body, resultType, pathConfig.QueryName) if err != nil { return events, err } - resultType = convertedMap.Data.ResultType + events = append(events, event) + case "vector": + evnts, err := getEventsFromVector(body, resultType, pathConfig.QueryName) + if err != nil { + return events, err + } + events = append(events, evnts...) + case "matrix": + evnts, err := getEventsFromMatrix(body, resultType, pathConfig.QueryName) + if err != nil { + return events, err + } + events = append(events, evnts...) + default: + msg := fmt.Sprintf("Unknown resultType '%v'", resultType) + return events, errors.New(msg) } + return events, nil +} - switch resultType { - case "scalar", "string": - if convertedArray.Data.Results != nil { - events = append(events, mb.Event{ - Timestamp: getTimestamp(convertedArray.Data.Results[0].(float64)), - MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.QueryName: attemptConvertToNumeric(convertedArray.Data.Results[1].(string)), - }, - }) - } else { - return events, errors.New("Could not retrieve results") - } - case "vector": - for _, result := range convertedMap.Data.Results { - if result.Vector != nil { +func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb.Event, error) { + events := []mb.Event{} + convertedMap, err := convertJSONToMapResponse(body) + if err != nil { + return events, err + } + results := convertedMap.Data.Results + for _, result := range results { + for _, vector := range result.Vectors { + if vector != nil { + if len(vector) != 2 { + return []mb.Event{}, errors.New("Could not parse results") + } + timestamp, ok := vector[0].(float64) + if !ok { + return []mb.Event{}, errors.New("Could not parse timestamp of result") + } events = append(events, mb.Event{ - Timestamp: getTimestamp(result.Vector[0].(float64)), - ModuleFields: common.MapStr{"labels": result.Metric}, + Timestamp: getTimestamp(timestamp), MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.QueryName: attemptConvertToNumeric(result.Vector[1].(string)), + "dataType": resultType, + queryName: attemptConvertToNumeric(vector[1].(string)), }, }) } else { - return events, errors.New("Could not retrieve results") + return []mb.Event{}, errors.New("Could not parse results") } } - case "matrix": - for _, result := range convertedMap.Data.Results { - for _, vector := range result.Vectors { - if vector != nil { - events = append(events, mb.Event{ - Timestamp: getTimestamp(vector[0].(float64)), - ModuleFields: common.MapStr{"labels": result.Metric}, - MetricSetFields: common.MapStr{ - "dataType": resultType, - pathConfig.QueryName: attemptConvertToNumeric(vector[1].(string)), - }, - }) - } else { - return events, errors.New("Could not retrieve results") - } + } + return events, nil +} + +func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb.Event, error) { + events := []mb.Event{} + convertedMap, err := convertJSONToMapResponse(body) + if err != nil { + return events, err + } + results := convertedMap.Data.Results + for _, result := range results { + if result.Vector != nil { + if len(result.Vector) != 2 { + return []mb.Event{}, errors.New("Could not parse results") + } + timestamp, ok := result.Vector[0].(float64) + if !ok { + return []mb.Event{}, errors.New("Could not parse timestamp of result") } + events = append(events, mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: attemptConvertToNumeric(result.Vector[1].(string)), + }, + }) + } else { + return []mb.Event{}, errors.New("Could not parse results") } - default: - return events, errors.New("Unknown resultType " + resultType) } return events, nil } +func getEventFromScalarOrString(body []byte, resultType string, queryName string) (mb.Event, error) { + convertedArray, err := convertJSONToArrayResponse(body) + if err != nil { + return mb.Event{}, err + } + if convertedArray.Data.Results != nil { + if len(convertedArray.Data.Results) != 2 { + return mb.Event{}, errors.New("Could not parse results") + } + timestamp, ok := convertedArray.Data.Results[0].(float64) + if !ok { + return mb.Event{}, errors.New("Could not parse timestamp of result") + } + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: attemptConvertToNumeric(convertedArray.Data.Results[1].(string)), + }, + }, nil + } + return mb.Event{}, errors.New("Could not parse results") +} + +func getResultType(body []byte) (string, error) { + response := Response{} + if err := json.Unmarshal(body, &response); err != nil { + return "", errors.Wrap(err, "Failed to parse api response") + } + if response.Status == "error" { + return "", errors.Errorf("Failed to query") + } + return response.Data.ResultType, nil +} + func convertJSONToMapResponse(body []byte) (MapResponse, error) { mapBody := MapResponse{} if err := json.Unmarshal(body, &mapBody); err != nil { return MapResponse{}, errors.Wrap(err, "Failed to parse api response") } + if mapBody.Status == "error" { + return mapBody, errors.Errorf("Failed to query") + } return mapBody, nil } diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index 390db235afa..29d3bb82529 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -82,9 +82,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { - err = errors.Wrapf(err, "unable to fetch data from prometheus endpoint: %v", pathConfig.Path) - m.Logger().Debug(err) - reporter.Error(err) + reporter.Error(errors.Wrapf(err, "unable to fetch data from prometheus endpoint: %v", pathConfig.Path)) continue } defer func() { @@ -100,9 +98,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { - parseErr = errors.Wrapf(parseErr, "error parsing response for: %v", pathConfig.QueryName) - m.Logger().Debug(parseErr) - reporter.Error(parseErr) + reporter.Error(errors.Wrapf(parseErr, "error parsing response for: %v", pathConfig.QueryName)) continue } for _, e := range events { diff --git a/metricbeat/module/prometheus/query/query_test.go b/metricbeat/module/prometheus/query/query_test.go index b9530361f7a..6d194248578 100644 --- a/metricbeat/module/prometheus/query/query_test.go +++ b/metricbeat/module/prometheus/query/query_test.go @@ -28,10 +28,18 @@ import ( mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) -func TestQueryFetchEventContent(t *testing.T) { +func TestQueryFetchEventContentInstantVector(t *testing.T) { absPath, _ := filepath.Abs("./_meta/test/") - response, _ := ioutil.ReadFile(absPath + "/querymetrics.json") + // test with response format like: + //[ + // { + // "metric": { "": "", ... }, + // "value": [ , "" ] + // }, + // ... + //] + response, _ := ioutil.ReadFile(absPath + "/querymetrics_instant_vector.json") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) w.Header().Set("Content-Type", "application/json;") @@ -58,6 +66,60 @@ func TestQueryFetchEventContent(t *testing.T) { metricSet := mbtest.NewReportingMetricSetV2Error(t, config) metricSet.Fetch(reporter) - e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) - t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) + events := reporter.GetEvents() + if len(events) != 2 { + t.Fatalf("Expected 2 events, had %d. %v\n", len(events), events) + } + for _, event := range events { + e := mbtest.StandardizeEvent(metricSet, event) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) + } +} + +func TestQueryFetchEventContentRangeVector(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test/") + + // test with response format like: + //[ + // { + // "metric": { "": "", ... }, + // "values": [ [ , "" ], ... ] + // }, + // ... + //] + response, _ := ioutil.ReadFile(absPath + "/querymetrics_range_vector.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{server.URL}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + events := reporter.GetEvents() + if len(events) != 6 { + t.Fatalf("Expected 6 events, had %d. %v\n", len(events), events) + } + for _, event := range events { + e := mbtest.StandardizeEvent(metricSet, event) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) + } } From f6b5b6db316cd6aba8629a549eeadaa0e416dc6f Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 16:39:33 +0200 Subject: [PATCH 15/27] Add more unit tests to cover scalar and string Signed-off-by: chrismark --- .../query/_meta/test/querymetrics_scalar.json | 10 +++ .../query/_meta/test/querymetrics_string.json | 10 +++ .../module/prometheus/query/query_test.go | 84 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json create mode 100644 metricbeat/module/prometheus/query/_meta/test/querymetrics_string.json diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json new file mode 100644 index 00000000000..4b03385de9d --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json @@ -0,0 +1,10 @@ +{ + "status":"success", + "data":{ + "resultType":"string", + "result":[ + 1584628364.185, + "100" + ] + } +} diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_string.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_string.json new file mode 100644 index 00000000000..9dc16d2a17f --- /dev/null +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_string.json @@ -0,0 +1,10 @@ +{ + "status":"success", + "data":{ + "resultType":"string", + "result":[ + 1584628642.569, + "apis" + ] + } +} diff --git a/metricbeat/module/prometheus/query/query_test.go b/metricbeat/module/prometheus/query/query_test.go index 6d194248578..5b99226eefc 100644 --- a/metricbeat/module/prometheus/query/query_test.go +++ b/metricbeat/module/prometheus/query/query_test.go @@ -123,3 +123,87 @@ func TestQueryFetchEventContentRangeVector(t *testing.T) { t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } } + +func TestQueryFetchEventContentScalar(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test/") + + // test with response format like: + //[ , "" ] + response, _ := ioutil.ReadFile(absPath + "/querymetrics_scalar.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{server.URL}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + events := reporter.GetEvents() + if len(events) != 1 { + t.Fatalf("Expected 1 events, had %d. %v\n", len(events), events) + } + for _, event := range events { + e := mbtest.StandardizeEvent(metricSet, event) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) + } +} + +func TestQueryFetchEventContentString(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test/") + + // test with response format like: + //[ , "" ] + response, _ := ioutil.ReadFile(absPath + "/querymetrics_string.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "prometheus", + "metricsets": []string{"query"}, + "hosts": []string{server.URL}, + "queries": []common.MapStr{ + common.MapStr{ + "query_name": "up", + "path": "/api/v1/query", + "query_params": common.MapStr{ + "query": "up", + }, + }, + }, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + events := reporter.GetEvents() + if len(events) != 1 { + t.Fatalf("Expected 1 events, had %d. %v\n", len(events), events) + } + for _, event := range events { + e := mbtest.StandardizeEvent(metricSet, event) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) + } +} From c60017ddf142c550df68c93ccd5be0f9f393a65e Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 17:06:19 +0200 Subject: [PATCH 16/27] Split range and instant vector responses into 2 different structs Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 83 ++++++++++++++++------ 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 2a402ad4fb5..3090e88f90a 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -38,6 +38,7 @@ type Response struct { } // ArrayResponse is for "scalar", "string" type. +// example: {"status":"success","data":{"resultType":"string","result":[1584628642.569,"100"]}} type ArrayResponse struct { Status string `json:"status"` Data arrayData `json:"data"` @@ -47,18 +48,47 @@ type arrayData struct { Results []interface{} `json:"result"` } -// MapResponse is for "vector", "matrix" type from Prometheus Query API Request -type MapResponse struct { - Status string `json:"status"` - Data mapData `json:"data"` +// InstantVectorResponse is for "vector" type from Prometheus Query API Request +// Format: +// [ +// { +// "metric": { "": "", ... }, +// "value": [ , "" ] +// }, +// ... +//] +type InstantVectorResponse struct { + Status string `json:"status"` + Data instantVectorData `json:"data"` } -type mapData struct { - ResultType string `json:"resultType"` - Results []mapResult `json:"result"` +type instantVectorData struct { + ResultType string `json:"resultType"` + Results []instantVectorResult `json:"result"` } -type mapResult struct { +type instantVectorResult struct { + Metric map[string]string `json:"metric"` + Vector []interface{} `json:"value"` +} + +// InstantVectorResponse is for "vector" type from Prometheus Query API Request +// Format: +// [ +// { +// "metric": { "": "", ... }, +// "values": [ [ , "" ], ... ] +// }, +// ... +//] +type RangeVectorResponse struct { + Status string `json:"status"` + Data rangeVectorData `json:"data"` +} +type rangeVectorData struct { + ResultType string `json:"resultType"` + Results []rangeVectorResult `json:"result"` +} +type rangeVectorResult struct { Metric map[string]string `json:"metric"` - Vector []interface{} `json:"value"` Vectors [][]interface{} `json:"values"` } @@ -98,7 +128,7 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb.Event, error) { events := []mb.Event{} - convertedMap, err := convertJSONToMapResponse(body) + convertedMap, err := convertJSONToRangeVectorResponse(body) if err != nil { return events, err } @@ -130,7 +160,7 @@ func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb.Event, error) { events := []mb.Event{} - convertedMap, err := convertJSONToMapResponse(body) + convertedMap, err := convertJSONToInstantVectorResponse(body) if err != nil { return events, err } @@ -193,10 +223,21 @@ func getResultType(body []byte) (string, error) { return response.Data.ResultType, nil } -func convertJSONToMapResponse(body []byte) (MapResponse, error) { - mapBody := MapResponse{} +func convertJSONToArrayResponse(body []byte) (ArrayResponse, error) { + arrayBody := ArrayResponse{} + if err := json.Unmarshal(body, &arrayBody); err != nil { + return arrayBody, errors.Wrap(err, "Failed to parse api response") + } + if arrayBody.Status == "error" { + return arrayBody, errors.Errorf("Failed to query") + } + return arrayBody, nil +} + +func convertJSONToRangeVectorResponse(body []byte) (RangeVectorResponse, error) { + mapBody := RangeVectorResponse{} if err := json.Unmarshal(body, &mapBody); err != nil { - return MapResponse{}, errors.Wrap(err, "Failed to parse api response") + return RangeVectorResponse{}, errors.Wrap(err, "Failed to parse api response") } if mapBody.Status == "error" { return mapBody, errors.Errorf("Failed to query") @@ -204,15 +245,15 @@ func convertJSONToMapResponse(body []byte) (MapResponse, error) { return mapBody, nil } -func convertJSONToArrayResponse(body []byte) (ArrayResponse, error) { - arrayBody := ArrayResponse{} - if err := json.Unmarshal(body, &arrayBody); err != nil { - return arrayBody, errors.Wrap(err, "Failed to parse api response") +func convertJSONToInstantVectorResponse(body []byte) (InstantVectorResponse, error) { + mapBody := InstantVectorResponse{} + if err := json.Unmarshal(body, &mapBody); err != nil { + return InstantVectorResponse{}, errors.Wrap(err, "Failed to parse api response") } - if arrayBody.Status == "error" { - return arrayBody, errors.Errorf("Failed to query") + if mapBody.Status == "error" { + return mapBody, errors.Errorf("Failed to query") } - return arrayBody, nil + return mapBody, nil } func getTimestamp(num float64) time.Time { From 0615a35d407b1b4c2b885d667d02ceb96ce2952f Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 17:18:17 +0200 Subject: [PATCH 17/27] Make use of scalar and string info when converting their types Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 44 +++++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 3090e88f90a..d17ac91f5f3 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -141,7 +141,8 @@ func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb } timestamp, ok := vector[0].(float64) if !ok { - return []mb.Event{}, errors.New("Could not parse timestamp of result") + msg := fmt.Sprintf("Could not parse timestamp of result: %v", vector) + return []mb.Event{}, errors.New(msg) } events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), @@ -172,7 +173,8 @@ func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb } timestamp, ok := result.Vector[0].(float64) if !ok { - return []mb.Event{}, errors.New("Could not parse timestamp of result") + msg := fmt.Sprintf("Could not parse timestamp of result: %v", result.Vector) + return []mb.Event{}, errors.New(msg) } events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), @@ -199,15 +201,37 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string } timestamp, ok := convertedArray.Data.Results[0].(float64) if !ok { - return mb.Event{}, errors.New("Could not parse timestamp of result") + msg := fmt.Sprintf("Could not parse timestamp of result: %v", convertedArray.Data.Results) + return mb.Event{}, errors.New(msg) } - return mb.Event{ - Timestamp: getTimestamp(timestamp), - MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: attemptConvertToNumeric(convertedArray.Data.Results[1].(string)), - }, - }, nil + value, ok := convertedArray.Data.Results[1].(string) + if !ok { + msg := fmt.Sprintf("Could not parse value of result: %v", convertedArray.Data.Results) + return mb.Event{}, errors.New(msg) + } + if resultType == "scalar" { + val, err := strconv.ParseFloat(value, 64) + if err != nil { + msg := fmt.Sprintf("Could not parse 'scalar' value of result: %v", convertedArray.Data.Results) + return mb.Event{}, errors.New(msg) + } + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: val, + }, + }, nil + } else if resultType == "string" { + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: value, + }, + }, nil + } + } return mb.Event{}, errors.New("Could not parse results") } From 4a3f35c956fc1653a41355a2fd9dabce655d34c5 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 17:35:42 +0200 Subject: [PATCH 18/27] Convert types explicitely according the dataType Signed-off-by: chrismark --- .../test/querymetrics_instant_vector.json | 4 +- .../query/_meta/test/querymetrics_scalar.json | 4 +- metricbeat/module/prometheus/query/data.go | 65 +++++++++++++------ 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json index 2646c453dfe..1ffdee97232 100644 --- a/metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json @@ -9,7 +9,7 @@ "job" : "prometheus", "instance" : "localhost:9090" }, - "value": [ 1435781451.781, "1" ] + "value": [ 1435781451.781, "1.0" ] }, { "metric" : { @@ -17,7 +17,7 @@ "job" : "node", "instance" : "localhost:9100" }, - "value" : [ 1435781451.781, "0" ] + "value" : [ 1435781451.781, "1.19" ] } ] } diff --git a/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json b/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json index 4b03385de9d..c7255fe9455 100644 --- a/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json +++ b/metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json @@ -1,10 +1,10 @@ { "status":"success", "data":{ - "resultType":"string", + "resultType":"scalar", "result":[ 1584628364.185, - "100" + "100.4" ] } } diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index d17ac91f5f3..ea1c59cf3b5 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -20,6 +20,7 @@ package query import ( "encoding/json" "fmt" + "math" "strconv" "time" @@ -144,11 +145,24 @@ func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb msg := fmt.Sprintf("Could not parse timestamp of result: %v", vector) return []mb.Event{}, errors.New(msg) } + value, ok := vector[1].(string) + if !ok { + msg := fmt.Sprintf("Could not parse value of result: %v", vector) + return []mb.Event{}, errors.New(msg) + } + val, err := strconv.ParseFloat(value, 64) + if err != nil { + msg := fmt.Sprintf("Could not parse 'range vector' value of result: %v", vector) + return []mb.Event{}, errors.New(msg) + } + if math.IsNaN(val) || math.IsInf(val, 0) { + continue + } events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), MetricSetFields: common.MapStr{ "dataType": resultType, - queryName: attemptConvertToNumeric(vector[1].(string)), + queryName: val, }, }) } else { @@ -176,11 +190,24 @@ func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb msg := fmt.Sprintf("Could not parse timestamp of result: %v", result.Vector) return []mb.Event{}, errors.New(msg) } + value, ok := result.Vector[1].(string) + if !ok { + msg := fmt.Sprintf("Could not parse value of result: %v", result.Vector) + return []mb.Event{}, errors.New(msg) + } + val, err := strconv.ParseFloat(value, 64) + if err != nil { + msg := fmt.Sprintf("Could not parse 'instant vector' value of result: %v", result.Vector) + return []mb.Event{}, errors.New(msg) + } + if math.IsNaN(val) || math.IsInf(val, 0) { + continue + } events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), MetricSetFields: common.MapStr{ "dataType": resultType, - queryName: attemptConvertToNumeric(result.Vector[1].(string)), + queryName: val, }, }) } else { @@ -215,13 +242,23 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string msg := fmt.Sprintf("Could not parse 'scalar' value of result: %v", convertedArray.Data.Results) return mb.Event{}, errors.New(msg) } - return mb.Event{ - Timestamp: getTimestamp(timestamp), - MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: val, - }, - }, nil + if math.IsNaN(val) || math.IsInf(val, 0) { + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: value, + }, + }, nil + } else { + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: val, + }, + }, nil + } } else if resultType == "string" { return mb.Event{ Timestamp: getTimestamp(timestamp), @@ -285,13 +322,3 @@ func getTimestamp(num float64) time.Time { ns := int64((num - float64(sec)) * 1000) return time.Unix(sec, ns) } - -func attemptConvertToNumeric(str string) interface{} { - if res, err := strconv.Atoi(str); err == nil { - return res - } else if res, err := strconv.ParseFloat(str, 64); err == nil { - return res - } else { - return str - } -} From 509d62cac1af06c90f74a94c0dbfe87b3c3dd5f3 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 20:23:08 +0200 Subject: [PATCH 19/27] fixes Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 4 ++-- metricbeat/module/prometheus/query/query_integration_test.go | 4 ++-- metricbeat/module/prometheus/query/query_test.go | 4 ++++ metricbeat/module/prometheus/test_prometheus.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index ea1c59cf3b5..f0a09dddf7c 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -50,7 +50,7 @@ type arrayData struct { } // InstantVectorResponse is for "vector" type from Prometheus Query API Request -// Format: +// instantVectorResult format: // [ // { // "metric": { "": "", ... }, @@ -72,7 +72,7 @@ type instantVectorResult struct { } // InstantVectorResponse is for "vector" type from Prometheus Query API Request -// Format: +// rangeVectorResult format: // [ // { // "metric": { "": "", ... }, diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 997bb0ac9b0..129908bf937 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -55,7 +55,7 @@ func TestData(t *testing.T) { if err == nil { return } - time.Sleep(300 * time.Millisecond) + time.Sleep(500 * time.Millisecond) } t.Fatal("write", err) } @@ -86,7 +86,7 @@ func TestQueryFetch(t *testing.T) { if len(events) > 0 { break } - time.Sleep(300 * time.Millisecond) + time.Sleep(500 * time.Millisecond) } if len(errors) > 0 { t.Fatalf("Expected 0 errors, had %d. %v\n", len(errors), errors) diff --git a/metricbeat/module/prometheus/query/query_test.go b/metricbeat/module/prometheus/query/query_test.go index 5b99226eefc..5416538a823 100644 --- a/metricbeat/module/prometheus/query/query_test.go +++ b/metricbeat/module/prometheus/query/query_test.go @@ -51,6 +51,7 @@ func TestQueryFetchEventContentInstantVector(t *testing.T) { "module": "prometheus", "metricsets": []string{"query"}, "hosts": []string{server.URL}, + // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ "query_name": "up", @@ -99,6 +100,7 @@ func TestQueryFetchEventContentRangeVector(t *testing.T) { "module": "prometheus", "metricsets": []string{"query"}, "hosts": []string{server.URL}, + // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ "query_name": "up", @@ -141,6 +143,7 @@ func TestQueryFetchEventContentScalar(t *testing.T) { "module": "prometheus", "metricsets": []string{"query"}, "hosts": []string{server.URL}, + // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ "query_name": "up", @@ -183,6 +186,7 @@ func TestQueryFetchEventContentString(t *testing.T) { "module": "prometheus", "metricsets": []string{"query"}, "hosts": []string{server.URL}, + // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ "query_name": "up", diff --git a/metricbeat/module/prometheus/test_prometheus.py b/metricbeat/module/prometheus/test_prometheus.py index 8b5376a7c04..5527e7fc103 100644 --- a/metricbeat/module/prometheus/test_prometheus.py +++ b/metricbeat/module/prometheus/test_prometheus.py @@ -56,7 +56,7 @@ def test_query(self): } }]) proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0) + self.wait_until(lambda: self.output_lines() > 0, 60) proc.check_kill_and_wait() self.assert_no_logged_warnings() From b7a50a7040f605857515150f2d16e4b9f3ae778c Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 19 Mar 2020 23:55:49 +0200 Subject: [PATCH 20/27] Increase promtetheus warmup waiting time Signed-off-by: chrismark --- metricbeat/module/prometheus/_meta/prometheus.yml | 2 +- .../module/prometheus/query/query_integration_test.go | 6 +++--- metricbeat/module/prometheus/test_prometheus.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/prometheus/_meta/prometheus.yml b/metricbeat/module/prometheus/_meta/prometheus.yml index fece2586616..70b37c9ce39 100644 --- a/metricbeat/module/prometheus/_meta/prometheus.yml +++ b/metricbeat/module/prometheus/_meta/prometheus.yml @@ -1,6 +1,6 @@ # my global config global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + scrape_interval: 1s # Set the scrape interval to every 15 seconds. Default is every 1 minute. evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. # scrape_timeout is set to the global default (10s). diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 129908bf937..626c1d45f01 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -69,10 +69,10 @@ func TestQueryFetch(t *testing.T) { "hosts": []string{service.Host()}, "queries": []common.MapStr{ common.MapStr{ - "query_name": "go_threads", + "query_name": "go_info", "path": "/api/v1/query", "query_params": common.MapStr{ - "query": "go_threads", + "query": "go_info", }, }, }, @@ -86,7 +86,7 @@ func TestQueryFetch(t *testing.T) { if len(events) > 0 { break } - time.Sleep(500 * time.Millisecond) + time.Sleep(10 * time.Second) } if len(errors) > 0 { t.Fatalf("Expected 0 errors, had %d. %v\n", len(errors), errors) diff --git a/metricbeat/module/prometheus/test_prometheus.py b/metricbeat/module/prometheus/test_prometheus.py index 5527e7fc103..2e2aa41ad71 100644 --- a/metricbeat/module/prometheus/test_prometheus.py +++ b/metricbeat/module/prometheus/test_prometheus.py @@ -50,8 +50,8 @@ def test_query(self): "extras": { "queries": [{ "path": "/api/v1/query", - 'query_name': 'instant_vector', - 'query_params': {'query': 'go_threads'} + 'query_name': 'go_info', + 'query_params': {'query': 'go_info'} }] } }]) From b49dcfb1f3e089db3243003b120e1aaa4f536461 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 20 Mar 2020 10:16:34 +0200 Subject: [PATCH 21/27] Fix configuration docstrings Signed-off-by: chrismark --- metricbeat/module/prometheus/_meta/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/_meta/prometheus.yml b/metricbeat/module/prometheus/_meta/prometheus.yml index 70b37c9ce39..06707841f8d 100644 --- a/metricbeat/module/prometheus/_meta/prometheus.yml +++ b/metricbeat/module/prometheus/_meta/prometheus.yml @@ -1,6 +1,6 @@ # my global config global: - scrape_interval: 1s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + scrape_interval: 1s # Set the scrape interval to every 1 second. Default is every 1 minute. evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. # scrape_timeout is set to the global default (10s). From 65057ab364818af142f82d9286fa841b162ace49 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 20 Mar 2020 10:21:44 +0200 Subject: [PATCH 22/27] Improve metricset configuration Signed-off-by: chrismark --- metricbeat/docs/modules/prometheus.asciidoc | 36 ++++++++++--------- metricbeat/metricbeat.reference.yml | 36 ++++++++++--------- metricbeat/module/prometheus/_meta/config.yml | 36 ++++++++++--------- metricbeat/modules.d/prometheus.yml.disabled | 36 ++++++++++--------- 4 files changed, 80 insertions(+), 64 deletions(-) diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index f19e9b6572e..2af58b2cd79 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -56,27 +56,31 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" - +# Metrics that will be collected using a PromQL - module: prometheus period: 10s hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "instant_vector" - path: "/api/v1/query" - query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" - - query_name: "range_vector" - path: "/api/v1/query_range" - query_params: - query: "up" - start: "2019-12-20T00:00:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h - - query_name: "scalar" - path: "/api/v1/query" - query_params: - query: "100" + #- query_name: "instant_vector" + # path: "/api/v1/query" + # query_params: + # query: "sum(rate(prometheus_http_requests_total[1m]))" + #- query_name: "range_vector" + # path: "/api/v1/query_range" + # query_params: + # query: "up" + # start: "2019-12-20T00:00:00.000Z" + # end: "2019-12-21T00:00:00.000Z" + # step: 1h + #- query_name: "scalar" + # path: "/api/v1/query" + # query_params: + # query: "100" + #- query_name: "string" + # path: "/api/v1/query" + # query_params: + # query: "some_value" ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index a693d3e80b6..c671a55b5b7 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -728,27 +728,31 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" - +# Metrics that will be collected using a PromQL - module: prometheus period: 10s hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "instant_vector" - path: "/api/v1/query" - query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" - - query_name: "range_vector" - path: "/api/v1/query_range" - query_params: - query: "up" - start: "2019-12-20T00:00:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h - - query_name: "scalar" - path: "/api/v1/query" - query_params: - query: "100" + #- query_name: "instant_vector" + # path: "/api/v1/query" + # query_params: + # query: "sum(rate(prometheus_http_requests_total[1m]))" + #- query_name: "range_vector" + # path: "/api/v1/query_range" + # query_params: + # query: "up" + # start: "2019-12-20T00:00:00.000Z" + # end: "2019-12-21T00:00:00.000Z" + # step: 1h + #- query_name: "scalar" + # path: "/api/v1/query" + # query_params: + # query: "100" + #- query_name: "string" + # path: "/api/v1/query" + # query_params: + # query: "some_value" #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index 00ceef9094c..e824ccfcbb6 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -26,24 +26,28 @@ #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" - +# Metrics that will be collected using a PromQL - module: prometheus period: 10s hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "instant_vector" - path: "/api/v1/query" - query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" - - query_name: "range_vector" - path: "/api/v1/query_range" - query_params: - query: "up" - start: "2019-12-20T00:00:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h - - query_name: "scalar" - path: "/api/v1/query" - query_params: - query: "100" + #- query_name: "instant_vector" + # path: "/api/v1/query" + # query_params: + # query: "sum(rate(prometheus_http_requests_total[1m]))" + #- query_name: "range_vector" + # path: "/api/v1/query_range" + # query_params: + # query: "up" + # start: "2019-12-20T00:00:00.000Z" + # end: "2019-12-21T00:00:00.000Z" + # step: 1h + #- query_name: "scalar" + # path: "/api/v1/query" + # query_params: + # query: "100" + #- query_name: "string" + # path: "/api/v1/query" + # query_params: + # query: "some_value" diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index a6d414e3ffc..6e236a27c5c 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -29,24 +29,28 @@ #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" - +# Metrics that will be collected using a PromQL - module: prometheus period: 10s hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "instant_vector" - path: "/api/v1/query" - query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" - - query_name: "range_vector" - path: "/api/v1/query_range" - query_params: - query: "up" - start: "2019-12-20T00:00:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h - - query_name: "scalar" - path: "/api/v1/query" - query_params: - query: "100" + #- query_name: "instant_vector" + # path: "/api/v1/query" + # query_params: + # query: "sum(rate(prometheus_http_requests_total[1m]))" + #- query_name: "range_vector" + # path: "/api/v1/query_range" + # query_params: + # query: "up" + # start: "2019-12-20T00:00:00.000Z" + # end: "2019-12-21T00:00:00.000Z" + # step: 1h + #- query_name: "scalar" + # path: "/api/v1/query" + # query_params: + # query: "100" + #- query_name: "string" + # path: "/api/v1/query" + # query_params: + # query: "some_value" From 8c01056df4fc81e0812e31f74d2f35f3a215a61d Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 20 Mar 2020 10:40:54 +0200 Subject: [PATCH 23/27] make update Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 36 ++++++++++++---------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 7251ef4f835..e2c4c3d5675 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1020,27 +1020,31 @@ metricbeat.modules: #ssl.certificate: "/etc/pki/server/cert.pem" #ssl.key: "/etc/pki/server/cert.key" - +# Metrics that will be collected using a PromQL - module: prometheus period: 10s hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "instant_vector" - path: "/api/v1/query" - query_params: - query: "sum(rate(prometheus_http_requests_total[1m]))" - - query_name: "range_vector" - path: "/api/v1/query_range" - query_params: - query: "up" - start: "2019-12-20T00:00:00.000Z" - end: "2019-12-21T00:00:00.000Z" - step: 1h - - query_name: "scalar" - path: "/api/v1/query" - query_params: - query: "100" + #- query_name: "instant_vector" + # path: "/api/v1/query" + # query_params: + # query: "sum(rate(prometheus_http_requests_total[1m]))" + #- query_name: "range_vector" + # path: "/api/v1/query_range" + # query_params: + # query: "up" + # start: "2019-12-20T00:00:00.000Z" + # end: "2019-12-21T00:00:00.000Z" + # step: 1h + #- query_name: "scalar" + # path: "/api/v1/query" + # query_params: + # query: "100" + #- query_name: "string" + # path: "/api/v1/query" + # query_params: + # query: "some_value" #------------------------------- RabbitMQ Module ------------------------------- - module: rabbitmq From 2bd38b27de5ad5e22a4d82dc7713deb19f1dc43d Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 20 Mar 2020 11:19:17 +0200 Subject: [PATCH 24/27] Move parts to helper functions Signed-off-by: chrismark --- metricbeat/module/prometheus/query/data.go | 130 ++++++++++---------- metricbeat/module/prometheus/query/query.go | 4 +- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index f0a09dddf7c..10c2e3985e0 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -109,13 +109,13 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { } events = append(events, event) case "vector": - evnts, err := getEventsFromVector(body, resultType, pathConfig.QueryName) + evnts, err := getEventsFromVector(body, pathConfig.QueryName) if err != nil { return events, err } events = append(events, evnts...) case "matrix": - evnts, err := getEventsFromMatrix(body, resultType, pathConfig.QueryName) + evnts, err := getEventsFromMatrix(body, pathConfig.QueryName) if err != nil { return events, err } @@ -127,8 +127,9 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { return events, nil } -func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb.Event, error) { +func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { events := []mb.Event{} + resultType := "matrix" convertedMap, err := convertJSONToRangeVectorResponse(body) if err != nil { return events, err @@ -137,23 +138,13 @@ func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb for _, result := range results { for _, vector := range result.Vectors { if vector != nil { - if len(vector) != 2 { - return []mb.Event{}, errors.New("Could not parse results") - } - timestamp, ok := vector[0].(float64) - if !ok { - msg := fmt.Sprintf("Could not parse timestamp of result: %v", vector) - return []mb.Event{}, errors.New(msg) - } - value, ok := vector[1].(string) - if !ok { - msg := fmt.Sprintf("Could not parse value of result: %v", vector) - return []mb.Event{}, errors.New(msg) + timestamp, err := getTimestampFromVector(vector) + if err != nil { + return []mb.Event{}, err } - val, err := strconv.ParseFloat(value, 64) + val, err := getValueFromVector(vector) if err != nil { - msg := fmt.Sprintf("Could not parse 'range vector' value of result: %v", vector) - return []mb.Event{}, errors.New(msg) + return []mb.Event{}, err } if math.IsNaN(val) || math.IsInf(val, 0) { continue @@ -173,8 +164,9 @@ func getEventsFromMatrix(body []byte, resultType string, queryName string) ([]mb return events, nil } -func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb.Event, error) { +func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { events := []mb.Event{} + resultType := "vector" convertedMap, err := convertJSONToInstantVectorResponse(body) if err != nil { return events, err @@ -182,23 +174,13 @@ func getEventsFromVector(body []byte, resultType string, queryName string) ([]mb results := convertedMap.Data.Results for _, result := range results { if result.Vector != nil { - if len(result.Vector) != 2 { - return []mb.Event{}, errors.New("Could not parse results") - } - timestamp, ok := result.Vector[0].(float64) - if !ok { - msg := fmt.Sprintf("Could not parse timestamp of result: %v", result.Vector) - return []mb.Event{}, errors.New(msg) - } - value, ok := result.Vector[1].(string) - if !ok { - msg := fmt.Sprintf("Could not parse value of result: %v", result.Vector) - return []mb.Event{}, errors.New(msg) + timestamp, err := getTimestampFromVector(result.Vector) + if err != nil { + return []mb.Event{}, err } - val, err := strconv.ParseFloat(value, 64) + val, err := getValueFromVector(result.Vector) if err != nil { - msg := fmt.Sprintf("Could not parse 'instant vector' value of result: %v", result.Vector) - return []mb.Event{}, errors.New(msg) + return []mb.Event{}, err } if math.IsNaN(val) || math.IsInf(val, 0) { continue @@ -223,43 +205,31 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string return mb.Event{}, err } if convertedArray.Data.Results != nil { - if len(convertedArray.Data.Results) != 2 { - return mb.Event{}, errors.New("Could not parse results") - } - timestamp, ok := convertedArray.Data.Results[0].(float64) - if !ok { - msg := fmt.Sprintf("Could not parse timestamp of result: %v", convertedArray.Data.Results) - return mb.Event{}, errors.New(msg) - } - value, ok := convertedArray.Data.Results[1].(string) - if !ok { - msg := fmt.Sprintf("Could not parse value of result: %v", convertedArray.Data.Results) - return mb.Event{}, errors.New(msg) + timestamp, err := getTimestampFromVector(convertedArray.Data.Results) + if err != nil { + return mb.Event{}, err } if resultType == "scalar" { - val, err := strconv.ParseFloat(value, 64) + val, err := getValueFromVector(convertedArray.Data.Results) if err != nil { - msg := fmt.Sprintf("Could not parse 'scalar' value of result: %v", convertedArray.Data.Results) - return mb.Event{}, errors.New(msg) + return mb.Event{}, err } if math.IsNaN(val) || math.IsInf(val, 0) { - return mb.Event{ - Timestamp: getTimestamp(timestamp), - MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: value, - }, - }, nil - } else { - return mb.Event{ - Timestamp: getTimestamp(timestamp), - MetricSetFields: common.MapStr{ - "dataType": resultType, - queryName: val, - }, - }, nil + return mb.Event{}, nil } + return mb.Event{ + Timestamp: getTimestamp(timestamp), + MetricSetFields: common.MapStr{ + "dataType": resultType, + queryName: val, + }, + }, nil } else if resultType == "string" { + value, ok := convertedArray.Data.Results[1].(string) + if !ok { + msg := fmt.Sprintf("Could not parse value of result: %v", convertedArray.Data.Results) + return mb.Event{}, errors.New(msg) + } return mb.Event{ Timestamp: getTimestamp(timestamp), MetricSetFields: common.MapStr{ @@ -268,11 +238,41 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string }, }, nil } - } return mb.Event{}, errors.New("Could not parse results") } +func getTimestampFromVector(vector []interface{}) (float64, error) { + // Example input: [ , "" ] + if len(vector) != 2 { + return 0, errors.New("Could not parse results") + } + timestamp, ok := vector[0].(float64) + if !ok { + msg := fmt.Sprintf("Could not parse timestamp of result: %v", vector) + return 0, errors.New(msg) + } + return timestamp, nil +} + +func getValueFromVector(vector []interface{}) (float64, error) { + // Example input: [ , "" ] + if len(vector) != 2 { + return 0, errors.New("Could not parse results") + } + value, ok := vector[1].(string) + if !ok { + msg := fmt.Sprintf("Could not parse value of result: %v", vector) + return 0, errors.New(msg) + } + val, err := strconv.ParseFloat(value, 64) + if err != nil { + msg := fmt.Sprintf("Could not parse value of result: %v", vector) + return 0, errors.New(msg) + } + return val, nil +} + func getResultType(body []byte) (string, error) { response := Response{} if err := json.Unmarshal(body, &response); err != nil { diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index 29d3bb82529..a2c25376b34 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -82,7 +82,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { - reporter.Error(errors.Wrapf(err, "unable to fetch data from prometheus endpoint: %v", pathConfig.Path)) + reporter.Error(errors.Wrapf(err, "unable to fetch data from prometheus endpoint: %v", url)) continue } defer func() { @@ -98,7 +98,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { events, parseErr := parseResponse(body, pathConfig) if parseErr != nil { - reporter.Error(errors.Wrapf(parseErr, "error parsing response for: %v", pathConfig.QueryName)) + reporter.Error(errors.Wrapf(parseErr, "error parsing response from: %v", url)) continue } for _, e := range events { From 175104d87c98f00930c9e0de7a38968729deb574 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 20 Mar 2020 12:24:17 +0200 Subject: [PATCH 25/27] Fix labels and move string query result to labels Signed-off-by: chrismark --- metricbeat/module/prometheus/query/_meta/data.json | 2 +- metricbeat/module/prometheus/query/data.go | 11 +++++++---- .../module/prometheus/query/query_integration_test.go | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 3d4a31914f3..6f03bd4ce7e 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -17,7 +17,7 @@ }, "query": { "dataType": "vector", - "go_threads": 24 + "go_threads": 26 } }, "service": { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 10c2e3985e0..12e31f4729c 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -150,7 +150,8 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { continue } events = append(events, mb.Event{ - Timestamp: getTimestamp(timestamp), + Timestamp: getTimestamp(timestamp), + ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ "dataType": resultType, queryName: val, @@ -186,7 +187,8 @@ func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { continue } events = append(events, mb.Event{ - Timestamp: getTimestamp(timestamp), + Timestamp: getTimestamp(timestamp), + ModuleFields: common.MapStr{"labels": result.Metric}, MetricSetFields: common.MapStr{ "dataType": resultType, queryName: val, @@ -231,10 +233,11 @@ func getEventFromScalarOrString(body []byte, resultType string, queryName string return mb.Event{}, errors.New(msg) } return mb.Event{ - Timestamp: getTimestamp(timestamp), + Timestamp: getTimestamp(timestamp), + ModuleFields: common.MapStr{"labels": common.MapStr{queryName: value}}, MetricSetFields: common.MapStr{ "dataType": resultType, - queryName: value, + queryName: 1, }, }, nil } diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 626c1d45f01..82ff59edcda 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -55,7 +55,7 @@ func TestData(t *testing.T) { if err == nil { return } - time.Sleep(500 * time.Millisecond) + time.Sleep(10 * time.Second) } t.Fatal("write", err) } From f8c2501ca75c8df84c137563b14aa12fd8b2a13f Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 23 Mar 2020 12:05:16 +0200 Subject: [PATCH 26/27] fixes Signed-off-by: chrismark --- metricbeat/docs/modules/prometheus.asciidoc | 16 +++++----- metricbeat/metricbeat.reference.yml | 16 +++++----- metricbeat/module/prometheus/_meta/config.yml | 16 +++++----- .../module/prometheus/query/_meta/data.json | 1 - .../prometheus/query/_meta/docs.asciidoc | 14 ++++----- metricbeat/module/prometheus/query/config.go | 14 ++++----- metricbeat/module/prometheus/query/data.go | 8 +++-- metricbeat/module/prometheus/query/query.go | 2 +- .../query/query_integration_test.go | 12 +++---- .../module/prometheus/query/query_test.go | 31 ++++++++++--------- .../module/prometheus/test_prometheus.py | 4 +-- metricbeat/modules.d/prometheus.yml.disabled | 16 +++++----- x-pack/metricbeat/metricbeat.reference.yml | 16 +++++----- 13 files changed, 85 insertions(+), 81 deletions(-) diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 2af58b2cd79..a6fc4b82bed 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -62,24 +62,24 @@ metricbeat.modules: hosts: ["localhost:9090"] metricsets: ["query"] queries: - #- query_name: "instant_vector" + #- name: "instant_vector" # path: "/api/v1/query" - # query_params: + # params: # query: "sum(rate(prometheus_http_requests_total[1m]))" - #- query_name: "range_vector" + #- name: "range_vector" # path: "/api/v1/query_range" - # query_params: + # params: # query: "up" # start: "2019-12-20T00:00:00.000Z" # end: "2019-12-21T00:00:00.000Z" # step: 1h - #- query_name: "scalar" + #- name: "scalar" # path: "/api/v1/query" - # query_params: + # params: # query: "100" - #- query_name: "string" + #- name: "string" # path: "/api/v1/query" - # query_params: + # params: # query: "some_value" ---- diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index c671a55b5b7..4db72883ad5 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -734,24 +734,24 @@ metricbeat.modules: hosts: ["localhost:9090"] metricsets: ["query"] queries: - #- query_name: "instant_vector" + #- name: "instant_vector" # path: "/api/v1/query" - # query_params: + # params: # query: "sum(rate(prometheus_http_requests_total[1m]))" - #- query_name: "range_vector" + #- name: "range_vector" # path: "/api/v1/query_range" - # query_params: + # params: # query: "up" # start: "2019-12-20T00:00:00.000Z" # end: "2019-12-21T00:00:00.000Z" # step: 1h - #- query_name: "scalar" + #- name: "scalar" # path: "/api/v1/query" - # query_params: + # params: # query: "100" - #- query_name: "string" + #- name: "string" # path: "/api/v1/query" - # query_params: + # params: # query: "some_value" #------------------------------- RabbitMQ Module ------------------------------- diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml index e824ccfcbb6..aa855dd2dec 100644 --- a/metricbeat/module/prometheus/_meta/config.yml +++ b/metricbeat/module/prometheus/_meta/config.yml @@ -32,22 +32,22 @@ hosts: ["localhost:9090"] metricsets: ["query"] queries: - #- query_name: "instant_vector" + #- name: "instant_vector" # path: "/api/v1/query" - # query_params: + # params: # query: "sum(rate(prometheus_http_requests_total[1m]))" - #- query_name: "range_vector" + #- name: "range_vector" # path: "/api/v1/query_range" - # query_params: + # params: # query: "up" # start: "2019-12-20T00:00:00.000Z" # end: "2019-12-21T00:00:00.000Z" # step: 1h - #- query_name: "scalar" + #- name: "scalar" # path: "/api/v1/query" - # query_params: + # params: # query: "100" - #- query_name: "string" + #- name: "string" # path: "/api/v1/query" - # query_params: + # params: # query: "some_value" diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 6f03bd4ce7e..26c2de382b0 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -11,7 +11,6 @@ }, "prometheus": { "labels": { - "__name__": "go_threads", "instance": "localhost:9090", "job": "prometheus" }, diff --git a/metricbeat/module/prometheus/query/_meta/docs.asciidoc b/metricbeat/module/prometheus/query/_meta/docs.asciidoc index d6342c3bedc..60025f0d96d 100644 --- a/metricbeat/module/prometheus/query/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/query/_meta/docs.asciidoc @@ -15,9 +15,9 @@ The following configuration performs an instant query for `up` metric at a singl hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: 'up' + - name: 'up' path: '/api/v1/query' - query_params: + params: query: "up" ------------------------------------------------------------------------------------- @@ -31,9 +31,9 @@ requests as measured over the last 5 minutes. hosts: ["localhost:9090"] metricsets: ["query"] queries: - - query_name: "rate_http_requests_total" + - name: "rate_http_requests_total" path: "/api/v1/query" - query_params: + params: query: "rate(prometheus_http_requests_total[5m])" ------------------------------------------------------------------------------------- @@ -52,10 +52,10 @@ The following example evaluates the expression `up` over a 30-second range with metricsets: ["query"] hosts: ["node:9100"] queries: - - query_name: "up_master" + - name: "up_master" path: "/api/v1/query_range" - query_params: - query: "up{node="master01"}" + params: + query: "up{node='master01'}" start: "2019-12-20T23:30:30.000Z" end: "2019-12-21T23:31:00.000Z" step: 15s diff --git a/metricbeat/module/prometheus/query/config.go b/metricbeat/module/prometheus/query/config.go index c5cfbe3828e..afb30f1c80d 100644 --- a/metricbeat/module/prometheus/query/config.go +++ b/metricbeat/module/prometheus/query/config.go @@ -31,24 +31,24 @@ type Config struct { // QueryConfig is used to make an API request. type QueryConfig struct { - Path string `config:"path"` - QueryParams common.MapStr `config:"query_params"` - QueryName string `config:"query_name"` + Path string `config:"path"` + Params common.MapStr `config:"params"` + Name string `config:"name"` } func defaultConfig() Config { return Config{ DefaultQuery: QueryConfig{ - Path: "/api/v1/query", - QueryName: "default", + Path: "/api/v1/query", + Name: "default", }, } } // Validate for Prometheus "query" metricset config func (p *QueryConfig) Validate() error { - if p.QueryName == "" { - return errors.New("`query_name` can not be empty in path configuration") + if p.Name == "" { + return errors.New("`name` can not be empty in path configuration") } if p.Path == "" { diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 12e31f4729c..32987393374 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -103,19 +103,19 @@ func parseResponse(body []byte, pathConfig QueryConfig) ([]mb.Event, error) { switch resultType { case "scalar", "string": - event, err := getEventFromScalarOrString(body, resultType, pathConfig.QueryName) + event, err := getEventFromScalarOrString(body, resultType, pathConfig.Name) if err != nil { return events, err } events = append(events, event) case "vector": - evnts, err := getEventsFromVector(body, pathConfig.QueryName) + evnts, err := getEventsFromVector(body, pathConfig.Name) if err != nil { return events, err } events = append(events, evnts...) case "matrix": - evnts, err := getEventsFromMatrix(body, pathConfig.QueryName) + evnts, err := getEventsFromMatrix(body, pathConfig.Name) if err != nil { return events, err } @@ -149,6 +149,7 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } + delete(result.Metric, "__name__") events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, @@ -186,6 +187,7 @@ func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } + delete(result.Metric, "__name__") events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, diff --git a/metricbeat/module/prometheus/query/query.go b/metricbeat/module/prometheus/query/query.go index a2c25376b34..3f419753a72 100644 --- a/metricbeat/module/prometheus/query/query.go +++ b/metricbeat/module/prometheus/query/query.go @@ -78,7 +78,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, pathConfig := range m.queries { - url := m.getURL(pathConfig.Path, pathConfig.QueryParams) + url := m.getURL(pathConfig.Path, pathConfig.Params) m.http.SetURI(url) response, err := m.http.FetchResponse() if err != nil { diff --git a/metricbeat/module/prometheus/query/query_integration_test.go b/metricbeat/module/prometheus/query/query_integration_test.go index 82ff59edcda..a54c9f1daf8 100644 --- a/metricbeat/module/prometheus/query/query_integration_test.go +++ b/metricbeat/module/prometheus/query/query_integration_test.go @@ -40,9 +40,9 @@ func TestData(t *testing.T) { "hosts": []string{service.Host()}, "queries": []common.MapStr{ common.MapStr{ - "query_name": "go_threads", - "path": "/api/v1/query", - "query_params": common.MapStr{ + "name": "go_threads", + "path": "/api/v1/query", + "params": common.MapStr{ "query": "go_threads", }, }, @@ -69,9 +69,9 @@ func TestQueryFetch(t *testing.T) { "hosts": []string{service.Host()}, "queries": []common.MapStr{ common.MapStr{ - "query_name": "go_info", - "path": "/api/v1/query", - "query_params": common.MapStr{ + "name": "go_info", + "path": "/api/v1/query", + "params": common.MapStr{ "query": "go_info", }, }, diff --git a/metricbeat/module/prometheus/query/query_test.go b/metricbeat/module/prometheus/query/query_test.go index 5416538a823..73a7b885f50 100644 --- a/metricbeat/module/prometheus/query/query_test.go +++ b/metricbeat/module/prometheus/query/query_test.go @@ -54,9 +54,9 @@ func TestQueryFetchEventContentInstantVector(t *testing.T) { // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", - "path": "/api/v1/query", - "query_params": common.MapStr{ + "name": "up", + "path": "/api/v1/query", + "params": common.MapStr{ "query": "up", }, }, @@ -103,10 +103,13 @@ func TestQueryFetchEventContentRangeVector(t *testing.T) { // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", - "path": "/api/v1/query", - "query_params": common.MapStr{ + "name": "up_range", + "path": "/api/v1/query", + "params": common.MapStr{ "query": "up", + "start": "2019-12-20T23:30:30.000Z", + "end": "2019-12-21T23:31:00.000Z", + "step": "15s", }, }, }, @@ -146,10 +149,10 @@ func TestQueryFetchEventContentScalar(t *testing.T) { // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", - "path": "/api/v1/query", - "query_params": common.MapStr{ - "query": "up", + "name": "scalar", + "path": "/api/v1/query", + "params": common.MapStr{ + "query": "100", }, }, }, @@ -189,10 +192,10 @@ func TestQueryFetchEventContentString(t *testing.T) { // queries do not have an actual role here since all http responses are mocked "queries": []common.MapStr{ common.MapStr{ - "query_name": "up", - "path": "/api/v1/query", - "query_params": common.MapStr{ - "query": "up", + "name": "string", + "path": "/api/v1/query", + "params": common.MapStr{ + "query": "some", }, }, }, diff --git a/metricbeat/module/prometheus/test_prometheus.py b/metricbeat/module/prometheus/test_prometheus.py index 2e2aa41ad71..e58b1138b99 100644 --- a/metricbeat/module/prometheus/test_prometheus.py +++ b/metricbeat/module/prometheus/test_prometheus.py @@ -50,8 +50,8 @@ def test_query(self): "extras": { "queries": [{ "path": "/api/v1/query", - 'query_name': 'go_info', - 'query_params': {'query': 'go_info'} + 'name': 'go_info', + 'params': {'query': 'go_info'} }] } }]) diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index 6e236a27c5c..34804913598 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -35,22 +35,22 @@ hosts: ["localhost:9090"] metricsets: ["query"] queries: - #- query_name: "instant_vector" + #- name: "instant_vector" # path: "/api/v1/query" - # query_params: + # params: # query: "sum(rate(prometheus_http_requests_total[1m]))" - #- query_name: "range_vector" + #- name: "range_vector" # path: "/api/v1/query_range" - # query_params: + # params: # query: "up" # start: "2019-12-20T00:00:00.000Z" # end: "2019-12-21T00:00:00.000Z" # step: 1h - #- query_name: "scalar" + #- name: "scalar" # path: "/api/v1/query" - # query_params: + # params: # query: "100" - #- query_name: "string" + #- name: "string" # path: "/api/v1/query" - # query_params: + # params: # query: "some_value" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index e2c4c3d5675..fadfba75cf9 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1026,24 +1026,24 @@ metricbeat.modules: hosts: ["localhost:9090"] metricsets: ["query"] queries: - #- query_name: "instant_vector" + #- name: "instant_vector" # path: "/api/v1/query" - # query_params: + # params: # query: "sum(rate(prometheus_http_requests_total[1m]))" - #- query_name: "range_vector" + #- name: "range_vector" # path: "/api/v1/query_range" - # query_params: + # params: # query: "up" # start: "2019-12-20T00:00:00.000Z" # end: "2019-12-21T00:00:00.000Z" # step: 1h - #- query_name: "scalar" + #- name: "scalar" # path: "/api/v1/query" - # query_params: + # params: # query: "100" - #- query_name: "string" + #- name: "string" # path: "/api/v1/query" - # query_params: + # params: # query: "some_value" #------------------------------- RabbitMQ Module ------------------------------- From c76cfc9c861f92cc06679e49b1618d4044a1851a Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 23 Mar 2020 12:22:08 +0200 Subject: [PATCH 27/27] put __name__ labels back Signed-off-by: chrismark --- metricbeat/module/prometheus/query/_meta/data.json | 1 + metricbeat/module/prometheus/query/data.go | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/module/prometheus/query/_meta/data.json b/metricbeat/module/prometheus/query/_meta/data.json index 26c2de382b0..6f03bd4ce7e 100644 --- a/metricbeat/module/prometheus/query/_meta/data.json +++ b/metricbeat/module/prometheus/query/_meta/data.json @@ -11,6 +11,7 @@ }, "prometheus": { "labels": { + "__name__": "go_threads", "instance": "localhost:9090", "job": "prometheus" }, diff --git a/metricbeat/module/prometheus/query/data.go b/metricbeat/module/prometheus/query/data.go index 32987393374..b89df6b4305 100644 --- a/metricbeat/module/prometheus/query/data.go +++ b/metricbeat/module/prometheus/query/data.go @@ -149,7 +149,6 @@ func getEventsFromMatrix(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } - delete(result.Metric, "__name__") events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric}, @@ -187,7 +186,6 @@ func getEventsFromVector(body []byte, queryName string) ([]mb.Event, error) { if math.IsNaN(val) || math.IsInf(val, 0) { continue } - delete(result.Metric, "__name__") events = append(events, mb.Event{ Timestamp: getTimestamp(timestamp), ModuleFields: common.MapStr{"labels": result.Metric},