Cherry-pick #17104 to 7.x: [Metricbeat] Add 'query' metricset for prometheus module

CHANGELOG.next.asciidoc

@@ -328,6 +328,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview dashboard for googlecloud compute metricset. {issue}16534[16534] {pull}16819[16819]
 - Add Prometheus remote write endpoint {pull}16609[16609]
 - Release STAN module as GA. {pull}16980[16980]
+- Add query metricset for prometheus module. {pull}17104[17104]
 - Release ActiveMQ module as GA. {issue}17047[17047] {pull}17049[17049]
 - Add support for CouchDB v2 {issue}16352[16352] {pull}16455[16455]
 - Release Zookeeper/connection module as GA. {issue}14281[14281] {pull}17043[17043]

metricbeat/docs/fields.asciidoc

@@ -31617,6 +31617,22 @@ type: object
 
 --
 
+*`prometheus.query.*`*::
++
+--
+Prometheus value resulted from PromQL
+
+
+type: object
+
+--
+
+[float]
+=== query
+
+query metricset
+
+
 [float]
 === remote_write
 

metricbeat/docs/modules/prometheus.asciidoc

@@ -45,6 +45,7 @@ metricbeat.modules:
   #ssl.certificate_authorities:
   #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
 
+
 # Metrics sent by a Prometheus server using remote_write option
 - module: prometheus
   metricsets: ["remote_write"]
@@ -54,6 +55,32 @@ metricbeat.modules:
   # Secure settings for the server using TLS/SSL:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"
+
+# Metrics that will be collected using a PromQL
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9090"]
+  metricsets: ["query"]
+  queries:
+  #- name: "instant_vector"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "sum(rate(prometheus_http_requests_total[1m]))"
+  #- name: "range_vector"
+  #  path: "/api/v1/query_range"
+  #  params:
+  #    query: "up"
+  #    start: "2019-12-20T00:00:00.000Z"
+  #    end:  "2019-12-21T00:00:00.000Z"
+  #    step: 1h
+  #- name: "scalar"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "100"
+  #- name: "string"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "some_value"
 ----
 
 This module supports TLS connections when using `ssl` config field, as described in <<configuration-ssl>>.
@@ -66,9 +93,13 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-prometheus-collector,collector>>
 
+* <<metricbeat-metricset-prometheus-query,query>>
+
 * <<metricbeat-metricset-prometheus-remote_write,remote_write>>
 
 include::prometheus/collector.asciidoc[]
 
+include::prometheus/query.asciidoc[]
+
 include::prometheus/remote_write.asciidoc[]
 

metricbeat/docs/modules/prometheus/query.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-prometheus-query]]
+=== Prometheus query metricset
+
+beta[]
+
+include::../../../module/prometheus/query/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-prometheus,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/prometheus/query/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -206,7 +206,8 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-postgresql-database,database>>   
 |<<metricbeat-metricset-postgresql-statement,statement>>   
 |<<metricbeat-module-prometheus,Prometheus>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.2+| .2+|  |<<metricbeat-metricset-prometheus-collector,collector>>   
+.3+| .3+|  |<<metricbeat-metricset-prometheus-collector,collector>>   
+|<<metricbeat-metricset-prometheus-query,query>> beta[]  
 |<<metricbeat-metricset-prometheus-remote_write,remote_write>> beta[]  
 |<<metricbeat-module-rabbitmq,RabbitMQ>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .4+| .4+|  |<<metricbeat-metricset-rabbitmq-connection,connection>>   

metricbeat/metricbeat.reference.yml

@@ -717,6 +717,7 @@ metricbeat.modules:
   #ssl.certificate_authorities:
   #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
 
+
 # Metrics sent by a Prometheus server using remote_write option
 - module: prometheus
   metricsets: ["remote_write"]
@@ -727,6 +728,32 @@ metricbeat.modules:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"
 
+# Metrics that will be collected using a PromQL
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9090"]
+  metricsets: ["query"]
+  queries:
+  #- name: "instant_vector"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "sum(rate(prometheus_http_requests_total[1m]))"
+  #- name: "range_vector"
+  #  path: "/api/v1/query_range"
+  #  params:
+  #    query: "up"
+  #    start: "2019-12-20T00:00:00.000Z"
+  #    end:  "2019-12-21T00:00:00.000Z"
+  #    step: 1h
+  #- name: "scalar"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "100"
+  #- name: "string"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "some_value"
+
 #------------------------------- RabbitMQ Module -------------------------------
 - module: rabbitmq
   metricsets: ["node", "queue", "connection"]

metricbeat/module/prometheus/_meta/config.yml

@@ -15,6 +15,7 @@
   #ssl.certificate_authorities:
   #  - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
 
+
 # Metrics sent by a Prometheus server using remote_write option
 - module: prometheus
   metricsets: ["remote_write"]
@@ -24,3 +25,29 @@
   # Secure settings for the server using TLS/SSL:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"
+
+# Metrics that will be collected using a PromQL
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9090"]
+  metricsets: ["query"]
+  queries:
+  #- name: "instant_vector"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "sum(rate(prometheus_http_requests_total[1m]))"
+  #- name: "range_vector"
+  #  path: "/api/v1/query_range"
+  #  params:
+  #    query: "up"
+  #    start: "2019-12-20T00:00:00.000Z"
+  #    end:  "2019-12-21T00:00:00.000Z"
+  #    step: 1h
+  #- name: "scalar"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "100"
+  #- name: "string"
+  #  path: "/api/v1/query"
+  #  params:
+  #    query: "some_value"

metricbeat/module/prometheus/_meta/fields.yml

@@ -21,3 +21,9 @@
           object_type_mapping_type: "*"
           description: >
             Prometheus metric
+        - name: query.*
+          type: object
+          object_type: double
+          object_type_mapping_type: "*"
+          description: >
+            Prometheus value resulted from PromQL

metricbeat/module/prometheus/_meta/prometheus.yml

@@ -1,6 +1,6 @@
 # my global config
 global:
-  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  scrape_interval:     1s # Set the scrape interval to every 1 second. Default is every 1 minute.
   evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
   # scrape_timeout is set to the global default (10s).
 

metricbeat/module/prometheus/query/_meta/data.json

@@ -0,0 +1,27 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "event": {
+        "dataset": "prometheus.query",
+        "duration": 115000,
+        "module": "prometheus"
+    },
+    "metricset": {
+        "name": "query",
+        "period": 10000
+    },
+    "prometheus": {
+        "labels": {
+            "__name__": "go_threads",
+            "instance": "localhost:9090",
+            "job": "prometheus"
+        },
+        "query": {
+            "dataType": "vector",
+            "go_threads": 26
+        }
+    },
+    "service": {
+        "address": "localhost:32768",
+        "type": "prometheus"
+    }
+}

metricbeat/module/prometheus/query/_meta/docs.asciidoc

@@ -0,0 +1,62 @@
+This is the `query` metricset to query from https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries[querying API of Promtheus].
+
+
+[float]
+=== Configuration
+
+[float]
+==== Instant queries
+
+The following configuration performs an instant query for `up` metric at a single point in time:
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9090"]
+  metricsets: ["query"]
+  queries:
+  - name: 'up'
+    path: '/api/v1/query'
+    params:
+      query: "up"
+-------------------------------------------------------------------------------------
+
+
+More complex PromQL expressions can also be used like the following one which calculates the per-second rate of HTTP
+requests as measured over the last 5 minutes.
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: prometheus
+  period: 10s
+  hosts: ["localhost:9090"]
+  metricsets: ["query"]
+  queries:
+  - name: "rate_http_requests_total"
+    path: "/api/v1/query"
+    params:
+      query: "rate(prometheus_http_requests_total[5m])"
+-------------------------------------------------------------------------------------
+
+
+
+
+[float]
+==== Range queries
+
+
+The following example evaluates the expression `up` over a 30-second range with a query resolution of 15 seconds:
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: prometheus
+  period: 10s
+  metricsets: ["query"]
+  hosts: ["node:9100"]
+  queries:
+  - name: "up_master"
+    path: "/api/v1/query_range"
+    params:
+      query: "up{node='master01'}"
+      start: "2019-12-20T23:30:30.000Z"
+      end: "2019-12-21T23:31:00.000Z"
+      step: 15s
+-------------------------------------------------------------------------------------

metricbeat/module/prometheus/query/_meta/fields.yml

@@ -0,0 +1,6 @@
+- name: query
+  type: group
+  description: >
+    query metricset
+  release: beta
+  fields:

metricbeat/module/prometheus/query/_meta/test/querymetrics_instant_vector.json

@@ -0,0 +1,24 @@
+{
+    "status" : "success",
+    "data" : {
+        "resultType" : "vector",
+        "result" : [
+            {
+                "metric" : {
+                    "__name__" : "up",
+                    "job" : "prometheus",
+                    "instance" : "localhost:9090"
+                },
+                "value": [ 1435781451.781, "1.0" ]
+            },
+            {
+                "metric" : {
+                    "__name__" : "up",
+                    "job" : "node",
+                    "instance" : "localhost:9100"
+                },
+                "value" : [ 1435781451.781, "1.19" ]
+            }
+        ]
+    }
+}

metricbeat/module/prometheus/query/_meta/test/querymetrics_range_vector.json

@@ -0,0 +1,32 @@
+{
+    "status" : "success",
+    "data" : {
+        "resultType" : "matrix",
+        "result" : [
+            {
+                "metric" : {
+                    "__name__" : "up",
+                    "job" : "prometheus",
+                    "instance" : "localhost:9090"
+                },
+                "values" : [
+                    [ 1435781430.781, "1" ],
+                    [ 1435781445.781, "1" ],
+                    [ 1435781460.781, "1" ]
+                ]
+            },
+            {
+                "metric" : {
+                    "__name__" : "up",
+                    "job" : "node",
+                    "instance" : "localhost:9091"
+                },
+                "values" : [
+                    [ 1435781430.781, "0" ],
+                    [ 1435781445.781, "0" ],
+                    [ 1435781460.781, "1" ]
+                ]
+            }
+        ]
+    }
+}

metricbeat/module/prometheus/query/_meta/test/querymetrics_scalar.json

@@ -0,0 +1,10 @@
+{
+    "status":"success",
+    "data":{
+        "resultType":"scalar",
+        "result":[
+            1584628364.185,
+            "100.4"
+        ]
+    }
+}