Add OpenMetrics Metricbeat module (elastic#16596)

(cherry picked from commit 3b35d0b)

CHANGELOG.next.asciidoc

@@ -275,6 +275,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
 - Add Load Balancing metricset to GCP {pull}15559[15559]
 - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
+- Add OpenMetrics Metricbeat module {pull}16596[16596]
+- Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671]
 - Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 - Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671]
 

metricbeat/docs/fields.asciidoc

@@ -60,6 +60,7 @@ grouped in the following categories:
 * <<exported-fields-mysql>>
 * <<exported-fields-nats>>
 * <<exported-fields-nginx>>
+* <<exported-fields-openmetrics>>
 * <<exported-fields-oracle>>
 * <<exported-fields-php_fpm>>
 * <<exported-fields-postgresql>>
@@ -29048,6 +29049,40 @@ type: long
 
 --
 
+[[exported-fields-openmetrics]]
+== Openmetrics fields
+
+Openmetrics module
+
+
+
+[float]
+=== openmetrics
+
+`openmetrics` contains metrics from endpoints that are following Openmetrics format.
+
+
+
+*`openmetrics.labels.*`*::
++
+--
+Prometheus metric labels
+
+
+type: object
+
+--
+
+*`openmetrics.metrics.*`*::
++
+--
+Prometheus metric
+
+
+type: object
+
+--
+
 [[exported-fields-oracle]]
 == Oracle fields
 

metricbeat/docs/modules/openmetrics.asciidoc

@@ -0,0 +1,67 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-module-openmetrics]]
+[role="xpack"]
+== Openmetrics module
+
+beta[]
+
+This module periodically fetches metrics from endpoints following https://openmetrics.io/[Openmetrics] format.
+
+[float]
+=== Filtering metrics
+
+In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings:
+
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ["localhost:9090"]
+  metrics_path: /metrics
+  metrics_filters:
+    include: ["node_filesystem_*"]
+    exclude: ["node_filesystem_device_*", "^node_filesystem_readonly$"]
+-------------------------------------------------------------------------------------
+
+The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*`
+and are not `node_filesystem_readonly` metric.
+
+
+[float]
+=== Example configuration
+
+The Openmetrics module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ['localhost:9090']
+
+  # This module uses the Prometheus collector metricset, all
+  # the options for this metricset are also available here.
+  metrics_path: /metrics
+  metrics_filters:
+    include: []
+    exclude: []
+----
+
+This module supports TLS connections when using `ssl` config field, as described in <<configuration-ssl>>.
+It also supports the options described in <<module-http-config-options>>.
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-openmetrics-collector,collector>>
+
+include::openmetrics/collector.asciidoc[]
+

metricbeat/docs/modules/openmetrics/collector.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-openmetrics-collector]]
+=== Openmetrics collector metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/openmetrics/collector/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-openmetrics,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/openmetrics/collector/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -190,6 +190,8 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-nats-subscriptions,subscriptions>>   
 |<<metricbeat-module-nginx,Nginx>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-nginx-stubstatus,stubstatus>>   
+|<<metricbeat-module-openmetrics,Openmetrics>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+.1+| .1+|  |<<metricbeat-metricset-openmetrics-collector,collector>> beta[]  
 |<<metricbeat-module-oracle,Oracle>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .2+| .2+|  |<<metricbeat-metricset-oracle-performance,performance>> beta[]  
 |<<metricbeat-metricset-oracle-tablespace,tablespace>> beta[]  
@@ -305,6 +307,7 @@ include::modules/munin.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/openmetrics.asciidoc[]
 include::modules/oracle.asciidoc[]
 include::modules/php_fpm.asciidoc[]
 include::modules/postgresql.asciidoc[]

metricbeat/module/prometheus/collector/collector.go

@@ -43,7 +43,7 @@ var (
 )
 
 func init() {
-	mb.Registry.MustAddMetricSet("prometheus", "collector", New,
+	mb.Registry.MustAddMetricSet("prometheus", "collector", MetricSetBuilder("prometheus"),
 		mb.WithHostParser(hostParser),
 		mb.DefaultMetricSet(),
 	)
@@ -55,45 +55,48 @@ type MetricSet struct {
 	prometheus     p.Prometheus
 	includeMetrics []*regexp.Regexp
 	excludeMetrics []*regexp.Regexp
+	namespace      string
 }
 
-// New creates a new metricset
-func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	config := defaultConfig
-	if err := base.Module().UnpackConfig(&config); err != nil {
-		return nil, err
-	}
-	prometheus, err := p.NewPrometheusClient(base)
-	if err != nil {
-		return nil, err
-	}
+// MetricSetBuilder returns a builder function for a new Prometheus metricset using the given namespace
+func MetricSetBuilder(namespace string) func(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	return func(base mb.BaseMetricSet) (mb.MetricSet, error) {
+		config := defaultConfig
+		if err := base.Module().UnpackConfig(&config); err != nil {
+			return nil, err
+		}
+		prometheus, err := p.NewPrometheusClient(base)
+		if err != nil {
+			return nil, err
+		}
 
-	ms := &MetricSet{
-		BaseMetricSet: base,
-		prometheus:    prometheus,
-	}
-	ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics)
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to compile exclude patterns")
-	}
-	ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics)
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to compile include patterns")
-	}
+		ms := &MetricSet{
+			BaseMetricSet: base,
+			prometheus:    prometheus,
+			namespace:     namespace,
+		}
+		ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to compile exclude patterns")
+		}
+		ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to compile include patterns")
+		}
 
-	return ms, nil
+		return ms, nil
+	}
 }
 
 // Fetch fetches data and reports it
 func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	families, err := m.prometheus.GetFamilies()
-
 	eventList := map[string]common.MapStr{}
 	if err != nil {
 		m.addUpEvent(eventList, 0)
 		for _, evt := range eventList {
 			reporter.Event(mb.Event{
-				RootFields: common.MapStr{"prometheus": evt},
+				RootFields: common.MapStr{m.namespace: evt},
 			})
 		}
 		return errors.Wrap(err, "unable to decode response from prometheus endpoint")
@@ -137,7 +140,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 	// Converts hash list to slice
 	for _, e := range eventList {
 		isOpen := reporter.Event(mb.Event{
-			RootFields: common.MapStr{"prometheus": e},
+			RootFields: common.MapStr{m.namespace: e},
 		})
 		if !isOpen {
 			break

x-pack/metricbeat/metricbeat.reference.yml

@@ -919,6 +919,19 @@ metricbeat.modules:
   # Path to server status. Default nginx_status
   server_status_path: "nginx_status"
 
+#----------------------------- Openmetrics Module -----------------------------
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ['localhost:9090']
+
+  # This module uses the Prometheus collector metricset, all
+  # the options for this metricset are also available here.
+  metrics_path: /metrics
+  metrics_filters:
+    include: []
+    exclude: []
+
 #-------------------------------- Oracle Module --------------------------------
 - module: oracle
   metricsets: ["tablespace", "performance"]

x-pack/metricbeat/module/openmetrics/_meta/config.yml

@@ -0,0 +1,11 @@
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ['localhost:9090']
+
+  # This module uses the Prometheus collector metricset, all
+  # the options for this metricset are also available here.
+  metrics_path: /metrics
+  metrics_filters:
+    include: []
+    exclude: []

x-pack/metricbeat/module/openmetrics/_meta/docs.asciidoc

@@ -0,0 +1,21 @@
+This module periodically fetches metrics from endpoints following https://openmetrics.io/[Openmetrics] format.
+
+[float]
+=== Filtering metrics
+
+In order to filter out/in metrics one can make use of `metrics_filters.include` `metrics_filters.exclude` settings:
+
+[source,yaml]
+-------------------------------------------------------------------------------------
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ["localhost:9090"]
+  metrics_path: /metrics
+  metrics_filters:
+    include: ["node_filesystem_*"]
+    exclude: ["node_filesystem_device_*", "^node_filesystem_readonly$"]
+-------------------------------------------------------------------------------------
+
+The configuration above will include only metrics that match `node_filesystem_*` pattern and do not match `node_filesystem_device_*`
+and are not `node_filesystem_readonly` metric.

x-pack/metricbeat/module/openmetrics/_meta/fields.yml

@@ -0,0 +1,24 @@
+- key: openmetrics
+  title: 'Openmetrics'
+  release: beta
+  description: >
+    Openmetrics module
+  settings: ["http", "ssl"]
+  fields:
+    - name: openmetrics
+      type: group
+      description: >
+        `openmetrics` contains metrics from endpoints that are following Openmetrics format.
+      fields:
+        # Order is important here, labels will match first, the rest are double
+        - name: labels.*
+          type: object
+          object_type: keyword
+          description: >
+            Prometheus metric labels
+        - name: metrics.*
+          type: object
+          object_type: double
+          object_type_mapping_type: "*"
+          description: >
+            Prometheus metric

x-pack/metricbeat/module/openmetrics/collector/_meta/data.json

@@ -0,0 +1,25 @@
+{
+    "@timestamp": "2019-03-01T08:05:34.853Z",
+    "event": {
+        "dataset": "openmetrics.collector",
+        "duration": 115000,
+        "module": "openmetrics"
+    },
+    "metricset": {
+        "name": "collector",
+        "period": 10000
+    },
+    "openmetrics": {
+        "labels": {
+            "device": "br-10229e3512d9",
+            "job": "openmetrics"
+        },
+        "metrics": {
+            "node_network_carrier": 0
+        }
+    },
+    "service": {
+        "address": "127.0.0.1:55555",
+        "type": "openmetrics"
+    }
+}

x-pack/metricbeat/module/openmetrics/collector/_meta/docs.asciidoc

@@ -0,0 +1 @@
+This is the `collector` metricset of the Openmetrics module. It collects metrics from endpoints that follow Openmetrics format.

x-pack/metricbeat/module/openmetrics/collector/_meta/fields.yml

@@ -0,0 +1 @@
+- release: beta

x-pack/metricbeat/module/openmetrics/collector/_meta/testdata/config.yml

@@ -0,0 +1,4 @@
+type: http
+url: "/metrics"
+suffix: plain
+remove_fields_from_comparison: ["openmetrics.labels.instance"]

x-pack/metricbeat/module/openmetrics/collector/_meta/testdata/docs.plain

@@ -0,0 +1,11 @@
+# HELP node_network_carrier carrier value of /sys/class/net/<iface>.
+# TYPE node_network_carrier gauge
+node_network_carrier{device="br-0cb306323b90"} 0
+node_network_carrier{device="br-10229e3512d9"} 0
+node_network_carrier{device="br-210476dc4ef8"} 0
+node_network_carrier{device="br-33d819d5f834"} 0
+node_network_carrier{device="br-38425a39f36b"} 0
+node_network_carrier{device="br-38feb0aad6ab"} 0
+node_network_carrier{device="br-3a285aa5e58c"} 0
+node_network_carrier{device="br-425cb4c454a6"} 0
+node_network_carrier{device="br-4e623477470e"} 0