Task/use etcd metrics endpoint (#11280)

Add Etcd V3 metrics from the /metrics endpoint

metricbeat/docs/fields.asciidoc

@@ -9501,6 +9501,16 @@ etcd Module
 
 
 
+*`etcd.api_version`*::
++
+--
+type: keyword
+
+Etcd API version for metrics retrieval
+
+
+--
+
 [float]
 == leader fields
 
@@ -9582,6 +9592,212 @@ type: keyword
 
 ID of actual leader
 
+--
+
+[float]
+== server fields
+
+Server metrics from the Etcd V3 /metrics endpoint
+
+
+
+*`etcd.server.has_leader`*::
++
+--
+type: byte
+
+Whether a leader exists in the cluster
+
+
+--
+
+*`etcd.server.leader_changes.count`*::
++
+--
+type: long
+
+Number of leader changes seen at the cluster
+
+
+--
+
+*`etcd.server.proposals_committed.count`*::
++
+--
+type: long
+
+Number of consensus proposals commited
+
+
+--
+
+*`etcd.server.proposals_pending.count`*::
++
+--
+type: long
+
+Number of consensus proposals pending
+
+
+--
+
+*`etcd.server.proposals_failed.count`*::
++
+--
+type: long
+
+Number of consensus proposals failed
+
+
+--
+
+*`etcd.server.grpc_started.count`*::
++
+--
+type: long
+
+Number of sent gRPC requests
+
+
+--
+
+*`etcd.server.grpc_handled.count`*::
++
+--
+type: long
+
+Number of received gRPC requests
+
+
+--
+
+[float]
+== disk fields
+
+Disk metrics from the Etcd V3 /metrics endpoint
+
+
+
+*`etcd.disk.mvcc_db_total_size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Size of stored data at MVCC
+
+
+--
+
+*`etcd.disk.wal_fsync_duration.ns.bucket.*`*::
++
+--
+type: object
+
+Latency for writing ahead logs to disk
+
+
+--
+
+*`etcd.disk.wal_fsync_duration.ns.count`*::
++
+--
+type: long
+
+Write ahead logs count
+
+
+--
+
+*`etcd.disk.wal_fsync_duration.ns.sum`*::
++
+--
+type: long
+
+Write ahead logs latency sum
+
+
+--
+
+*`etcd.disk.backend_commit_duration.ns.bucket.*`*::
++
+--
+type: object
+
+Latency for writing backend changes to disk
+
+
+--
+
+*`etcd.disk.backend_commit_duration.ns.count`*::
++
+--
+type: long
+
+Backend commits count
+
+
+--
+
+*`etcd.disk.backend_commit_duration.ns.sum`*::
++
+--
+type: long
+
+Backend commits latency sum
+
+
+--
+
+[float]
+== memory fields
+
+Memory metrics from the Etcd V3 /metrics endpoint
+
+
+
+*`etcd.memory.go_memstats_alloc.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Memory allocated bytes as of MemStats Go
+
+
+--
+
+[float]
+== network fields
+
+Network metrics from the Etcd V3 /metrics endpoint
+
+
+
+*`etcd.network.client_grpc_sent.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+gRPC sent bytes total
+
+
+--
+
+*`etcd.network.client_grpc_received.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+gRPC received bytes total
+
+
 --
 
 [float]

metricbeat/docs/modules/etcd.asciidoc

@@ -5,9 +5,13 @@ This file is generated! See scripts/docs_collector.py
 [[metricbeat-module-etcd]]
 == Etcd module
 
-This is the Etcd Module. The Etcd module uses https://coreos.com/etcd/docs/latest/v2/api.html [Etcd v2 API] to collect metrics.
+This module targets Etcd V2 and V3.
 
-The default metricsets are `leader`, `self` and `store`.
+When using V2, metrics are collected using https://coreos.com/etcd/docs/latest/v2/api.html[Etcd v2 API].
+When using V3, metrics are retrieved from the `/metrics` endpoint as intended for https://coreos.com/etcd/docs/latest/metrics.html[Etcd v3]
+
+When using V3, metricsest are bundled into `metrics`
+When using V2, metricsets available are `leader`, `self` and `store`.
 
 [float]
 === Compatibility
@@ -40,12 +44,16 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-etcd-leader,leader>>
 
+* <<metricbeat-metricset-etcd-metrics,metrics>>
+
 * <<metricbeat-metricset-etcd-self,self>>
 
 * <<metricbeat-metricset-etcd-store,store>>
 
 include::etcd/leader.asciidoc[]
 
+include::etcd/metrics.asciidoc[]
+
 include::etcd/self.asciidoc[]
 
 include::etcd/store.asciidoc[]

metricbeat/docs/modules/etcd/metrics.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-etcd-metrics]]
+=== Etcd metrics metricset
+
+beta[]
+
+include::../../../module/etcd/metrics/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-etcd,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/etcd/metrics/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -58,7 +58,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-envoyproxy,envoyproxy>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-envoyproxy-server,server>>   
 |<<metricbeat-module-etcd,Etcd>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.3+| .3+|  |<<metricbeat-metricset-etcd-leader,leader>>   
+.4+| .4+|  |<<metricbeat-metricset-etcd-leader,leader>>   
+|<<metricbeat-metricset-etcd-metrics,metrics>> beta[]  
 |<<metricbeat-metricset-etcd-self,self>>   
 |<<metricbeat-metricset-etcd-store,store>>   
 |<<metricbeat-module-golang,Golang>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  

metricbeat/module/etcd/_meta/config.yml

@@ -3,6 +3,7 @@
   #  - leader
   #  - self
   #  - store
+  #  - metrics
   period: 10s
   hosts: ["localhost:2379"]
   #username: "user"

metricbeat/module/etcd/_meta/docs.asciidoc

@@ -1,6 +1,10 @@
-This is the Etcd Module. The Etcd module uses https://coreos.com/etcd/docs/latest/v2/api.html [Etcd v2 API] to collect metrics.
+This module targets Etcd V2 and V3.
 
-The default metricsets are `leader`, `self` and `store`.
+When using V2, metrics are collected using https://coreos.com/etcd/docs/latest/v2/api.html[Etcd v2 API].
+When using V3, metrics are retrieved from the `/metrics` endpoint as intended for https://coreos.com/etcd/docs/latest/metrics.html[Etcd v3]
+
+When using V3, metricsest are bundled into `metrics`
+When using V2, metricsets available are `leader`, `self` and `store`.
 
 [float]
 === Compatibility

metricbeat/module/etcd/_meta/fields.yml

@@ -10,3 +10,9 @@
       description: >
         `etcd` contains statistics that were read from Etcd
       fields:
+        - name: api_version
+          type: keyword
+          description: >
+            Etcd API version for metrics retrieval
+
+

metricbeat/module/etcd/leader/_meta/data.json

@@ -1,10 +1,7 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "etcd": {
+        "api_version": "2",
         "leader": {
             "followers": {},
             "leader": "8e9e05c52164694d"

metricbeat/module/etcd/leader/_meta/docs.asciidoc

@@ -1 +1,2 @@
 This is the leader metricset of the module etcd.
+This metrics is being read from the Etcd V2 endpoint and won't show any activity regarding Etcd V3.

metricbeat/module/etcd/leader/leader.go

@@ -20,6 +20,7 @@ package leader
 import (
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/metricbeat/helper"
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/metricbeat/mb/parse"
@@ -28,6 +29,7 @@ import (
 const (
 	defaultScheme = "http"
 	defaultPath   = "/v2/stats/leader"
+	apiVersion    = "2"
 )
 
 var (
@@ -76,6 +78,7 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 
 	reporter.Event(mb.Event{
 		MetricSetFields: eventMapping(content),
+		ModuleFields:    common.MapStr{"api_version": apiVersion},
 	})
 	return nil
 }