Cherry-pick #16455 to 7.x: [Metricbeat] Update Couchdb to v2 (#17078)

CHANGELOG.next.asciidoc

@@ -311,6 +311,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Prometheus remote write endpoint {pull}16609[16609]
 - Release STAN module as GA. {pull}16980[16980]
 - Release ActiveMQ module as GA. {issue}17047[17047] {pull}17049[17049]
+- Add support for CouchDB v2 {issue}16352[16352] {pull}16455[16455]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -5800,7 +5800,7 @@ type: long
 --
 
 [[exported-fields-couchdb]]
-== couchdb fields
+== CouchDB fields
 
 couchdb module
 
@@ -5809,7 +5809,7 @@ couchdb module
 [float]
 === couchdb
 
-
+Couchdb metrics
 
 
 [float]

metricbeat/docs/modules/couchdb.asciidoc

@@ -3,7 +3,7 @@ This file is generated! See scripts/mage/docs_collector.go
 ////
 
 [[metricbeat-module-couchdb]]
-== couchdb module
+== CouchDB module
 
 This is the couchdb module.
 
@@ -12,7 +12,10 @@ The default metricset is `server`.
 [float]
 === Compatibility
 
-The Couchdb module is tested with Couchdb 1.7.
+The Couchdb module is tested in CI with Couchdb 1.7 and 2.3. Because of the differences between v1 and v2 for exposing metrics, the path to request metrics for each version is different:
+
+* v1.* uses `[host]:5984/_stats` so the hosts of your config should just be `[host]:5984`
+* v2.* exposes metrics in various places. Local in `[host]:5986/_stats` and cluster wide in `[host]:5984/_node/[node-name]/_stats` or `[host]:5984/_node/_local/_stats`. Recommended config is `[host]:5986` to use the local path (double check that you are using port `5986`) or to use the full path on `5984`  `[host]:5984/_node/[node name or _local]/_stats`
 
 
 [float]
@@ -26,7 +29,7 @@ image::./images/metricbeat-couchdb-overview.png[]
 [float]
 === Example configuration
 
-The couchdb module supports the standard configuration options that are described
+The CouchDB module supports the standard configuration options that are described
 in <<configuration-metricbeat>>. Here is an example configuration:
 
 [source,yaml]

metricbeat/docs/modules/couchdb/server.asciidoc

@@ -3,7 +3,7 @@ This file is generated! See scripts/mage/docs_collector.go
 ////
 
 [[metricbeat-metricset-couchdb-server]]
-=== couchdb server metricset
+=== CouchDB server metricset
 
 include::../../../module/couchdb/server/_meta/docs.asciidoc[]
 

metricbeat/docs/modules_list.asciidoc

@@ -70,7 +70,7 @@ This file is generated! See scripts/mage/docs_collector.go
 .3+| .3+|  |<<metricbeat-metricset-couchbase-bucket,bucket>>   
 |<<metricbeat-metricset-couchbase-cluster,cluster>>   
 |<<metricbeat-metricset-couchbase-node,node>>   
-|<<metricbeat-module-couchdb,couchdb>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
+|<<metricbeat-module-couchdb,CouchDB>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-couchdb-server,server>>   
 |<<metricbeat-module-docker,Docker>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .9+| .9+|  |<<metricbeat-metricset-docker-container,container>>   

metricbeat/metricbeat.reference.yml

@@ -199,7 +199,7 @@ metricbeat.modules:
   hosts: ["localhost:8091"]
   enabled: true
 
-#------------------------------- Couchdb Module -------------------------------
+#------------------------------- CouchDB Module -------------------------------
 - module: couchdb
   metricsets: ["server"]
   period: 10s

metricbeat/module/couchdb/_meta/docs.asciidoc

@@ -5,7 +5,10 @@ The default metricset is `server`.
 [float]
 === Compatibility
 
-The Couchdb module is tested with Couchdb 1.7.
+The Couchdb module is tested in CI with Couchdb 1.7 and 2.3. Because of the differences between v1 and v2 for exposing metrics, the path to request metrics for each version is different:
+
+* v1.* uses `[host]:5984/_stats` so the hosts of your config should just be `[host]:5984`
+* v2.* exposes metrics in various places. Local in `[host]:5986/_stats` and cluster wide in `[host]:5984/_node/[node-name]/_stats` or `[host]:5984/_node/_local/_stats`. Recommended config is `[host]:5986` to use the local path (double check that you are using port `5986`) or to use the full path on `5984`  `[host]:5984/_node/[node name or _local]/_stats`
 
 
 [float]

metricbeat/module/couchdb/_meta/fields.yml

@@ -1,10 +1,10 @@
 - key: couchdb
-  title: "couchdb"
+  title: "CouchDB"
   description: >
     couchdb module
   release: ga
   fields:
     - name: couchdb
       type: group
-      description: >
+      description: Couchdb metrics
       fields:

metricbeat/module/couchdb/server/_meta/data.json

@@ -1,9 +1,5 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "agent": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
-    },
     "couchdb": {
         "server": {
             "couchdb": {
@@ -13,25 +9,25 @@
                 "database_writes": 0,
                 "open_databases": 0,
                 "open_os_files": 0,
-                "request_time": 96
+                "request_time": 12.667
             },
             "httpd": {
                 "bulk_requests": 0,
                 "clients_requesting_changes": 0,
-                "requests": 159,
+                "requests": 4,
                 "temporary_view_reads": 0,
                 "view_reads": 0
             },
             "httpd_request_methods": {
                 "COPY": 0,
                 "DELETE": 0,
-                "GET": 159,
+                "GET": 4,
                 "HEAD": 0,
                 "POST": 0,
                 "PUT": 0
             },
             "httpd_status_codes": {
-                "200": 159,
+                "200": 4,
                 "201": 0,
                 "202": 0,
                 "301": 0,
@@ -53,10 +49,13 @@
         "module": "couchdb"
     },
     "metricset": {
-        "name": "server"
+        "name": "server",
+        "period": 10000
     },
     "service": {
-        "address": "127.0.0.1:5984",
-        "type": "couchdb"
+        "address": "172.19.0.2:5984",
+        "id": "7021f7f554d9dd612adf13a6987d1358",
+        "type": "couchdb",
+        "version": "1.7.2"
     }
 }

metricbeat/module/couchdb/server/_meta/fields.yml

@@ -30,7 +30,7 @@
             Number of temporary view reads
 
         - name: requests
-          type: long 
+          type: long
           description: >
             Number of HTTP requests
 
@@ -139,7 +139,7 @@
           description: >
             Number of HTTP 500 Internal Server Error responses
 
-    - name: couchdb 
+    - name: couchdb
       type: group
       description: >
         couchdb statistics
@@ -175,6 +175,6 @@
             Number of authentication cache hits
 
         - name: open_os_files
-          type: long 
+          type: long
           description: >
             Number of file descriptors CouchDB has open

metricbeat/module/couchdb/server/server.go

@@ -18,11 +18,17 @@
 package server
 
 import (
-	"github.com/pkg/errors"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common"
 
 	"github.com/elastic/beats/v7/metricbeat/helper"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+
+	"github.com/pkg/errors"
 )
 
 const (
@@ -37,6 +43,10 @@ var (
 	}.Build()
 )
 
+type VersionStrategy interface {
+	MapEvent(info *CommonInfo, byt []byte) (mb.Event, error)
+}
+
 func init() {
 	mb.Registry.MustAddMetricSet("couchdb", "server", New,
 		mb.WithHostParser(hostParser),
@@ -47,7 +57,9 @@ func init() {
 // MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
-	http *helper.HTTP
+	http    *helper.HTTP
+	fetcher VersionStrategy
+	info    *CommonInfo
 }
 
 // New creates a new instance of the MetricSet.
@@ -61,10 +73,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &MetricSet{
-		base,
-		http,
-	}, nil
+
+	m := &MetricSet{
+		BaseMetricSet: base,
+		http:          http,
+		fetcher:       nil,
+	}
+	if err = m.retrieveFetcher(); err != nil {
+		m.Logger().Warnf("error trying to get CouchDB version: '%s'. Retrying on next fetch...", err.Error())
+	}
+
+	return m, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right
@@ -76,8 +95,78 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
 		return errors.Wrap(err, "error in http fetch")
 	}
 
-	event, _ := eventMapping(content)
-	reporter.Event(mb.Event{MetricSetFields: event})
+	if err = m.retrieveFetcher(); err != nil {
+		return errors.Wrapf(err, "error trying to get CouchDB version. Retrying on next fetch...")
+	}
+	event, err := m.fetcher.MapEvent(m.info, content)
+	if err != nil {
+		return errors.Wrap(err, "error trying to get couchdb data")
+	}
+	reporter.Event(event)
 
 	return nil
 }
+
+func (m *MetricSet) retrieveFetcher() (err error) {
+	if m.fetcher != nil {
+		return nil
+	}
+
+	m.info, err = m.getInfoFromCouchdbHost(m.Host())
+	if err != nil {
+		return errors.Wrap(err, "cannot start CouchDB metricbeat module")
+	}
+
+	version, err := common.NewVersion(m.info.Version)
+	if err != nil {
+		return errors.Wrap(err, "could not capture couchdb version")
+	}
+
+	m.Logger().Debugf("found couchdb version %d", version.Major)
+
+	switch version.Major {
+	case 1:
+		m.fetcher = &V1{}
+	case 2:
+		m.fetcher = &V2{}
+	default:
+		m.fetcher = nil
+	}
+
+	return
+}
+
+// CommonInfo defines the data you receive when you make a GET request to the root path of a Couchdb server
+type CommonInfo struct {
+	Version string `json:"version"`
+	UUID    string `json:"uuid"`
+}
+
+// Extract basic information from "/" path in Couchdb host
+func (m *MetricSet) getInfoFromCouchdbHost(h string) (*CommonInfo, error) {
+	c := http.DefaultClient
+	c.Timeout = 30 * time.Second
+
+	hpb := parse.URLHostParserBuilder{
+		DefaultScheme: defaultScheme,
+		DefaultPath:   "/",
+	}
+
+	hostdata, err := hpb.Build()(m.Module(), h)
+	if err != nil {
+		return nil, errors.Wrap(err, "error using host parser")
+	}
+
+	res, err := c.Get(hostdata.URI)
+	if err != nil {
+		return nil, errors.Wrap(err, "error trying to do GET request to couchdb")
+	}
+	defer res.Body.Close()
+
+	var info CommonInfo
+	if err = json.NewDecoder(res.Body).Decode(&info); err != nil {
+		return nil, errors.Wrap(err, "error trying to parse couchdb info")
+	}
+
+	return &info, nil
+}