Add the cluster_uuid field to Enterprise search stats metricset (#28287)

* Add the cluster_uuid field to Enterprise search stats metricset to make it easier to correlate with health metricset events * Make Enterprise Search module field docs prettier * Updated the docs * Move cluster_uuid to the module level to make it easier to filter those events in Stack monitoring * Add product usage metrics to the stats event * Update mb docs following the addition of new fields * Remove duplicate uuid fields now that we put the uuid at the module level (cherry picked from commit 78bcab3) # Conflicts: # metricbeat/docs/fields.asciidoc
elastic · Oct 18, 2021 · 6e9796e · 6e9796e
1 parent 055e859
commit 6e9796e
Show file tree

Hide file tree

Showing 19 changed files with 174 additions and 33 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -25100,8 +25100,16 @@ type: keyword
 
 --
 
+<<<<<<< HEAD
 [float]
 === index.summary
+=======
+[[exported-fields-enterprisesearch]]
+== Enterprise Search fields
+
+Enterprise Search module
+
+>>>>>>> 78bcab36c7 (Add the cluster_uuid field to Enterprise search stats metricset (#28287))
 
 index
 
@@ -25113,6 +25121,7 @@ index
 --
 Total number of documents in the index.
 
+<<<<<<< HEAD
 
 type: long
 
@@ -25122,13 +25131,29 @@ type: long
 +
 --
 Total number of deleted documents in the index.
+=======
+*`enterprisesearch.cluster_uuid`*::
++
+--
+Cluster UUID for the Elasticsearch cluster used as the data store for Enterprise Search.
+>>>>>>> 78bcab36c7 (Add the cluster_uuid field to Enterprise search stats metricset (#28287))
 
 
 type: long
 
 --
 
+<<<<<<< HEAD
 *`elasticsearch.index.summary.primaries.store.size.bytes`*::
+=======
+[float]
+=== health
+
+Enterprise Search health
+
+
+*`enterprisesearch.health.name`*::
+>>>>>>> 78bcab36c7 (Add the cluster_uuid field to Enterprise search stats metricset (#28287))
 +
 --
 Total size of the index in bytes.
@@ -25892,6 +25917,61 @@ type: long
 
 --
 
+<<<<<<< HEAD
+=======
+[float]
+=== product_usage
+
+Aggregate product usage statistics for the Enterprise Search deployment.
+
+
+[float]
+=== app_search
+
+App Search product usage statistics.
+
+
+*`enterprisesearch.stats.product_usage.app_search.total_engines`*::
++
+--
+Current number of App Search engines within the deployment.
+
+type: long
+
+--
+
+[float]
+=== workplace_search
+
+Workplace Search product usage statistics.
+
+
+*`enterprisesearch.stats.product_usage.workplace_search.total_org_sources`*::
++
+--
+Current number of Workplace Search org-wide content sources within the deployment.
+
+type: long
+
+--
+
+*`enterprisesearch.stats.product_usage.workplace_search.total_private_sources`*::
++
+--
+Current number of Workplace Search private content sources within the deployment.
+
+type: long
+
+--
+
+[[exported-fields-envoyproxy]]
+== Envoyproxy fields
+
+envoyproxy module
+
+
+
+>>>>>>> 78bcab36c7 (Add the cluster_uuid field to Enterprise search stats metricset (#28287))
 [float]
 === jvm
 

diff --git a/metricbeat/docs/modules/enterprisesearch.asciidoc b/metricbeat/docs/modules/enterprisesearch.asciidoc
@@ -6,7 +6,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 [[metricbeat-module-enterprisesearch]]
 [role="xpack"]
-== enterprisesearch module
+== Enterprise Search module
 
 beta[]
 
@@ -24,7 +24,7 @@ The Enterprise Search module requires a set of credentials (a username and a pas
 [float]
 === Example configuration
 
-The enterprisesearch module supports the standard configuration options that are described
+The Enterprise Search module supports the standard configuration options that are described
 in <<configuration-metricbeat>>. Here is an example configuration:
 
 [source,yaml]

diff --git a/metricbeat/docs/modules/enterprisesearch/health.asciidoc b/metricbeat/docs/modules/enterprisesearch/health.asciidoc
@@ -4,7 +4,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 [[metricbeat-metricset-enterprisesearch-health]]
 [role="xpack"]
-=== enterprisesearch health metricset
+=== Enterprise Search health metricset
 
 beta[]
 

diff --git a/metricbeat/docs/modules/enterprisesearch/stats.asciidoc b/metricbeat/docs/modules/enterprisesearch/stats.asciidoc
@@ -4,7 +4,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 [[metricbeat-metricset-enterprisesearch-stats]]
 [role="xpack"]
-=== enterprisesearch stats metricset
+=== Enterprise Search stats metricset
 
 beta[]
 

diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -107,7 +107,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-elasticsearch-node_stats,node_stats>>   
 |<<metricbeat-metricset-elasticsearch-pending_tasks,pending_tasks>>   
 |<<metricbeat-metricset-elasticsearch-shard,shard>>   
-|<<metricbeat-module-enterprisesearch,enterprisesearch>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
+|<<metricbeat-module-enterprisesearch,Enterprise Search>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .2+| .2+|  |<<metricbeat-metricset-enterprisesearch-health,health>> beta[]  
 |<<metricbeat-metricset-enterprisesearch-stats,stats>> beta[]  
 |<<metricbeat-module-envoyproxy,Envoyproxy>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  

diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -513,7 +513,7 @@ metricbeat.modules:
   #xpack.enabled: false
   #scope: node
 
-#--------------------------- Enterprisesearch Module ---------------------------
+#-------------------------- Enterprise Search Module --------------------------
 - module: enterprisesearch
   metricsets: ["health", "stats"]
   enabled: true

diff --git a/x-pack/metricbeat/module/enterprisesearch/_meta/fields.yml b/x-pack/metricbeat/module/enterprisesearch/_meta/fields.yml
@@ -1,5 +1,5 @@
 - key: enterprisesearch
-  title: "enterprisesearch"
+  title: "Enterprise Search"
   release: beta
   settings: ["ssl", "http"]
   description: >
@@ -10,3 +10,6 @@
       description: >
         `enterprisesearch` contains metrics and health information for Enterprise Search
       fields:
+        - name: cluster_uuid
+          type: keyword
+          description: Cluster UUID for the Elasticsearch cluster used as the data store for Enterprise Search.
diff --git a/x-pack/metricbeat/module/enterprisesearch/fields.go b/x-pack/metricbeat/module/enterprisesearch/fields.go
diff --git a/x-pack/metricbeat/module/enterprisesearch/health/_meta/docs.asciidoc b/x-pack/metricbeat/module/enterprisesearch/health/_meta/docs.asciidoc
@@ -1,3 +1,3 @@
-This is the health metricset of the module enterprisesearch.
+This is the `health` metricset of the Enterprise Search module.
 
 This metricset allows users to fetch and ingest Enterprise Search solution health information from the https://www.elastic.co/guide/en/enterprise-search/current/monitoring-apis.html#health-api[Health API].
diff --git a/x-pack/metricbeat/module/enterprisesearch/health/_meta/fields.yml b/x-pack/metricbeat/module/enterprisesearch/health/_meta/fields.yml
@@ -3,10 +3,6 @@
   release: beta
   description: Enterprise Search health
   fields:
-    - name: cluster_uuid
-      type: keyword
-      description: Elasticsearch Cluster UUID used by Enterprise Search
-
     - name: name
       type: keyword
       description: Host name for the Enterprise Search node

diff --git a/x-pack/metricbeat/module/enterprisesearch/health/_meta/testdata/health.json-expected.json b/x-pack/metricbeat/module/enterprisesearch/health/_meta/testdata/health.json-expected.json
@@ -1,8 +1,8 @@
 [
     {
         "enterprisesearch": {
+            "cluster_uuid": "MTD6fXYGTZylTMoW5AG5JA",
             "health": {
-                "cluster_uuid": "MTD6fXYGTZylTMoW5AG5JA",
                 "crawler": {
                     "workers": {
                         "active": 0,

diff --git a/x-pack/metricbeat/module/enterprisesearch/health/data.go b/x-pack/metricbeat/module/enterprisesearch/health/data.go
@@ -13,12 +13,12 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	s "github.com/elastic/beats/v7/libbeat/common/schema"
 	c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface"
+	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
 var (
 	schema = s.Schema{
-		"cluster_uuid": c.Str("cluster_uuid"), // This is going to be included in 7.16+
-		"name":         c.Str("name"),
+		"name": c.Str("name"),
 
 		"version": c.Dict("version", s.Schema{
 			"number":     c.Str("number"),
@@ -73,14 +73,21 @@ var (
 	}
 )
 
-func eventMapping(input []byte) (common.MapStr, error) {
+func eventMapping(report mb.ReporterV2, input []byte) error {
 	var data map[string]interface{}
 	err := json.Unmarshal(input, &data)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	var errs multierror.Errors
 
+	// All events need to have a cluster_uuid to work with Stack Monitoring
+	event := mb.Event{
+		ModuleFields:    common.MapStr{},
+		MetricSetFields: common.MapStr{},
+	}
+	event.ModuleFields.Put("cluster_uuid", data["cluster_uuid"])
+
 	// Collect process info in a form ready for mapping
 	process := make(map[string]interface{})
 	process["filebeat"] = data["filebeat"]
@@ -108,10 +115,12 @@ func eventMapping(input []byte) (common.MapStr, error) {
 	// Set the process info we have collected
 	data["process"] = process
 
-	dataFields, err := schema.Apply(data)
+	event.MetricSetFields, err = schema.Apply(data)
 	if err != nil {
 		errs = append(errs, errors.Wrap(err, "failure to apply health schema"))
+	} else {
+		report.Event(event)
 	}
 
-	return dataFields, errs.Err()
+	return errs.Err()
 }
diff --git a/x-pack/metricbeat/module/enterprisesearch/health/health.go b/x-pack/metricbeat/module/enterprisesearch/health/health.go
@@ -63,12 +63,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 		return errors.Wrap(err, "error in fetch")
 	}
 
-	event, err := eventMapping(content)
+	err = eventMapping(report, content)
 	if err != nil {
 		return errors.Wrap(err, "error converting event")
 	}
 
-	report.Event(mb.Event{MetricSetFields: event})
-
 	return nil
 }
diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/enterprisesearch/stats/_meta/docs.asciidoc
@@ -1,3 +1,3 @@
-This is the stats metricset of the module enterprisesearch.
+This is the `stats` metricset of the Enterprise Search module.
 
 This metricset allows users to fetch and ingest Enterprise Search solution statistics information from the https://www.elastic.co/guide/en/enterprise-search/current/monitoring-apis.html#stats-api[Stats API].
diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/_meta/fields.yml b/x-pack/metricbeat/module/enterprisesearch/stats/_meta/fields.yml
@@ -18,7 +18,7 @@
             - name: working
               type: long
               description: Number of connectors jobs currently being processed.
-            
+
             - name: job_types
               type: group
               description: Breakdown of connectors jobs by types.
@@ -218,15 +218,39 @@
         - name: engine_destroyer.count
           type: long
           description: Total number of jobs processed via the engine_destroyer queue since the start of the instance.
-          
+
         - name: mailer.count
           type: long
           description: Total number of jobs processed via the mailer queue since the start of the instance.
 
         - name: process_crawl.count
           type: long
           description: Total number of jobs processed via the process_crawl queue since the start of the instance.
-          
+
         - name: failed.count
           type: long
           description: Total number of jobs waiting in the failed queue.
+
+    - name: product_usage
+      type: group
+      description: Aggregate product usage statistics for the Enterprise Search deployment.
+      fields:
+        - name: app_search
+          type: group
+          description: App Search product usage statistics.
+          fields:
+          - name: total_engines
+            type: long
+            description: Current number of App Search engines within the deployment.
+
+        - name: workplace_search
+          type: group
+          description: Workplace Search product usage statistics.
+          fields:
+            - name: total_org_sources
+              type: long
+              description: Current number of Workplace Search org-wide content sources within the deployment.
+
+            - name: total_private_sources
+              type: long
+              description: Current number of Workplace Search private content sources within the deployment.
diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/_meta/testdata/stats.json b/x-pack/metricbeat/module/enterprisesearch/stats/_meta/testdata/stats.json
@@ -1,4 +1,5 @@
 {
+  "cluster_uuid": "s7QHFAuAQmqeqnGZmAqXIA",
   "http": {
     "connections": {
       "current": 1,

diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/_meta/testdata/stats.json-expected.json b/x-pack/metricbeat/module/enterprisesearch/stats/_meta/testdata/stats.json-expected.json
@@ -1,6 +1,7 @@
 [
     {
         "enterprisesearch": {
+            "cluster_uuid": "s7QHFAuAQmqeqnGZmAqXIA",
             "stats": {
                 "connectors": {
                     "job_store": {
@@ -75,6 +76,15 @@
                         "5xx": 5
                     }
                 },
+                "product_usage": {
+                    "app_search": {
+                        "total_engines": 1
+                    },
+                    "workplace_search": {
+                        "total_org_sources": 2,
+                        "total_private_sources": 3
+                    }
+                },
                 "queues": {
                     "engine_destroyer": {
                         "count": 1