From 20c1e826f7524ac6c036348b26b14eccf3650d72 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 5 Jul 2018 19:48:46 -0700 Subject: [PATCH 01/66] WIP: Starting to create (X-Pack Monitoring) stats metricset for Kibana module --- metricbeat/helper/xpack.go | 19 ++ metricbeat/module/kibana/stats/data_xpack.go | 217 +++++++++++++++++++ metricbeat/module/kibana/stats/stats.go | 17 +- 3 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 metricbeat/helper/xpack.go create mode 100644 metricbeat/module/kibana/stats/data_xpack.go diff --git a/metricbeat/helper/xpack.go b/metricbeat/helper/xpack.go new file mode 100644 index 00000000000..990b7c79f13 --- /dev/null +++ b/metricbeat/helper/xpack.go @@ -0,0 +1,19 @@ +package helper + +import ( + "fmt" + "time" +) + +// MakeMonitoringIndexName method returns the name of the monitoring index for +// a given product { elasticsearch, kibana, logstash, beats } +func MakeMonitoringIndexName(product string) string { + if !(product == "elasticsearch" || product == "kibana" || product == "logstash" || product == "beats") { + panic("Invalid product '" + product + "'") + } + + today := time.Now().UTC().Format("2006.01.02") + const version = "6" + + return fmt.Sprintf(".monitoring-%v-%v-mb-%v", product, version, today) +} diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go new file mode 100644 index 00000000000..0e9651cac79 --- /dev/null +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -0,0 +1,217 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "encoding/json" + "time" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" +) + +var ( + schemaXPack = s.Schema{ + "concurrent_connections": c.Int("concurrent_connections"), + "os": c.Dict("os", s.Schema{ + "load": c.Dict("cpu.load_average", s.Schema{ + "1m": c.Float("1m"), + "5m": c.Float("5m"), + "15m": c.Float("15m"), + }), + "memory": c.Dict("mem", s.Schema{ + "total_in_bytes": c.Int("total_in_bytes"), + "free_in_bytes": c.Int("free_in_bytes"), + "used_in_bytes": c.Int("used_in_bytes"), + }), + "uptime_in_millis": c.Int("uptime_ms"), // TODO: Verify that this field is being sent + }), + "process": c.Dict("process", s.Schema{ + "event_loop_delay": c.Float("event_loop_delay_ms"), + "memory": c.Dict("mem", s.Schema{ + "heap": s.Object{ + "total_in_bytes": c.Int("heap_max_bytes"), + "used_in_bytes": c.Int("heap_used_bytes"), + "size_limit": c.Int("size_limit"), // TODO: Verify that this field is being sent + }, + }), + "uptime_in_millis": c.Int("uptime_ms"), + }), + "requests": c.Dict("requests", s.Schema{ + "disconnects": c.Int("disconnects"), + "total": c.Int("total"), + }), + "response_times": c.Dict("response_times", s.Schema{ + "average": c.Float("avg_ms"), + "max": c.Float("max_ms"), + }), + "sockets": c.Dict("sockets", s.Schema{ + "http": c.Dict("http", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }), + "https": c.Dict("https", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }), + }), + "kibana": c.Dict("kibana", s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "index": c.Str("index"), + "host": c.Str("host"), + "transport_address": c.Str("transport_address"), + "version": c.Str("version"), + "snapshot": c.Bool("snapshot"), + "status": c.Str("status"), + }), + "usage": c.Dict("usage", s.Schema{ + "index": c.Str("index"), + "dashboard": c.Dict("dashboard", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "visualization": c.Dict("visualization", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "search": c.Dict("search", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "index_pattern": c.Dict("index_pattern", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "graph_workspace": c.Dict("graph_workspace", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "xpack": c.Dict("xpack", s.Schema{ + "reporting": c.Dict("reporting", s.Schema{ + "available": c.Bool("available"), + "enabled": c.Bool("enabled"), + "browser_type": c.Str("browser_type"), + "_all": s.Object{ + "count": c.Int("_all"), + }, + "csv": c.Dict("csv", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "printable_pdf": c.Dict("printable_pdf", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "lastDay": c.Dict("lastDay", s.Schema{ + "_all": s.Object{ + "count": c.Int("_all"), + }, + "csv": c.Dict("csv", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "printable_pdf": c.Dict("printable_pdf", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + }, c.DictOptional), + "last7Days": c.Dict("last7Days", s.Schema{ + "_all": s.Object{ + "count": c.Int("_all"), + }, + "csv": c.Dict("csv", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + "printable_pdf": c.Dict("printable_pdf", s.Schema{ + "available": c.Bool("available"), + "total": s.Object{ + "count": c.Int("total"), + }, + }, c.DictOptional), + }, c.DictOptional), + }, c.DictOptional), + }, c.DictOptional), + }), + } +) + +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + r.Error(err) + return err + } + + kibanaStats := data["kibana_stats"].(map[string]interface{}) + kibanaStatsFields, err := schemaXPack.Apply(kibanaStats) + if err != nil { + r.Error(err) + return err + } + + process := kibanaStats["process"].(map[string]interface{}) + mem := process["mem"].(map[string]interface{}) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", mem["resident_set_size_bytes"].(int)) + + timestamp := time.Now() + kibanaStatsFields.Put("timestamp", timestamp) + + // TODO: reporting status object: dynamic statuses + their counts --- toe level, inside lastDay, inside last7days + + var event mb.Event + event.RootFields = common.MapStr{ + "cluster_uuid": data["cluster_uuid"].(string), + "timestamp": timestamp, + "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, + "type": "kibana_stats", + "kibana_stats": kibanaStatsFields, + } + + event.Index = helper.MakeMonitoringIndexName("kibana") + r.Event(event) + + return nil +} diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 67dd0f5b417..026c414c81b 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -71,5 +71,20 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - eventMapping(r, content) + config := struct { + xpack bool `config:"xpack.enabled"` + }{false} + + if err := m.Module().UnpackConfig(&config); err != nil { + r.Error(err) + return + } + + if config.xpack { + cfgwarn.Experimental("The experimental xpack.enabled flag in kibana/stats metricset is enabled.") + eventMappingXPack(r, m, content) + } else { + eventMapping(r, content) + } + } From 1dfb4da1e10e29667a78559920465a13584cccda Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 15:41:59 -0700 Subject: [PATCH 02/66] Fleshing out some TODOs --- metricbeat/module/kibana/stats/data_xpack.go | 40 ++++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 0e9651cac79..3264e1da150 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -38,19 +38,20 @@ var ( "15m": c.Float("15m"), }), "memory": c.Dict("mem", s.Schema{ - "total_in_bytes": c.Int("total_in_bytes"), - "free_in_bytes": c.Int("free_in_bytes"), - "used_in_bytes": c.Int("used_in_bytes"), + "total_in_bytes": c.Int("total_bytes"), + "free_in_bytes": c.Int("free_bytes"), + "used_in_bytes": c.Int("used_bytes"), }), - "uptime_in_millis": c.Int("uptime_ms"), // TODO: Verify that this field is being sent + "uptime_in_millis": c.Int("uptime_ms"), }), "process": c.Dict("process", s.Schema{ "event_loop_delay": c.Float("event_loop_delay_ms"), "memory": c.Dict("mem", s.Schema{ "heap": s.Object{ - "total_in_bytes": c.Int("heap_max_bytes"), - "used_in_bytes": c.Int("heap_used_bytes"), - "size_limit": c.Int("size_limit"), // TODO: Verify that this field is being sent + "total_in_bytes": c.Int("heap_max_bytes"), + "used_in_bytes": c.Int("heap_used_bytes"), + "external_in_bytes": c.Int("external_bytes"), // TODO: new field, must update monitoring-kibana template in ES x-pack plugin + "size_limit": c.Int("size_limit"), }, }), "uptime_in_millis": c.Int("uptime_ms"), @@ -137,6 +138,12 @@ var ( "count": c.Int("total"), }, }, c.DictOptional), + "status": c.Dict("status", s.Schema{ + "completed": c.Int("completed"), + "failed": c.Int("failed"), + "processing": c.Int("processing"), + "pending": c.Int("pending"), + }), "lastDay": c.Dict("lastDay", s.Schema{ "_all": s.Object{ "count": c.Int("_all"), @@ -153,6 +160,12 @@ var ( "count": c.Int("total"), }, }, c.DictOptional), + "status": c.Dict("status", s.Schema{ + "completed": c.Int("completed"), + "failed": c.Int("failed"), + "processing": c.Int("processing"), + "pending": c.Int("pending"), + }), }, c.DictOptional), "last7Days": c.Dict("last7Days", s.Schema{ "_all": s.Object{ @@ -170,6 +183,12 @@ var ( "count": c.Int("total"), }, }, c.DictOptional), + "status": c.Dict("status", s.Schema{ + "completed": c.Int("completed"), + "failed": c.Int("failed"), + "processing": c.Int("processing"), + "pending": c.Int("pending"), + }), }, c.DictOptional), }, c.DictOptional), }, c.DictOptional), @@ -185,22 +204,19 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { return err } - kibanaStats := data["kibana_stats"].(map[string]interface{}) - kibanaStatsFields, err := schemaXPack.Apply(kibanaStats) + kibanaStatsFields, err := schemaXPack.Apply(data) if err != nil { r.Error(err) return err } - process := kibanaStats["process"].(map[string]interface{}) + process := data["process"].(map[string]interface{}) mem := process["mem"].(map[string]interface{}) kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", mem["resident_set_size_bytes"].(int)) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) - // TODO: reporting status object: dynamic statuses + their counts --- toe level, inside lastDay, inside last7days - var event mb.Event event.RootFields = common.MapStr{ "cluster_uuid": data["cluster_uuid"].(string), From 9bf3354944d69084934075f3835469402d6311fe Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 17:41:33 -0700 Subject: [PATCH 03/66] Marking optional fields --- metricbeat/module/kibana/stats/data_xpack.go | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 3264e1da150..d65c157251b 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -57,8 +57,8 @@ var ( "uptime_in_millis": c.Int("uptime_ms"), }), "requests": c.Dict("requests", s.Schema{ - "disconnects": c.Int("disconnects"), - "total": c.Int("total"), + "disconnects": c.Int("disconnects", s.Optional), + "total": c.Int("total", s.Optional), }), "response_times": c.Dict("response_times", s.Schema{ "average": c.Float("avg_ms"), @@ -139,11 +139,11 @@ var ( }, }, c.DictOptional), "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed"), - "failed": c.Int("failed"), - "processing": c.Int("processing"), - "pending": c.Int("pending"), - }), + "completed": c.Int("completed", s.Optional), + "failed": c.Int("failed", s.Optional), + "processing": c.Int("processing", s.Optional), + "pending": c.Int("pending", s.Optional), + }, c.DictOptional), "lastDay": c.Dict("lastDay", s.Schema{ "_all": s.Object{ "count": c.Int("_all"), @@ -161,11 +161,11 @@ var ( }, }, c.DictOptional), "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed"), - "failed": c.Int("failed"), - "processing": c.Int("processing"), - "pending": c.Int("pending"), - }), + "completed": c.Int("completed", s.Optional), + "failed": c.Int("failed", s.Optional), + "processing": c.Int("processing", s.Optional), + "pending": c.Int("pending", s.Optional), + }, c.DictOptional), }, c.DictOptional), "last7Days": c.Dict("last7Days", s.Schema{ "_all": s.Object{ @@ -184,11 +184,11 @@ var ( }, }, c.DictOptional), "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed"), - "failed": c.Int("failed"), - "processing": c.Int("processing"), - "pending": c.Int("pending"), - }), + "completed": c.Int("completed", s.Optional), + "failed": c.Int("failed", s.Optional), + "processing": c.Int("processing", s.Optional), + "pending": c.Int("pending", s.Optional), + }, c.DictOptional), }, c.DictOptional), }, c.DictOptional), }, c.DictOptional), From 33bc9f5b4e9ffc426059446197d24826c8d674d6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 17:44:14 -0700 Subject: [PATCH 04/66] Rename variable to be more accurate --- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index d65c157251b..eee647de1ba 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -29,7 +29,7 @@ import ( ) var ( - schemaXPack = s.Schema{ + schemaXPackMonitoring = s.Schema{ "concurrent_connections": c.Int("concurrent_connections"), "os": c.Dict("os", s.Schema{ "load": c.Dict("cpu.load_average", s.Schema{ @@ -204,7 +204,7 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { return err } - kibanaStatsFields, err := schemaXPack.Apply(data) + kibanaStatsFields, err := schemaXPackMonitoring.Apply(data) if err != nil { r.Error(err) return err From 95f975abbab4c9893628c8bc4089c593ef491ecf Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 18:27:01 -0700 Subject: [PATCH 05/66] Simplifying schema --- metricbeat/module/kibana/stats/data.go | 89 +++++++++++++++----- metricbeat/module/kibana/stats/data_xpack.go | 68 ++++----------- 2 files changed, 84 insertions(+), 73 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 126b01c3f2b..fcb4d4a147f 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -60,30 +60,75 @@ var ( "total": c.Int("total"), }), }), - "event_loop_delay": c.Float("event_loop_delay"), - "process": c.Dict("process", s.Schema{ - "memory": c.Dict("mem", s.Schema{ - "heap": s.Object{ - "max": s.Object{ - "bytes": c.Int("heap_max_in_bytes"), - }, - "used": s.Object{ - "bytes": c.Int("heap_used_in_bytes"), - }, - }, - "resident_set_size": s.Object{ - "bytes": c.Int("resident_set_size_in_bytes"), - }, - "external": s.Object{ - "bytes": c.Int("external_in_bytes"), - }, - }), - "pid": c.Int("pid"), - "uptime": s.Object{ - "ms": c.Int("uptime_ms"), - }, + "kibana": c.Dict("kibana", s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "index": c.Str("index"), + "host": c.Str("host"), + "transport_address": c.Str("transport_address"), + "version": c.Str("version"), + "snapshot": c.Bool("snapshot"), + "status": c.Str("status"), + }), + "usage": c.Dict("usage", s.Schema{ + "index": c.Str("index"), + "dashboard": c.Dict("dashboard", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "visualization": c.Dict("visualization", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "search": c.Dict("search", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "index_pattern": c.Dict("index_pattern", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "graph_workspace": c.Dict("graph_workspace", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "xpack": c.Dict("xpack", s.Schema{ + "reporting": c.Dict("reporting", s.Schema{ + "available": c.Bool("available"), + "enabled": c.Bool("enabled"), + "browser_type": c.Str("browser_type"), + "_all": c.Int("_all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), + "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), + }, c.DictOptional), + }, c.DictOptional), }), } + + reportingCsvDict = c.Dict("csv", s.Schema{ + "available": c.Bool("available"), + "total": c.Int("total"), + }, c.DictOptional) + + reportingPrintablePdfDict = c.Dict("printable_pdf", s.Schema{ + "available": c.Bool("available"), + "total": c.Int("total"), + }, c.DictOptional) + + reportingStatusDict = c.Dict("status", s.Schema{ + "completed": c.Int("completed", s.Optional), + "failed": c.Int("failed", s.Optional), + "processing": c.Int("processing", s.Optional), + "pending": c.Int("pending", s.Optional), + }, c.DictOptional) + + reportingPeriodSchema = s.Schema{ + "_all": c.Int("_all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + } ) func eventMapping(r mb.ReporterV2, content []byte) error { diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index eee647de1ba..1cb839830c6 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -66,14 +66,10 @@ var ( }), "sockets": c.Dict("sockets", s.Schema{ "http": c.Dict("http", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }), "https": c.Dict("https", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }), }), "kibana": c.Dict("kibana", s.Schema{ @@ -89,54 +85,36 @@ var ( "usage": c.Dict("usage", s.Schema{ "index": c.Str("index"), "dashboard": c.Dict("dashboard", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "visualization": c.Dict("visualization", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "search": c.Dict("search", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "index_pattern": c.Dict("index_pattern", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "graph_workspace": c.Dict("graph_workspace", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "xpack": c.Dict("xpack", s.Schema{ "reporting": c.Dict("reporting", s.Schema{ "available": c.Bool("available"), "enabled": c.Bool("enabled"), "browser_type": c.Str("browser_type"), - "_all": s.Object{ - "count": c.Int("_all"), - }, + "_all": c.Int("_all"), "csv": c.Dict("csv", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "printable_pdf": c.Dict("printable_pdf", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "status": c.Dict("status", s.Schema{ "completed": c.Int("completed", s.Optional), @@ -145,20 +123,14 @@ var ( "pending": c.Int("pending", s.Optional), }, c.DictOptional), "lastDay": c.Dict("lastDay", s.Schema{ - "_all": s.Object{ - "count": c.Int("_all"), - }, + "_all": c.Int("_all"), "csv": c.Dict("csv", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "printable_pdf": c.Dict("printable_pdf", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "status": c.Dict("status", s.Schema{ "completed": c.Int("completed", s.Optional), @@ -168,20 +140,14 @@ var ( }, c.DictOptional), }, c.DictOptional), "last7Days": c.Dict("last7Days", s.Schema{ - "_all": s.Object{ - "count": c.Int("_all"), - }, + "_all": c.Int("_all"), "csv": c.Dict("csv", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "printable_pdf": c.Dict("printable_pdf", s.Schema{ "available": c.Bool("available"), - "total": s.Object{ - "count": c.Int("total"), - }, + "total": c.Int("total"), }, c.DictOptional), "status": c.Dict("status", s.Schema{ "completed": c.Int("completed", s.Optional), From 5f18859d4b4cd4a71454ad35e333f3eff6eb88b3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 18:28:08 -0700 Subject: [PATCH 06/66] Updating schema --- metricbeat/module/kibana/stats/data.go | 67 ++++++++++++++++++++------ 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index fcb4d4a147f..d3322ad6901 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -28,30 +28,67 @@ import ( var ( schema = s.Schema{ - "cluster_uuid": c.Str("cluster_uuid"), - "name": c.Str("name"), - "uuid": c.Str("uuid"), - "version": c.Dict("version", s.Schema{ - "number": c.Str("number"), + "cluster_uuid": c.Str("cluster_uuid"), + "concurrent_connections": c.Int("concurrent_connections"), + "os": c.Dict("os", s.Schema{ + "load": s.Object{ + "avg": c.Dict("cpu.load_average", s.Schema{ + "1m": c.Float("1m"), + "5m": c.Float("5m"), + "15m": c.Float("15m"), + }), + }, + "memory": c.Dict("mem", s.Schema{ + "total": s.Object{ + "bytes": c.Int("total_bytes"), + }, + "free": s.Object{ + "bytes": c.Int("free_bytes"), + }, + "used": s.Object{ + "bytes": c.Int("used_bytes"), + }, + }), + "uptime": s.Object{ + "ms": c.Int("uptime_ms"), + }, }), - "status": c.Dict("status", s.Schema{ - "overall": c.Dict("overall", s.Schema{ - "state": c.Str("state"), + "process": c.Dict("process", s.Schema{ + "event_loop_delay": s.Object{ + "ms": c.Float("event_loop_delay_ms"), + }, + "memory": c.Dict("mem", s.Schema{ + "heap": s.Object{ + "total": s.Object{ + "bytes": c.Int("heap_max_bytes"), + }, + "used": s.Object{ + "bytes": c.Int("heap_used_bytes"), + }, + "external": s.Object{ + "bytes": c.Int("external_bytes"), + }, + "size_limit": s.Object{ + "bytes": c.Int("size_limit"), + }, + }, }), + "uptime": s.Object{ + "bytes": c.Int("uptime_ms"), + }, + }), + "requests": c.Dict("requests", s.Schema{ + "disconnects": c.Int("disconnects", s.Optional), + "total": c.Int("total", s.Optional), }), "response_times": c.Dict("response_times", s.Schema{ "avg": s.Object{ - "ms": c.Float("avg_in_millis"), + "ms": c.Float("avg_ms"), }, "max": s.Object{ - "ms": c.Int("max_in_millis"), + "ms": c.Float("max_ms"), }, }), - "requests": c.Dict("requests", s.Schema{ - "total": c.Int("total", s.Optional), - "disconnects": c.Int("disconnects", s.Optional), - }), - "concurrent_connections": c.Int("concurrent_connections"), "sockets": c.Dict("sockets", s.Schema{ "http": c.Dict("http", s.Schema{ "total": c.Int("total"), From a965a949d1c35dfec3f50f4a1f7abd345b935081 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 18:42:23 -0700 Subject: [PATCH 07/66] Refactoring out common conversions --- metricbeat/module/kibana/stats/data.go | 124 +++++++++++-------- metricbeat/module/kibana/stats/data_xpack.go | 103 +-------------- 2 files changed, 73 insertions(+), 154 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index d3322ad6901..036b1527bd6 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -77,10 +77,7 @@ var ( "bytes": c.Int("uptime_ms"), }, }), - "requests": c.Dict("requests", s.Schema{ - "disconnects": c.Int("disconnects", s.Optional), - "total": c.Int("total", s.Optional), - }), + "requests": RequestsDict, "response_times": c.Dict("response_times", s.Schema{ "avg": s.Object{ "ms": c.Float("avg_ms"), @@ -89,59 +86,76 @@ var ( "ms": c.Float("max_ms"), }, }), - "sockets": c.Dict("sockets", s.Schema{ - "http": c.Dict("http", s.Schema{ - "total": c.Int("total"), - }), - "https": c.Dict("https", s.Schema{ - "total": c.Int("total"), - }), - }), - "kibana": c.Dict("kibana", s.Schema{ - "uuid": c.Str("uuid"), - "name": c.Str("name"), - "index": c.Str("index"), - "host": c.Str("host"), - "transport_address": c.Str("transport_address"), - "version": c.Str("version"), - "snapshot": c.Bool("snapshot"), - "status": c.Str("status"), + "sockets": SocketsDict, + "kibana": KibanaDict, + "usage": UsageDict, + } + + // RequestsDict defines how to convert the requests field + RequestsDict = c.Dict("requests", s.Schema{ + "disconnects": c.Int("disconnects", s.Optional), + "total": c.Int("total", s.Optional), + }) + + // SocketsDict defines how to convert the sockets field + SocketsDict = c.Dict("sockets", s.Schema{ + "http": c.Dict("http", s.Schema{ + "total": c.Int("total"), }), - "usage": c.Dict("usage", s.Schema{ - "index": c.Str("index"), - "dashboard": c.Dict("dashboard", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "visualization": c.Dict("visualization", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "search": c.Dict("search", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "index_pattern": c.Dict("index_pattern", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "graph_workspace": c.Dict("graph_workspace", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "xpack": c.Dict("xpack", s.Schema{ - "reporting": c.Dict("reporting", s.Schema{ - "available": c.Bool("available"), - "enabled": c.Bool("enabled"), - "browser_type": c.Str("browser_type"), - "_all": c.Int("_all"), - "csv": reportingCsvDict, - "printable_pdf": reportingPrintablePdfDict, - "status": reportingStatusDict, - "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), - "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), - }, c.DictOptional), - }, c.DictOptional), + "https": c.Dict("https", s.Schema{ + "total": c.Int("total"), }), - } + }) + + // KibanaDict defines how to convert the kibana field + KibanaDict = c.Dict("kibana", s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "index": c.Str("index"), + "host": c.Str("host"), + "transport_address": c.Str("transport_address"), + "version": c.Str("version"), + "snapshot": c.Bool("snapshot"), + "status": c.Str("status"), + }) + + // UsageDict defines how to convert the usage field + UsageDict = c.Dict("usage", s.Schema{ + "index": c.Str("index"), + "dashboard": c.Dict("dashboard", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "visualization": c.Dict("visualization", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "search": c.Dict("search", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "index_pattern": c.Dict("index_pattern", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "graph_workspace": c.Dict("graph_workspace", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ + "total": c.Int("total"), + }, c.DictOptional), + "xpack": xPackDict, + }) + + xPackDict = c.Dict("xpack", s.Schema{ + "reporting": c.Dict("reporting", s.Schema{ + "available": c.Bool("available"), + "enabled": c.Bool("enabled"), + "browser_type": c.Str("browser_type"), + "_all": c.Int("_all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), + "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), + }, c.DictOptional), + }, c.DictOptional) reportingCsvDict = c.Dict("csv", s.Schema{ "available": c.Bool("available"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 1cb839830c6..79a8c62a024 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -56,109 +56,14 @@ var ( }), "uptime_in_millis": c.Int("uptime_ms"), }), - "requests": c.Dict("requests", s.Schema{ - "disconnects": c.Int("disconnects", s.Optional), - "total": c.Int("total", s.Optional), - }), + "requests": RequestsDict), "response_times": c.Dict("response_times", s.Schema{ "average": c.Float("avg_ms"), "max": c.Float("max_ms"), }), - "sockets": c.Dict("sockets", s.Schema{ - "http": c.Dict("http", s.Schema{ - "total": c.Int("total"), - }), - "https": c.Dict("https", s.Schema{ - "total": c.Int("total"), - }), - }), - "kibana": c.Dict("kibana", s.Schema{ - "uuid": c.Str("uuid"), - "name": c.Str("name"), - "index": c.Str("index"), - "host": c.Str("host"), - "transport_address": c.Str("transport_address"), - "version": c.Str("version"), - "snapshot": c.Bool("snapshot"), - "status": c.Str("status"), - }), - "usage": c.Dict("usage", s.Schema{ - "index": c.Str("index"), - "dashboard": c.Dict("dashboard", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "visualization": c.Dict("visualization", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "search": c.Dict("search", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "index_pattern": c.Dict("index_pattern", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "graph_workspace": c.Dict("graph_workspace", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "xpack": c.Dict("xpack", s.Schema{ - "reporting": c.Dict("reporting", s.Schema{ - "available": c.Bool("available"), - "enabled": c.Bool("enabled"), - "browser_type": c.Str("browser_type"), - "_all": c.Int("_all"), - "csv": c.Dict("csv", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "printable_pdf": c.Dict("printable_pdf", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed", s.Optional), - "failed": c.Int("failed", s.Optional), - "processing": c.Int("processing", s.Optional), - "pending": c.Int("pending", s.Optional), - }, c.DictOptional), - "lastDay": c.Dict("lastDay", s.Schema{ - "_all": c.Int("_all"), - "csv": c.Dict("csv", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "printable_pdf": c.Dict("printable_pdf", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed", s.Optional), - "failed": c.Int("failed", s.Optional), - "processing": c.Int("processing", s.Optional), - "pending": c.Int("pending", s.Optional), - }, c.DictOptional), - }, c.DictOptional), - "last7Days": c.Dict("last7Days", s.Schema{ - "_all": c.Int("_all"), - "csv": c.Dict("csv", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "printable_pdf": c.Dict("printable_pdf", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional), - "status": c.Dict("status", s.Schema{ - "completed": c.Int("completed", s.Optional), - "failed": c.Int("failed", s.Optional), - "processing": c.Int("processing", s.Optional), - "pending": c.Int("pending", s.Optional), - }, c.DictOptional), - }, c.DictOptional), - }, c.DictOptional), - }, c.DictOptional), - }), + "sockets": SocketsDict, + "kibana": KibanaDict, + "usage": UsageDict, } ) From f9eafc23c257b2ee892230ffb5b785963a97ba08 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 6 Jul 2018 18:43:28 -0700 Subject: [PATCH 08/66] Updating test fixture --- .../kibana/stats/_meta/test/stats.700.json | 167 +++++++++++++----- 1 file changed, 121 insertions(+), 46 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 01ad405e54a..dbf482ae733 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -1,53 +1,128 @@ { - "cluster_uuid": "G279hqGeSDqjj_OgBq6wUw", - "name": "ruflin", - "uuid": "3a64c6a4-b758-41b4-9564-375ca3165039", - "version": { - "number": "6.4.0" - }, - "status": { - "overall": { - "state": "green" - } - }, - "response_times": { - "avg_in_millis": 401, - "max_in_millis": 411 - }, - "requests": { - "status_codes": {} - }, - "concurrent_connections": 0, - "sockets": { - "http": { - "total": 76 + "cluster_uuid":"4Yg2A7RuT0SIZBMzTE2quA", + "timestamp":"2018-07-05T18:07:12.974Z", + "kibana":{ + "uuid":"fc5ec764-19e4-4e51-86e7-3f5a2a2b7cc0", + "name":"spicy.local", + "host":"spicy.local", + "index":".kibana", + "transport_address":"spicy.local:5601", + "version":"7.0.0-alpha1", + "snapshot":false, + "status":"green" }, - "https": { - "total": 0 - } - }, - "event_loop_delay": 46.37134699523449, - "process": { - "mem": { - "heap_max_in_bytes": 171405312, - "heap_used_in_bytes": 143607728, - "resident_set_size_in_bytes": 237985792, - "external_in_bytes": 2075915 + "last_updated":"2018-07-05T18:14:12.574Z", + "collection_interval_ms":5000, + "response_times":{ + "avg_ms":31.666666666666668, + "max_ms":40 }, - "pid": 18204, - "uptime_ms": 584494 - }, - "os": { - "cpu": { - "load_average": { - "1m": 1.96044921875, - "5m": 2.00732421875, - "15m": 2.07470703125 + "requests":{ + "total":220, + "disconnects":0 + }, + "concurrent_connections":0, + "sockets":{ + "http":{ + "total":5 + }, + "https":{ + "total":0 } }, - "mem": { - "free_in_bytes": 896122880, - "total_in_bytes": 17179869184 + "process":{ + "mem":{ + "heap_max_bytes":344432640, + "heap_used_bytes":184244040, + "resident_set_size_bytes":440311808, + "size_limit":1501560832, + "external_bytes":2168139 + }, + "pid":51785, + "uptime_ms":34725, + "event_loop_delay_ms":3.7904850244522095 + }, + "os":{ + "uptime_ms":1151777000, + "cpu":{ + "load_average":{ + "1m":3.1123046875, + "5m":2.94189453125, + "15m":2.83349609375 + } + }, + "mem":{ + "free_bytes":567361536, + "total_bytes":17179869184, + "used_bytes":16612507648 + } + }, + "usage":{ + "index":".kibana", + "dashboard":{ + "total":0 + }, + "visualization":{ + "total":1 + }, + "search":{ + "total":2 + }, + "index_pattern":{ + "total":3 + }, + "graph_workspace":{ + "total":0 + }, + "timelion_sheet":{ + "total":0 + }, + "xpack":{ + "reporting":{ + "available":true, + "enabled":true, + "browser_type":"phantom", + "_all":3, + "csv":{ + "available":true, + "total":3 + }, + "printable_pdf":{ + "available":false, + "total":0 + }, + "status":{ + "completed":3 + }, + "lastDay":{ + "_all":0, + "csv":{ + "available":true, + "total":0 + }, + "printable_pdf":{ + "available":false, + "total":0 + }, + "status":{ + + } + }, + "last7Days":{ + "_all":3, + "csv":{ + "available":true, + "total":3 + }, + "printable_pdf":{ + "available":false, + "total":0 + }, + "status":{ + "completed":3 + } + } + } + } } } -} From 10d1c3325eae49d2ccd883f0d507b7f63fe54754 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 9 Jul 2018 09:20:35 -0700 Subject: [PATCH 09/66] Fixing syntax error --- metricbeat/module/kibana/stats/data_xpack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 79a8c62a024..1703369f2b9 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -56,7 +56,7 @@ var ( }), "uptime_in_millis": c.Int("uptime_ms"), }), - "requests": RequestsDict), + "requests": RequestsDict, "response_times": c.Dict("response_times", s.Schema{ "average": c.Float("avg_ms"), "max": c.Float("max_ms"), From 0d4ac923e2fc3e17d8f8e27084eb5a6063929145 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 14:04:06 -0700 Subject: [PATCH 10/66] Populating fields.yml --- metricbeat/docs/fields.asciidoc | 686 ++++++++++++++++-- .../module/kibana/stats/_meta/fields.yml | 416 +++++++++-- metricbeat/module/kibana/stats/data.go | 30 +- 3 files changed, 992 insertions(+), 140 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ebc748e4743..750f3cbb979 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6691,42 +6691,109 @@ UUID of the Elasticsearch cluster to which Kibana connects. -- -*`kibana.stats.name`*:: +*`kibana.stats.concurrent_connections`*:: + -- -type: keyword +type: long -Kibana instance name. +Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. -- -*`kibana.stats.uuid`*:: +[float] +== os fields + +Operating system metrics + + + +[float] +== load.avg fields + +Operating system load averages + + + +*`kibana.stats.os.load.avg.1m`*:: + -- -type: keyword +type: scaled_float -Kibana instance uuid. +1 minute load average -- -*`kibana.stats.version.number`*:: +*`kibana.stats.os.load.avg.5m`*:: + -- -type: keyword +type: scaled_float -Kibana version number. +5 minute load average -- -*`kibana.stats.status.overall.state`*:: +*`kibana.stats.os.load.avg.15m`*:: + -- -type: keyword +type: scaled_float -Kibana overall state. +15 minute load average + + +-- + +[float] +== memory fields + +Operating system memory metrics + + + +*`kibana.stats.os.memory.total.bytes`*:: ++ +-- +type: long + +format: bytes + +Total operating system memory available in bytes + + +-- + +*`kibana.stats.os.memory.free.bytes`*:: ++ +-- +type: long + +format: bytes + +Operating system memory free in bytes + + +-- + +*`kibana.stats.os.memory.used.bytes`*:: ++ +-- +type: long + +format: bytes + +Operating system memory used in bytes + + +-- + +*`kibana.stats.os.uptime.ms`*:: ++ +-- +type: long + +Operating system uptime in seconds -- @@ -6734,81 +6801,108 @@ Kibana overall state. [float] == process fields -Kibana process metrics. +Process metrics + + + +*`kibana.stats.process.event_loop_delay.ms`*:: ++ +-- +type: scaled_float + +Event loop delay in milliseconds +-- [float] -== mem fields +== memory.heap fields -Memory usage metrics of the Kibana instance. +Process heap metrics -*`kibana.stats.process.mem.heap.max.bytes`*:: +*`kibana.stats.process.memory.heap.total.bytes`*:: + -- type: long format: bytes -Total amount of heap memory used by V8. +Total heap allocated to process in bytes -- -*`kibana.stats.process.mem.heap.used.bytes`*:: +*`kibana.stats.process.memory.heap.used.bytes`*:: + -- type: long format: bytes -Amount of memory in use by V8. +Heap used by process in bytes -- -*`kibana.stats.process.mem.resident_set_size.bytes`*:: +*`kibana.stats.process.memory.heap.external.bytes`*:: + -- type: long format: bytes -The amount of space occupied in main memory for the process. Includes heap, code segment, and stack. +Heap used by C++ objects bound to Javascript objects managed by V8, in bytes -- -*`kibana.stats.process.mem.external.bytes`*:: +*`kibana.stats.process.memory.heap.size_limit.bytes`*:: + -- type: long format: bytes -Memory usage of C++ objects bound to JavaScript objects managed by V8. +Max. old space size allocated to Node.js process, in bytes -- -*`kibana.stats.process.pid`*:: +*`kibana.stats.process.memory.heap.uptime.ms`*:: + -- type: long -Process ID of the Kibana instance. +Uptime of process in milliseconds -- -*`kibana.stats.process.uptime.ms`*:: +[float] +== requests fields + +Request count metrics + + + +*`kibana.stats.requests.disconnects`*:: + -- type: long -Amount of time that the Kibana process has been running in milliseconds. +Number of requests that were disconnected + + +-- + +*`kibana.stats.requests.total`*:: ++ +-- +type: long + +Total number of requests -- @@ -6816,16 +6910,16 @@ Amount of time that the Kibana process has been running in milliseconds. [float] == response_times fields -HTTP Server response time metrics +Response times metrics *`kibana.stats.response_times.avg.ms`*:: + -- -type: long +type: scaled_float -Accumulated averages for response times, for all responses in a 5-second time window. +Average response time in milliseconds -- @@ -6833,107 +6927,591 @@ Accumulated averages for response times, for all responses in a 5-second time wi *`kibana.stats.response_times.max.ms`*:: + -- +type: scaled_float + +Maximum response time in milliseconds + + +-- + +[float] +== sockets fields + +Socket connections metrics + + + +*`kibana.stats.sockets.http.total`*:: ++ +-- type: long -Accumulated maximums for response times, for all responses in a 5-second time window. +Number of HTTP connections + + +-- + +*`kibana.stats.sockets.https.total`*:: ++ +-- +type: long + +Number of HTTPS connections -- [float] -== requests fields +== kibana fields -HTTP Server request metrics +Kibana metadata -*`kibana.stats.requests.status_codes`*:: +*`kibana.stats.kibana.uuid`*:: + -- -type: object +type: keyword -Key-value pairs for each status code sent by the server, and the number of times it sent that code. +Kibana instance UUID -- -*`kibana.stats.requests.total`*:: +*`kibana.stats.kibana.name`*:: ++ +-- +type: text + +Kibana instance name + + +-- + +*`kibana.stats.kibana.index`*:: ++ +-- +type: keyword + +Name of Kibana's internal index + + +-- + +*`kibana.stats.kibana.host`*:: ++ +-- +type: keyword + +Kibana instance hostname + + +-- + +*`kibana.stats.kibana.transport_address`*:: ++ +-- +type: keyword + +Kibana server's hostname and port + + +-- + +*`kibana.stats.kibana.version`*:: ++ +-- +type: keyword + +Kibana version + + +-- + +*`kibana.stats.kibana.snapshot`*:: ++ +-- +type: boolean + +Whether the Kibana build is a snapshot build + + +-- + +*`kibana.stats.kibana.status`*:: ++ +-- +type: keyword + +Kibana instance's health status + + +-- + +[float] +== usage fields + +Kibana plugins' usage metrics + + + +[float] +== kibana fields + +Kibana core plugins' usage metrics + + + +*`kibana.stats.usage.kibana.index`*:: ++ +-- +type: keyword + +Name of Kibana's internal index + + +-- + +*`kibana.stats.usage.kibana.dashboard.total`*:: + -- type: long -Total number of requests sent by the server. +Total number of dashboards saved -- -*`kibana.stats.requests.disconnects`*:: +*`kibana.stats.usage.kibana.visualization.total`*:: + -- type: long -Total number of client disconnects encountered by the server. +Total number of visualizations saved -- -*`kibana.stats.concurrent_connections`*:: +*`kibana.stats.usage.kibana.search.total`*:: ++ +-- +type: long + +Total number of searches saved + + +-- + +*`kibana.stats.usage.kibana.index_pattern.total`*:: ++ +-- +type: long + +Total number of index patterns saved + + +-- + +*`kibana.stats.usage.kibana.graph_workspace.total`*:: + -- type: long -Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request multiple server assets at once, and they can re-use established connections. +Total number of graph workspaces saved + + +-- + +*`kibana.stats.usage.kibana.timelion_sheet.total`*:: ++ +-- +type: long + +Total number of timelion sheets saved -- [float] -== sockets fields +== reporting fields + +Kibana Reporting plugin's usage metrics + + + +*`kibana.stats.usage.reporting.available`*:: ++ +-- +type: boolean + +Whether the Reporting plugin is available + + +-- -HTTP Web Sockets metrics +*`kibana.stats.usage.reporting.enabled`*:: ++ +-- +type: boolean +Whether the Reporting plugin is enabled + + +-- + +*`kibana.stats.usage.reporting.browser_type`*:: ++ +-- +type: keyword +Type of browser being used by Reporting for screenshots + + +-- + +*`kibana.stats.usage.reporting._all`*:: ++ +-- +type: long + +Total number of reports generated across all types of reports + + +-- [float] -== http fields +== csv fields -Web Sockets over plaintext HTTP +Metrics for CSV reports -*`kibana.stats.sockets.http.total`*:: +*`kibana.stats.usage.reporting.csv.available`*:: ++ +-- +type: boolean + +Whether CSV reporting is available + + +-- + +*`kibana.stats.usage.reporting.csv.total`*:: + -- type: long -Number of HTTP web socket connections established +Total number of CSV reports generated -- [float] -== https fields +== printable_pdf fields -Web Sockets over encrypted HTTPS +Metrics for PDF reports -*`kibana.stats.sockets.https.total`*:: +*`kibana.stats.usage.reporting.printable_pdf.available`*:: ++ +-- +type: boolean + +Whether PDF reporting is available + + +-- + +*`kibana.stats.usage.reporting.printable_pdf.total`*:: ++ +-- +type: long + +Total number of PDF reports generated + + +-- + +[float] +== status fields + +Report status metrics + + + +*`kibana.stats.usage.reporting.status.completed`*:: ++ +-- +type: long + +Number of reports completed + + +-- + +*`kibana.stats.usage.reporting.status.failed`*:: ++ +-- +type: long + +Number of reports failed + + +-- + +*`kibana.stats.usage.reporting.status.processing`*:: ++ +-- +type: long + +Number of reports currently processing + + +-- + +*`kibana.stats.usage.reporting.status.pending`*:: ++ +-- +type: long + +Number of reports waiting to start processing + + +-- + +[float] +== lastDay fields + +Metrics for the last day + + + +*`kibana.stats.usage.reporting.lastDay._all`*:: ++ +-- +type: long + +Total number of reports generated across all types of reports + + +-- + +[float] +== csv fields + +Metrics for CSV reports + + + +*`kibana.stats.usage.reporting.lastDay.csv.available`*:: ++ +-- +type: boolean + +Whether CSV reporting is available + + +-- + +*`kibana.stats.usage.reporting.lastDay.csv.total`*:: ++ +-- +type: long + +Total number of CSV reports generated + + +-- + +[float] +== printable_pdf fields + +Metrics for PDF reports + + + +*`kibana.stats.usage.reporting.lastDay.printable_pdf.available`*:: ++ +-- +type: boolean + +Whether PDF reporting is available + + +-- + +*`kibana.stats.usage.reporting.lastDay.printable_pdf.total`*:: ++ +-- +type: long + +Total number of PDF reports generated + + +-- + +[float] +== status fields + +Report status metrics + + + +*`kibana.stats.usage.reporting.lastDay.status.completed`*:: ++ +-- +type: long + +Number of reports completed + + +-- + +*`kibana.stats.usage.reporting.lastDay.status.failed`*:: ++ +-- +type: long + +Number of reports failed + + +-- + +*`kibana.stats.usage.reporting.lastDay.status.processing`*:: ++ +-- +type: long + +Number of reports currently processing + + +-- + +*`kibana.stats.usage.reporting.lastDay.status.pending`*:: ++ +-- +type: long + +Number of reports waiting to start processing + + +-- + +[float] +== last7Days fields + +Metrics for the last seven days + + + +*`kibana.stats.usage.reporting.last7Days._all`*:: ++ +-- +type: long + +Total number of reports generated across all types of reports + + +-- + +[float] +== csv fields + +Metrics for CSV reports + + + +*`kibana.stats.usage.reporting.last7Days.csv.available`*:: ++ +-- +type: boolean + +Whether CSV reporting is available + + +-- + +*`kibana.stats.usage.reporting.last7Days.csv.total`*:: ++ +-- +type: long + +Total number of CSV reports generated + + +-- + +[float] +== printable_pdf fields + +Metrics for PDF reports + + + +*`kibana.stats.usage.reporting.last7Days.printable_pdf.available`*:: ++ +-- +type: boolean + +Whether PDF reporting is available + + +-- + +*`kibana.stats.usage.reporting.last7Days.printable_pdf.total`*:: ++ +-- +type: long + +Total number of PDF reports generated + + +-- + +[float] +== status fields + +Report status metrics + + + +*`kibana.stats.usage.reporting.last7Days.status.completed`*:: ++ +-- +type: long + +Number of reports completed + + +-- + +*`kibana.stats.usage.reporting.last7Days.status.failed`*:: ++ +-- +type: long + +Number of reports failed + + +-- + +*`kibana.stats.usage.reporting.last7Days.status.processing`*:: + -- type: long -Number of HTTPS web socket connections established +Number of reports currently processing -- -*`kibana.stats.event_loop_delay`*:: +*`kibana.stats.usage.reporting.last7Days.status.pending`*:: + -- type: long -Node event loop delay calculated with internal benchmarking. +Number of reports waiting to start processing -- diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 6360ae46948..a7286255cbf 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -8,118 +8,390 @@ type: keyword description: > UUID of the Elasticsearch cluster to which Kibana connects. - - name: name - type: keyword - description: > - Kibana instance name. - - name: uuid - type: keyword - description: > - Kibana instance uuid. - - name: version.number - type: keyword + - name: concurrent_connections + type: long description: > - Kibana version number. - - name: status.overall.state - type: keyword + Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. + - name: os + type: group description: > - Kibana overall state. + Operating system metrics + fields: + - name: load.avg + type: group + description: > + Operating system load averages + fields: + - name: 1m + type: scaled_float + description: > + 1 minute load average + - name: 5m + type: scaled_float + description: > + 5 minute load average + - name: 15m + type: scaled_float + description: > + 15 minute load average + - name: memory + type: group + description: > + Operating system memory metrics + fields: + - name: total.bytes + type: long + format: bytes + description: > + Total operating system memory available in bytes + - name: free.bytes + type: long + format: bytes + description: > + Operating system memory free in bytes + - name: used.bytes + type: long + format: bytes + description: > + Operating system memory used in bytes + - name: uptime.ms + type: long + description: > + Operating system uptime in seconds - name: process type: group description: > - Kibana process metrics. + Process metrics fields: - - name: mem - type: group + - name: event_loop_delay.ms + type: scaled_float + description: > + Event loop delay in milliseconds + - name: memory.heap + type: group + description: > + Process heap metrics + fields: + - name: total.bytes + type: long + format: bytes description: > - Memory usage metrics of the Kibana instance. - fields: - - name: heap.max.bytes - type: long - format: bytes - description: > - Total amount of heap memory used by V8. - - name: heap.used.bytes - type: long - format: bytes - description: > - Amount of memory in use by V8. - - name: resident_set_size.bytes - type: long - format: bytes - description: > - The amount of space occupied in main memory for the process. Includes heap, code segment, and stack. - - name: external.bytes - type: long - format: bytes - description: > - Memory usage of C++ objects bound to JavaScript objects managed by V8. - - name: pid + Total heap allocated to process in bytes + - name: used.bytes type: long + format: bytes description: > - Process ID of the Kibana instance. + Heap used by process in bytes + - name: external.bytes + type: long + format: bytes + description: > + Heap used by C++ objects bound to Javascript objects managed by V8, in bytes + - name: size_limit.bytes + type: long + format: bytes + description: > + Max. old space size allocated to Node.js process, in bytes - name: uptime.ms type: long description: > - Amount of time that the Kibana process has been running in milliseconds. - - name: response_times + Uptime of process in milliseconds + - name: requests type: group description: > - HTTP Server response time metrics + Request count metrics fields: - - name: avg.ms + - name: disconnects type: long description: > - Accumulated averages for response times, for all responses in a 5-second time window. - - name: max.ms + Number of requests that were disconnected + - name: total type: long description: > - Accumulated maximums for response times, for all responses in a 5-second time window. - - name: requests + Total number of requests + - name: response_times type: group description: > - HTTP Server request metrics + Response times metrics fields: - - name: status_codes - type: object + - name: avg.ms + type: scaled_float description: > - Key-value pairs for each status code sent by the server, and the number of times it sent that code. - - name: total + Average response time in milliseconds + - name: max.ms + type: scaled_float + description: > + Maximum response time in milliseconds + - name: sockets + type: group + description: > + Socket connections metrics + fields: + - name: http.total type: long description: > - Total number of requests sent by the server. - - name: disconnects + Number of HTTP connections + - name: https.total type: long description: > - Total number of client disconnects encountered by the server. - - name: concurrent_connections - type: long + Number of HTTPS connections + - name: kibana + type: group description: > - Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request multiple server assets at once, and they can re-use established connections. - - name: sockets + Kibana metadata + fields: + - name: uuid + type: keyword + description: > + Kibana instance UUID + - name: name + type: text + description: > + Kibana instance name + - name: index + type: keyword + description: > + Name of Kibana's internal index + - name: host + type: keyword + description: > + Kibana instance hostname + - name: transport_address + type: keyword + description: > + Kibana server's hostname and port + - name: version + type: keyword + description: > + Kibana version + - name: snapshot + type: boolean + description: > + Whether the Kibana build is a snapshot build + - name: status + type: keyword + description: > + Kibana instance's health status + - name: usage type: group description: > - HTTP Web Sockets metrics + Kibana plugins' usage metrics fields: - - name: http + - name: kibana + type: group + description: > + Kibana core plugins' usage metrics + fields: + - name: index + type: keyword + description: > + Name of Kibana's internal index + - name: dashboard.total + type: long + description: > + Total number of dashboards saved + - name: visualization.total + type: long + description: > + Total number of visualizations saved + - name: search.total + type: long + description: > + Total number of searches saved + - name: index_pattern.total + type: long + description: > + Total number of index patterns saved + - name: graph_workspace.total + type: long + description: > + Total number of graph workspaces saved + - name: timelion_sheet.total + type: long + description: > + Total number of timelion sheets saved + - name: reporting + type: group + description: > + Kibana Reporting plugin's usage metrics + fields: + - name: available + type: boolean + description: > + Whether the Reporting plugin is available + - name: enabled + type: boolean + description: > + Whether the Reporting plugin is enabled + - name: browser_type + type: keyword + description: > + Type of browser being used by Reporting for screenshots + - name: _all + type: long + description: > + Total number of reports generated across all types of reports + - name: csv type: group description: > - Web Sockets over plaintext HTTP + Metrics for CSV reports fields: + - name: available + type: boolean + description: > + Whether CSV reporting is available + - name: total + type: long + description: > + Total number of CSV reports generated + - name: printable_pdf + type: group + description: > + Metrics for PDF reports + fields: + - name: available + type: boolean + description: > + Whether PDF reporting is available + - name: total + type: long + description: > + Total number of PDF reports generated + - name: status + type: group + description: > + Report status metrics + fields: + - name: completed + type: long + description: > + Number of reports completed + - name: failed + type: long + description: > + Number of reports failed + - name: processing + type: long + description: > + Number of reports currently processing + - name: pending + type: long + description: > + Number of reports waiting to start processing + - name: lastDay + type: group + description: > + Metrics for the last day + fields: + - name: _all + type: long + description: > + Total number of reports generated across all types of reports + - name: csv + type: group + description: > + Metrics for CSV reports + fields: + - name: available + type: boolean + description: > + Whether CSV reporting is available + - name: total + type: long + description: > + Total number of CSV reports generated + - name: printable_pdf + type: group + description: > + Metrics for PDF reports + fields: + - name: available + type: boolean + description: > + Whether PDF reporting is available - name: total type: long description: > - Number of HTTP web socket connections established - - name: https + Total number of PDF reports generated + - name: status + type: group + description: > + Report status metrics + fields: + - name: completed + type: long + description: > + Number of reports completed + - name: failed + type: long + description: > + Number of reports failed + - name: processing + type: long + description: > + Number of reports currently processing + - name: pending + type: long + description: > + Number of reports waiting to start processing + - name: last7Days type: group description: > - Web Sockets over encrypted HTTPS + Metrics for the last seven days fields: + - name: _all + type: long + description: > + Total number of reports generated across all types of reports + - name: csv + type: group + description: > + Metrics for CSV reports + fields: + - name: available + type: boolean + description: > + Whether CSV reporting is available - name: total type: long description: > - Number of HTTPS web socket connections established - - name: event_loop_delay - type: long - description: > - Node event loop delay calculated with internal benchmarking. + Total number of CSV reports generated + - name: printable_pdf + type: group + description: > + Metrics for PDF reports + fields: + - name: available + type: boolean + description: > + Whether PDF reporting is available + - name: total + type: long + description: > + Total number of PDF reports generated + - name: status + type: group + description: > + Report status metrics + fields: + - name: completed + type: long + description: > + Number of reports completed + - name: failed + type: long + description: > + Number of reports failed + - name: processing + type: long + description: > + Number of reports currently processing + - name: pending + type: long + description: > + Number of reports waiting to start processing + + + + + diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 036b1527bd6..e9fbd3c8506 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -74,7 +74,7 @@ var ( }, }), "uptime": s.Object{ - "bytes": c.Int("uptime_ms"), + "ms": c.Int("uptime_ms"), }, }), "requests": RequestsDict, @@ -121,6 +121,11 @@ var ( // UsageDict defines how to convert the usage field UsageDict = c.Dict("usage", s.Schema{ + "kibana": kibanaUsageDict, + "reporting": reportingUsageDict, + }) + + kibanaUsageDict = c.Dict("kibana", s.Schema{ "index": c.Str("index"), "dashboard": c.Dict("dashboard", s.Schema{ "total": c.Int("total"), @@ -140,21 +145,18 @@ var ( "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ "total": c.Int("total"), }, c.DictOptional), - "xpack": xPackDict, }) - xPackDict = c.Dict("xpack", s.Schema{ - "reporting": c.Dict("reporting", s.Schema{ - "available": c.Bool("available"), - "enabled": c.Bool("enabled"), - "browser_type": c.Str("browser_type"), - "_all": c.Int("_all"), - "csv": reportingCsvDict, - "printable_pdf": reportingPrintablePdfDict, - "status": reportingStatusDict, - "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), - "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), - }, c.DictOptional), + reportingUsageDict = c.Dict("reporting", s.Schema{ + "available": c.Bool("available"), + "enabled": c.Bool("enabled"), + "browser_type": c.Str("browser_type"), + "_all": c.Int("_all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), + "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), }, c.DictOptional) reportingCsvDict = c.Dict("csv", s.Schema{ From 83e5afa7e1f335fbdac7207b758f3591e370f29f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 14:06:13 -0700 Subject: [PATCH 11/66] Updating test fixture --- .../kibana/stats/_meta/test/stats.700.json | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index dbf482ae733..5193f7bedbb 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -58,30 +58,58 @@ } }, "usage":{ - "index":".kibana", - "dashboard":{ - "total":0 - }, - "visualization":{ - "total":1 - }, - "search":{ - "total":2 - }, - "index_pattern":{ - "total":3 - }, - "graph_workspace":{ - "total":0 - }, - "timelion_sheet":{ - "total":0 + "kibana":{ + "index":".kibana", + "dashboard":{ + "total":0 + }, + "visualization":{ + "total":1 + }, + "search":{ + "total":2 + }, + "index_pattern":{ + "total":3 + }, + "graph_workspace":{ + "total":0 + }, + "timelion_sheet":{ + "total":0 + } }, - "xpack":{ - "reporting":{ + "reporting":{ + "available":true, + "enabled":true, + "browser_type":"phantom", + "_all":3, + "csv":{ "available":true, - "enabled":true, - "browser_type":"phantom", + "total":3 + }, + "printable_pdf":{ + "available":false, + "total":0 + }, + "status":{ + "completed":3 + }, + "lastDay":{ + "_all":0, + "csv":{ + "available":true, + "total":0 + }, + "printable_pdf":{ + "available":false, + "total":0 + }, + "status":{ + + } + }, + "last7Days":{ "_all":3, "csv":{ "available":true, @@ -93,34 +121,6 @@ }, "status":{ "completed":3 - }, - "lastDay":{ - "_all":0, - "csv":{ - "available":true, - "total":0 - }, - "printable_pdf":{ - "available":false, - "total":0 - }, - "status":{ - - } - }, - "last7Days":{ - "_all":3, - "csv":{ - "available":true, - "total":3 - }, - "printable_pdf":{ - "available":false, - "total":0 - }, - "status":{ - "completed":3 - } } } } From da32edfddfa00c7a76b9f8b15f7275fb0d23ce01 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 14:14:37 -0700 Subject: [PATCH 12/66] Only pass in intervalMs instead of entire metricset struct --- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- metricbeat/module/kibana/stats/stats.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 1703369f2b9..ca3c9e2faf3 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -67,7 +67,7 @@ var ( } ) -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -92,7 +92,7 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { event.RootFields = common.MapStr{ "cluster_uuid": data["cluster_uuid"].(string), "timestamp": timestamp, - "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, + "interval_ms": intervalMs, "type": "kibana_stats", "kibana_stats": kibanaStatsFields, } diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 026c414c81b..6be313c9091 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -82,7 +82,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { if config.xpack { cfgwarn.Experimental("The experimental xpack.enabled flag in kibana/stats metricset is enabled.") - eventMappingXPack(r, m, content) + intervalMs := m.Module().Config().Period.Nanoseconds() / 1000 / 1000 + eventMappingXPack(r, intervalMs, content) } else { eventMapping(r, content) } From 9be7eff3c5eb3ce884afcd922c19fe0a2962f075 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 14:14:59 -0700 Subject: [PATCH 13/66] Add unit test for data_xpack.go --- .../module/kibana/stats/data_xpack_test.go | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 metricbeat/module/kibana/stats/data_xpack_test.go diff --git a/metricbeat/module/kibana/stats/data_xpack_test.go b/metricbeat/module/kibana/stats/data_xpack_test.go new file mode 100644 index 00000000000..cefced0eb10 --- /dev/null +++ b/metricbeat/module/kibana/stats/data_xpack_test.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package stats + +import ( + "io/ioutil" + "path/filepath" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestStats(t *testing.T) { + + files, err := filepath.Glob("./_meta/test/stats.*.json") + assert.NoError(t, err) + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + err = eventMappingXPack(reporter, 10000, input) + assert.NoError(t, err, f) + assert.True(t, len(reporter.GetEvents()) >= 1, f) + assert.Equal(t, 0, len(reporter.GetErrors()), f) + } +} From b347c68c9b063cd9835194f9bbdba5d0e6c886bc Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 17:08:39 -0700 Subject: [PATCH 14/66] Fixing collisions in function names --- metricbeat/module/kibana/stats/data_test.go | 2 +- metricbeat/module/kibana/stats/data_xpack_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_test.go b/metricbeat/module/kibana/stats/data_test.go index ca01331137c..19daa556252 100644 --- a/metricbeat/module/kibana/stats/data_test.go +++ b/metricbeat/module/kibana/stats/data_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestStats(t *testing.T) { +func TestEventMapping(t *testing.T) { files, err := filepath.Glob("./_meta/test/stats.*.json") assert.NoError(t, err) diff --git a/metricbeat/module/kibana/stats/data_xpack_test.go b/metricbeat/module/kibana/stats/data_xpack_test.go index cefced0eb10..d2d2a042b58 100644 --- a/metricbeat/module/kibana/stats/data_xpack_test.go +++ b/metricbeat/module/kibana/stats/data_xpack_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestStats(t *testing.T) { +func TestEventMappingXPack(t *testing.T) { files, err := filepath.Glob("./_meta/test/stats.*.json") assert.NoError(t, err) From 6a56f54296b1dd2c336f88095941fff87ed0d98d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Jul 2018 17:09:55 -0700 Subject: [PATCH 15/66] Updating fields.go --- metricbeat/module/kibana/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index f8c9dec28a0..079a973d1f0 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsWE1v20YQvftXDHyNpVuBQocCRVqgaZAgqJz2UBTCcjkSt9oPdmcoWf31xSwpiaJJWZHpwAHKgxGRmzdvZufj7U5gjbsZrE2mvLoBYMMWZ3D7Pr24vQHIkXQ0JZvgZ/DDDQBA/RFcyCuLNwBUhMgLHfzSrGawVJbkbUSLinAGGbJAEzIbv6IZ/HlLZG//ugFYGrQ5zRLqBLxy2OIiD+9KnMEqhqps3vTwOcVpYxErpsPbPrhByPppXE04oHwOsfITNg7BIUejadpa3vV4/3TZtRlqWxFjXFSVyU8W7MmucbcNsfvtDGV5Pn9+9xOEJXCB8LNVxEYTqqiLvT3gANvC6GLvoQ7eo+YTf44s5e947BqTxhMrrzGh99sdNypdu4Leb3eDkUzwU1+5DOPoDBp4qOH7KUjKVTQNG4zK2qn8HH8LGvRkbGALyhg0EvWa7hbS5YYb1L4qgoGKaZNy6B59O0fqAmLyfEAX4g4qUqtDge/LqJM8XcbnWLeZF6jKqVMP02zH2I3qqSM2+NXAgmWITvEMzoFc4K8894GVBeVC5Vl8FYIS3zoQmEO2g9+/73O345QsfjVe/Xjwp3HFePHmEmciksnR84KQF2T+xVfj1H2BrY2iUmmEoHVVGszFQafkT+3vMsSUtk2hTeGd17bKkdJm3YEOOQLhyqHnuzTZiJVenw8NPjBGr+yrichJwYYlvH3zBkL2twwyyELlc5lzv6qNmie0wzenvFoN5/ah9z2aPk86eAHxT03vO47oJ3vLYR6Woj2mri9sz6V1LJmkb7hQ3Ca379iFIsgQvSghb/wqJZ6x1hDq4PMBARGRyuAJF4I92jD55f7+E8wxbjAeLEBbnX3hWFGb1QvFVuvKVVYx5qBk5q6QUomesKa79E4G8v49SXgVfDepo1s7tzU+D9vhLJHx8uJuOPVgXOVGduOYL/9USPxCmZKwr8yRWpctpH8Oh7juM9cF+T3uJhtlK4RSmVjHF5UuGsv7zu1ZupcUKCW36iYuv2tVuS9kAsP18lTR8r+HM4dFDLxA4tQi40hsv709fgyTyw3tTylfgaK2Rri1jAJ6LS0SYz05hjgfjnbB6ypGkRINhAm+P6N7aD9B+WOXaMsEOJWjTL4WQ/gYuGnqWQxbwkiglZcNyMFVlk1pEcjIP5XHkBLtiMjhWDaHxXU5KSKUwzFD8BoPabhL8BEnIruQWGXWUIF5G3bg5BP0Gkcu/T8wg3mNe2XdF8x9p4pnnznazOQ4BqVVxjM+cGJ+5TFjqJDhEoF2ofg6pmAK8RazZutOMqe19WdjO1zTYwYXvY67UuaXcJ5/M9Gdf2l4D5J9I/3HhlAucrRqN1b7kSGUsEGwIWGDVlY38mBruADJYzkvQIZeF07FtfGrae8tXfWsa7p5PRrrvXvmjdz/d12jMviW7rr6J8PVk+dDc4P0KC3hgnlzkXqAEYTP29pKy+DwiIanJfr5iF3I6bdGa8hemXR7fe1923nRCCM27K6APFo+J3q6fL/GaHkkddvk/gsAAP//Ii6tlQ==" + return "eJzsm99v47gRx9/zVwz2JQ93a2AfDi38UKDYXHFtcelik70+FIUxlsYWLxSpkpQT319fkPphWSYl+oecLBA/HPZsZb4fzgyHQ0r6CE+0ncMTW6LAGwDDDKc5fPin++LDDUBKOlGsMEyKOfzlBgCg+hFymZacbgB0JpVZJFKs2HoOK+TafquIE2qaw5KMNa3JGCbWeg7/+aA1//DfG4AVI57qubP6EQTm1GGxH7MtaA5rJcui/sbDs2+na0sbNLr91mcuaLL61EN1dgBFCqoUHw3LCXIyiiV61rm8P+Lm06frEia81IbUoixZundBA/tE22ep+r8NINvPt29/vwO5ApMR/MxRG5ZoQpVkjR4YCc8ZS7JmhIkUghKzN54OpRRJqRQJF2Z7IZNCe3m5FOvjYO/LfEnK4iackTDQkYAcU7KwdiSa1IbUDO6lITAZGlgq+axJaUhQgCaRQl5ywwpOoJn9JwqSpd6zaCQo+l9J2tiLmXEXO8uAWpONswEpEvrRBdxktHXmFX0sNQFpg0vOdEZp16zfb9Lvo34CRjjpXwUptPMH9FYbypv8613oS7UdDpeYznDTj88Q1iiYB83KAG5I4Zr6gCHEHeSn3KNRAeoEOaWLFZdoPBeNkgJ8gpyJ0tAeYxDlpylRfjoK5dOkLJ9iYBqUnHKpthPnUCUSyPLxJDLSIJ8tt8aTgAOVypmWKkczh9AfR7jz0aqDDIwJN8g4LjkBEwGVZhgrRfRqowjFxEKNo5ea0jeHbqFC6C14Ydf3WX4oPUB9fIJXMhZGUyJFqr0LSKFkQvpiq8iXytxJiwdt7PLPpSwWKXHchj00WI9GPfWz1QGrA07HuihnnDOfn/pVaZYRHpafs0pT4zNr+buuR24AyLlM0FBq+6A6u974ZP7FcruZu9zGI9OLISVe0el72J9/+AHk8nfbYcNSlsK5/x+4wcpM+1uOAtfVn/z25x/HR6nZH7TgLGfm1cb5K77MQPIUdIEJOaL9LLuXKc1+103oIkYVLsEjw4nA/VbVXbnqplKwvjRE9Y7hYoUY4Gu9B0lkKcxRBXlHlTLd7Nsm8NRuX9aMvtp0PZOijjT196a9qjcBWVXPxAFfIHS6kELTwsb9YgH8WlsFZ/XE+OFmPZTk5/f1f60a+dYH0PQcAwtqZ0nFl2nxfsUXlpf5EXht3ZPJE11uOj44c/sHDydFNDOmmE2V9rsJ+cvj4xcIn8Ts8+grAT0EibxHe/sQJwStOYYkgymavtmxSHkO3GDk0C3SKzUXE9qgSMgdxgUx7H+DGIZeTpxZfQavTMPAREov0/jiHqu1tuK5tYtt1ZYFRNu8ldo38gmiY5UGvWMUCl1IZRaYpupwM3ZZruok8la3XO4Q0soHATekNJNiUqyQRluPBRY6k+GYLaXkhD7ICIh/Z2QyUu4QuAZaloynwDRgq119F0Y0aMppg9ck1a3bLXKT+UR3e6rDI7Zza2HByzUT+rYyftI231ujz9xCt/cYFMUhjpfvN1izUtTZUqJKJ1tv+31vq6hB42agEd8wXSJnf6C1fjW8PdUxxOr+1NXYKjkao3IRXxRobAZcDc6pQq06hrhWWGSLZ6me3Ab8apBOF1rdMUzb2nMmxUJnROZqlI0sOFk/5G6naNdZ5pG+RPX72livS+CtPrMEtvcRpl90++xu4Q3Kt0dwwv4c7rInxQuJN3D1reOFRZlmHXncFm4dqZVgSZawORHcMa+kAp0oImG7mPBeboH8OocpFkzDmgQpd4aHiZJaA3LuxHTnqiBsojdBVv88ijsxqOaK89nnh9+CGOHZEzd/xlM0CneXpjtYG/LB6TN+fjYS9Ei0fuA7/twFPxjgQjFhLP+iSFdTh/rL3d++n1DvYN9sqDv+jAj1yM7pjBhXRbAWCC6EMRFOZF5w8o0CLuTD+4MKOaTZ3sFH5l8Cp4IKCvZuKfvanOmo6qfH+HZYvmUkkV4X8BmZm65G2mxUZoizfZwKtbnDwydhLlz8bIdjpSD1ao3PjUDjME1pOa99GGsgxlwbCR3fRgz5N3Z9iVlhIsFPbChi1pnRdIhGPK2xiG8tJkiC4QbjzSbBUa3G6yZBXMsR0XRcKPqxrUdc7Ifbj8t59bgmJKoNmRJuQDayHZnUdXFNSVxbMiXoqc3Jn+5wO0nf7m1PNG1I2CbltA7+vUvpf967lDDle5fi+bzRJHjvUk4mfO9SrgD33qVcBDSmSznqH8FJdvRrnA/V/KnmSOeXU97Y9DwsdMabmr4HtfzvEl72DdG+rrXu160f/ZlVRfLiBLX5ugb7EaoEmMkNKeR8Zv/38iGorTuxQAj89ffkh3bapb2flhBx9zfqlVy4wP3Bz5VKRzD83iuMPzI/7LFIpubheRsr5t5uPuSIWxKHH6GHCVuN7hP0wx6FV26K9uD+HwAA//9MAaRi" } From 92e161795c6561418566cc8ee8b085d25efa277e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 11 Jul 2018 16:16:15 -0700 Subject: [PATCH 16/66] Removing external_bytes field (for now) --- metricbeat/docs/fields.asciidoc | 12 ------------ metricbeat/module/kibana/fields.go | 2 +- metricbeat/module/kibana/stats/_meta/fields.yml | 5 ----- .../module/kibana/stats/_meta/test/stats.700.json | 3 +-- metricbeat/module/kibana/stats/data.go | 3 --- metricbeat/module/kibana/stats/data_xpack.go | 7 +++---- 6 files changed, 5 insertions(+), 27 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 750f3cbb979..fd54c7b4bde 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6844,18 +6844,6 @@ format: bytes Heap used by process in bytes --- - -*`kibana.stats.process.memory.heap.external.bytes`*:: -+ --- -type: long - -format: bytes - -Heap used by C++ objects bound to Javascript objects managed by V8, in bytes - - -- *`kibana.stats.process.memory.heap.size_limit.bytes`*:: diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 079a973d1f0..cf087507156 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm99v47gRx9/zVwz2JQ93a2AfDi38UKDYXHFtcelik70+FIUxlsYWLxSpkpQT319fkPphWSYl+oecLBA/HPZsZb4fzgyHQ0r6CE+0ncMTW6LAGwDDDKc5fPin++LDDUBKOlGsMEyKOfzlBgCg+hFymZacbgB0JpVZJFKs2HoOK+TafquIE2qaw5KMNa3JGCbWeg7/+aA1//DfG4AVI57qubP6EQTm1GGxH7MtaA5rJcui/sbDs2+na0sbNLr91mcuaLL61EN1dgBFCqoUHw3LCXIyiiV61rm8P+Lm06frEia81IbUoixZundBA/tE22ep+r8NINvPt29/vwO5ApMR/MxRG5ZoQpVkjR4YCc8ZS7JmhIkUghKzN54OpRRJqRQJF2Z7IZNCe3m5FOvjYO/LfEnK4iackTDQkYAcU7KwdiSa1IbUDO6lITAZGlgq+axJaUhQgCaRQl5ywwpOoJn9JwqSpd6zaCQo+l9J2tiLmXEXO8uAWpONswEpEvrRBdxktHXmFX0sNQFpg0vOdEZp16zfb9Lvo34CRjjpXwUptPMH9FYbypv8613oS7UdDpeYznDTj88Q1iiYB83KAG5I4Zr6gCHEHeSn3KNRAeoEOaWLFZdoPBeNkgJ8gpyJ0tAeYxDlpylRfjoK5dOkLJ9iYBqUnHKpthPnUCUSyPLxJDLSIJ8tt8aTgAOVypmWKkczh9AfR7jz0aqDDIwJN8g4LjkBEwGVZhgrRfRqowjFxEKNo5ea0jeHbqFC6C14Ydf3WX4oPUB9fIJXMhZGUyJFqr0LSKFkQvpiq8iXytxJiwdt7PLPpSwWKXHchj00WI9GPfWz1QGrA07HuihnnDOfn/pVaZYRHpafs0pT4zNr+buuR24AyLlM0FBq+6A6u974ZP7FcruZu9zGI9OLISVe0el72J9/+AHk8nfbYcNSlsK5/x+4wcpM+1uOAtfVn/z25x/HR6nZH7TgLGfm1cb5K77MQPIUdIEJOaL9LLuXKc1+103oIkYVLsEjw4nA/VbVXbnqplKwvjRE9Y7hYoUY4Gu9B0lkKcxRBXlHlTLd7Nsm8NRuX9aMvtp0PZOijjT196a9qjcBWVXPxAFfIHS6kELTwsb9YgH8WlsFZ/XE+OFmPZTk5/f1f60a+dYH0PQcAwtqZ0nFl2nxfsUXlpf5EXht3ZPJE11uOj44c/sHDydFNDOmmE2V9rsJ+cvj4xcIn8Ts8+grAT0EibxHe/sQJwStOYYkgymavtmxSHkO3GDk0C3SKzUXE9qgSMgdxgUx7H+DGIZeTpxZfQavTMPAREov0/jiHqu1tuK5tYtt1ZYFRNu8ldo38gmiY5UGvWMUCl1IZRaYpupwM3ZZruok8la3XO4Q0soHATekNJNiUqyQRluPBRY6k+GYLaXkhD7ICIh/Z2QyUu4QuAZaloynwDRgq119F0Y0aMppg9ck1a3bLXKT+UR3e6rDI7Zza2HByzUT+rYyftI231ujz9xCt/cYFMUhjpfvN1izUtTZUqJKJ1tv+31vq6hB42agEd8wXSJnf6C1fjW8PdUxxOr+1NXYKjkao3IRXxRobAZcDc6pQq06hrhWWGSLZ6me3Ab8apBOF1rdMUzb2nMmxUJnROZqlI0sOFk/5G6naNdZ5pG+RPX72livS+CtPrMEtvcRpl90++xu4Q3Kt0dwwv4c7rInxQuJN3D1reOFRZlmHXncFm4dqZVgSZawORHcMa+kAp0oImG7mPBeboH8OocpFkzDmgQpd4aHiZJaA3LuxHTnqiBsojdBVv88ijsxqOaK89nnh9+CGOHZEzd/xlM0CneXpjtYG/LB6TN+fjYS9Ei0fuA7/twFPxjgQjFhLP+iSFdTh/rL3d++n1DvYN9sqDv+jAj1yM7pjBhXRbAWCC6EMRFOZF5w8o0CLuTD+4MKOaTZ3sFH5l8Cp4IKCvZuKfvanOmo6qfH+HZYvmUkkV4X8BmZm65G2mxUZoizfZwKtbnDwydhLlz8bIdjpSD1ao3PjUDjME1pOa99GGsgxlwbCR3fRgz5N3Z9iVlhIsFPbChi1pnRdIhGPK2xiG8tJkiC4QbjzSbBUa3G6yZBXMsR0XRcKPqxrUdc7Ifbj8t59bgmJKoNmRJuQDayHZnUdXFNSVxbMiXoqc3Jn+5wO0nf7m1PNG1I2CbltA7+vUvpf967lDDle5fi+bzRJHjvUk4mfO9SrgD33qVcBDSmSznqH8FJdvRrnA/V/KnmSOeXU97Y9DwsdMabmr4HtfzvEl72DdG+rrXu160f/ZlVRfLiBLX5ugb7EaoEmMkNKeR8Zv/38iGorTuxQAj89ffkh3bapb2flhBx9zfqlVy4wP3Bz5VKRzD83iuMPzI/7LFIpubheRsr5t5uPuSIWxKHH6GHCVuN7hP0wx6FV26K9uD+HwAA//9MAaRi" + return "eJzsm89v67gRx+/5KwbvksvGQA6LAj4UKDZbbFFs+vCS1x6KwqClscWGIlUO5cT71xekfliWSYn+IScPiA+LrK033w9nhsMhJd3BC27n8MKXTLIbAMONwDl8+bv74ssNQIqUaF4YruQc/nwDAFD9CLlKS4E3AJQpbRaJkiu+nsOKCbLfahTICOewRGNNExrD5Zrm8O8vROLLf24AVhxFSnNn9Q4ky7HDYj9mW+Ac1lqVRf2Nh2ffTtcWGWao/dZnLmiy+tRDdXaAyRR0Ke8MzxFyNJonNOtc3h9x8+nTdQkTUZJBvShLnu5d0MC+4PZV6f5vA8j28/373x5ArcBkCL8KRoYnhEwnWaMHRsFrxpOsGWGipMTE7I2nQ6lkUmqN0oXZXsiVJC+vUHJ9HOxjmS9RW9xEcJQGOhKQsxQtrB0Jod6gnsGjMggmYwaWWr0SaoKESSCUKeSlMLwQCMTtn0yiKmnPolGg8X8lkrEXc+MudpaBEaGNswElE/zJBdxkuHXmNd6VhIBk2FJwyjDtmvX7Tfl91E/ACCf9o0DN7PwB2pLBvMm/3oW+VNvhCMXSGdv04zOENQrmQbMywDao2Rr7gCHEHeR97tGoAClhAtPFSihmPBeNkgLcQ85laXCPMYjy85QoPx+Fcj8py30MTIOSY670duIcqkQCWT6eREYZJmbLrfEk4EClcqaVzpmZQ+gfR7jz2aqDCoyJbRgXbCkQuAyoNMNYacR3G0UoJhZqHL0kTD8cuoUKobfghV3fZ/mh9AD18QleyVgYwkTJlLwLSKFVgnSxVeRrZe6kxQM3dvkXShWLFAXbhj00WI9GPfWr1QGrA07HuijnQnCfn/pVaZYhOyw/Z5WmxmfW8g9dj9wAmBAqYQZT2wfV2fXBJ/NvltvN3OU2Hpn4H7gQPOfm3cB/Z28zUCIFKliCjmjf/48qxdl/qRnUTxGBCBankeFE4H6vKpJadZ0cnHkNUd1LX6xEAXyru/NEldIcVap2VCmnZkczgad2O5Zm9NV25BU1dqSxv2vr1YMJyKqZLg/4AqGjQknChY37xQL4rbYKzuqJ8WOb9VCSn9/x/qVqcVsfQLMaDyw1ncWGvU2L9zt743mZH4HX1j2VvODlpuOTM7e/JT8popkxxWyqtN9NyN+en79C+Ixin4euBPQUJPIeeu1DnBC05oAODUuZ6Zsdi5TnKApGjqMivVJzcUmGyQTdMVUQw/43iGHw7cSZ1WfwyjQMXKb4No0vHlm11lY8t3axNaglEwHRNm8V+UY+QXSs0qB3jGaSCqXNgqWpPtymXJarOqO7pZbLHc9Z+SDgBjVxJSfFCmm09ViygjIVjtlSKYHMBxkB8a8MTYbaHY/WQMuSixQ4AWu1q+/CiIaZctrgNUl16/ZRwmQ+0d1u4/Dw6dxaWIhyzSXdVsZP2gB7a/SZm8v29F1jHOJ4+f6ANStllC0V0+lk622/720VCYhtBhrxDaeSCf4Hs9avhrenOoZY3bm5Glslh2NULuKLghmbAVeDc6pQq44hrjUrssWr0i9uA341SKcLre4Ypm3tBVdyQRmiuRplIwtO1g+52ynadZZ7pC9R/b411usSeEtnlsD2hH36RbfP7hbeoHx7lirtz+Eue1K8kHgDV99UXViUadaR523h1pFaCZZoCZsjvh3zSmmgRCNK28WE93ILJq5zmGLBCNYoUbszPJZoRQRMCCdGnauCsAltgqz+eRR3YlDNFeezX57+GcQIz564+TOeolG4uzTdwdqQD06f8fOzkaBHovUD3/HnLvjBABeaS2P5F0W6mjrUXx/++uOEegf7YUPd8WdEqEd2TmfEuCqCtUBwIYyJcKLyQqBvFHAhHz4eVMghzfbeNuP+JXAqqKBg72arr82Zjqp+rkpsh+VbRpTpdQFfGXfT1SibjdoMcbYPGjEyD+zwGZELFz/b4VgpSL1a43Mj0DhMU1rOax/GGogx10ZCx7cRQ/6NXV9iVphI8BMbiph1ZjQdohFPayziW4sJkmC4wfiwSXBUq/G+SRDXckQ0HReKfmzrERf74fbjcl49rgmJakOmhBuQjWxHJnVdXFMS15ZMCXpqc/KnB7adpG/3tieEG5S2STmtg//sUvqfzy4lTPnZpXg+HzQJPruUkwk/u5QrwH12KRcBjelSjvojOMmOfsHxqZo/1Rzp/HLKu4yeh4XOeIfR96CW/y27y7472de11v269aM/s6pIXpygNl/XYD9ClQAztUHNhJjZ/718CGrrTiwQAn/9PfmhnXZp76clRNz9jXpZFS5wf/CXSqUjGH4jFMYfmR/2WCRT8/C8jRV37/0ecsQticOP0MOErUb3Cfphj8I7N0V7cP8PAAD//+fPaFU=" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index a7286255cbf..f2997d86844 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -82,11 +82,6 @@ format: bytes description: > Heap used by process in bytes - - name: external.bytes - type: long - format: bytes - description: > - Heap used by C++ objects bound to Javascript objects managed by V8, in bytes - name: size_limit.bytes type: long format: bytes diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 5193f7bedbb..2079c89d1c6 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -35,8 +35,7 @@ "heap_max_bytes":344432640, "heap_used_bytes":184244040, "resident_set_size_bytes":440311808, - "size_limit":1501560832, - "external_bytes":2168139 + "size_limit":1501560832 }, "pid":51785, "uptime_ms":34725, diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index e9fbd3c8506..3ac3b420a0c 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -65,9 +65,6 @@ var ( "used": s.Object{ "bytes": c.Int("heap_used_bytes"), }, - "external": s.Object{ - "bytes": c.Int("external_bytes"), - }, "size_limit": s.Object{ "bytes": c.Int("size_limit"), }, diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index ca3c9e2faf3..5f316961626 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -48,10 +48,9 @@ var ( "event_loop_delay": c.Float("event_loop_delay_ms"), "memory": c.Dict("mem", s.Schema{ "heap": s.Object{ - "total_in_bytes": c.Int("heap_max_bytes"), - "used_in_bytes": c.Int("heap_used_bytes"), - "external_in_bytes": c.Int("external_bytes"), // TODO: new field, must update monitoring-kibana template in ES x-pack plugin - "size_limit": c.Int("size_limit"), + "total_in_bytes": c.Int("heap_max_bytes"), + "used_in_bytes": c.Int("heap_used_bytes"), + "size_limit": c.Int("size_limit"), }, }), "uptime_in_millis": c.Int("uptime_ms"), From 76f4ea9737f6cf3924d925bdd8c229f56a19f827 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 11 Jul 2018 16:33:20 -0700 Subject: [PATCH 17/66] Fixing schema conversion syntax --- metricbeat/module/kibana/stats/data.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 3ac3b420a0c..1fa8054bafc 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -31,13 +31,13 @@ var ( "cluster_uuid": c.Str("cluster_uuid"), "concurrent_connections": c.Int("concurrent_connections"), "os": c.Dict("os", s.Schema{ - "load": s.Object{ - "avg": c.Dict("cpu.load_average", s.Schema{ + "load": c.Dict("cpu", s.Schema{ + "avg": c.Dict("load_average", s.Schema{ "1m": c.Float("1m"), "5m": c.Float("5m"), "15m": c.Float("15m"), }), - }, + }), "memory": c.Dict("mem", s.Schema{ "total": s.Object{ "bytes": c.Int("total_bytes"), From c214ddfec5d4e102269be2c3805cac37dd80e126 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 13 Jul 2018 10:10:40 -0700 Subject: [PATCH 18/66] Using the new xpack helper package --- metricbeat/helper/xpack.go | 19 ------------------- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 metricbeat/helper/xpack.go diff --git a/metricbeat/helper/xpack.go b/metricbeat/helper/xpack.go deleted file mode 100644 index 990b7c79f13..00000000000 --- a/metricbeat/helper/xpack.go +++ /dev/null @@ -1,19 +0,0 @@ -package helper - -import ( - "fmt" - "time" -) - -// MakeMonitoringIndexName method returns the name of the monitoring index for -// a given product { elasticsearch, kibana, logstash, beats } -func MakeMonitoringIndexName(product string) string { - if !(product == "elasticsearch" || product == "kibana" || product == "logstash" || product == "beats") { - panic("Invalid product '" + product + "'") - } - - today := time.Now().UTC().Format("2006.01.02") - const version = "6" - - return fmt.Sprintf(".monitoring-%v-%v-mb-%v", product, version, today) -} diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 5f316961626..7744fcceb25 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -24,7 +24,7 @@ import ( "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/helper/xpack" "github.com/elastic/beats/metricbeat/mb" ) @@ -96,7 +96,7 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error "kibana_stats": kibanaStatsFields, } - event.Index = helper.MakeMonitoringIndexName("kibana") + event.Index = xpack.MakeMonitoringIndexName(xpack.Kibana) r.Event(event) return nil From 50be6f4d9c784dbd056dc5a22170228fccad9c2c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 13 Jul 2018 10:25:57 -0700 Subject: [PATCH 19/66] Fixing type assertion + adding type conversion --- metricbeat/module/kibana/stats/data_xpack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 7744fcceb25..93865a729d9 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -82,7 +82,7 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error process := data["process"].(map[string]interface{}) mem := process["mem"].(map[string]interface{}) - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", mem["resident_set_size_bytes"].(int)) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(mem["resident_set_size_bytes"].(float64))) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From d0c7de9eb0bbe650f1273fcf6e6511f4bb7fd9ca Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 13 Jul 2018 16:35:48 -0700 Subject: [PATCH 20/66] Adapting to slight changes in Kibana API response --- .../kibana/stats/_meta/test/stats.700.json | 136 +++++++++++------- metricbeat/module/kibana/stats/data.go | 20 +-- metricbeat/module/kibana/stats/data_xpack.go | 20 +-- 3 files changed, 101 insertions(+), 75 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 2079c89d1c6..2ae7c77f5cf 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -1,61 +1,62 @@ { - "cluster_uuid":"4Yg2A7RuT0SIZBMzTE2quA", - "timestamp":"2018-07-05T18:07:12.974Z", "kibana":{ - "uuid":"fc5ec764-19e4-4e51-86e7-3f5a2a2b7cc0", - "name":"spicy.local", - "host":"spicy.local", + "uuid":"5b2de169-2785-441b-ae8c-186a1936b17d", + "name":"Shaunaks-MBP-2", "index":".kibana", - "transport_address":"spicy.local:5601", + "host":"localhost", + "transport_address":"localhost:5601", "version":"7.0.0-alpha1", "snapshot":false, "status":"green" }, - "last_updated":"2018-07-05T18:14:12.574Z", + "last_updated":"2018-07-13T23:24:06.884Z", "collection_interval_ms":5000, + "process":{ + "memory":{ + "heap":{ + "total_bytes":205746176, + "used_bytes":186202032, + "size_limit":1501560832 + }, + "resident_set_size_bytes":197615616 + }, + "event_loop_delay":0.32519304752349854, + "pid":5355, + "uptime_ms":438358 + }, + "os":{ + "load":{ + "1m":2.58544921875, + "5m":3.12646484375, + "15m":3.509765625 + }, + "memory":{ + "total_bytes":17179869184, + "free_bytes":869421056, + "used_bytes":16310448128 + }, + "uptime_ms":1837583000 + }, "response_times":{ - "avg_ms":31.666666666666668, - "max_ms":40 + "avg_ms":14.25, + "max_ms":22 }, "requests":{ - "total":220, - "disconnects":0 + "total":0, + "disconnects":0, + "status_codes":{ + + } }, - "concurrent_connections":0, + "concurrent_connections":7, "sockets":{ "http":{ - "total":5 + "total":0 }, "https":{ "total":0 } }, - "process":{ - "mem":{ - "heap_max_bytes":344432640, - "heap_used_bytes":184244040, - "resident_set_size_bytes":440311808, - "size_limit":1501560832 - }, - "pid":51785, - "uptime_ms":34725, - "event_loop_delay_ms":3.7904850244522095 - }, - "os":{ - "uptime_ms":1151777000, - "cpu":{ - "load_average":{ - "1m":3.1123046875, - "5m":2.94189453125, - "15m":2.83349609375 - } - }, - "mem":{ - "free_bytes":567361536, - "total_bytes":17179869184, - "used_bytes":16612507648 - } - }, "usage":{ "kibana":{ "index":".kibana", @@ -63,13 +64,13 @@ "total":0 }, "visualization":{ - "total":1 + "total":0 }, "search":{ - "total":2 + "total":0 }, "index_pattern":{ - "total":3 + "total":1 }, "graph_workspace":{ "total":0 @@ -82,17 +83,25 @@ "available":true, "enabled":true, "browser_type":"phantom", - "_all":3, + "_all":0, "csv":{ "available":true, - "total":3 + "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ - "completed":3 + }, "lastDay":{ "_all":0, @@ -101,27 +110,44 @@ "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ } }, "last7Days":{ - "_all":3, + "_all":0, "csv":{ "available":true, - "total":3 + "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ - "completed":3 + } } } - } + }, + "cluster_uuid":"cGwLbjF8SyygI5eNqTX6qw" } diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 1fa8054bafc..ba7df08ff6f 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -31,14 +31,14 @@ var ( "cluster_uuid": c.Str("cluster_uuid"), "concurrent_connections": c.Int("concurrent_connections"), "os": c.Dict("os", s.Schema{ - "load": c.Dict("cpu", s.Schema{ - "avg": c.Dict("load_average", s.Schema{ + "load": c.Dict("load", s.Schema{ + "avg": s.Object{ "1m": c.Float("1m"), "5m": c.Float("5m"), "15m": c.Float("15m"), - }), + }, }), - "memory": c.Dict("mem", s.Schema{ + "memory": c.Dict("memory", s.Schema{ "total": s.Object{ "bytes": c.Int("total_bytes"), }, @@ -55,20 +55,20 @@ var ( }), "process": c.Dict("process", s.Schema{ "event_loop_delay": s.Object{ - "ms": c.Float("event_loop_delay_ms"), + "ms": c.Float("event_loop_delay"), }, - "memory": c.Dict("mem", s.Schema{ - "heap": s.Object{ + "memory": c.Dict("memory", s.Schema{ + "heap": c.Dict("heap", s.Schema{ "total": s.Object{ - "bytes": c.Int("heap_max_bytes"), + "bytes": c.Int("total_bytes"), }, "used": s.Object{ - "bytes": c.Int("heap_used_bytes"), + "bytes": c.Int("used_bytes"), }, "size_limit": s.Object{ "bytes": c.Int("size_limit"), }, - }, + }), }), "uptime": s.Object{ "ms": c.Int("uptime_ms"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 93865a729d9..960681c8c4c 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -32,12 +32,12 @@ var ( schemaXPackMonitoring = s.Schema{ "concurrent_connections": c.Int("concurrent_connections"), "os": c.Dict("os", s.Schema{ - "load": c.Dict("cpu.load_average", s.Schema{ + "load": c.Dict("load", s.Schema{ "1m": c.Float("1m"), "5m": c.Float("5m"), "15m": c.Float("15m"), }), - "memory": c.Dict("mem", s.Schema{ + "memory": c.Dict("memory", s.Schema{ "total_in_bytes": c.Int("total_bytes"), "free_in_bytes": c.Int("free_bytes"), "used_in_bytes": c.Int("used_bytes"), @@ -45,13 +45,13 @@ var ( "uptime_in_millis": c.Int("uptime_ms"), }), "process": c.Dict("process", s.Schema{ - "event_loop_delay": c.Float("event_loop_delay_ms"), - "memory": c.Dict("mem", s.Schema{ - "heap": s.Object{ - "total_in_bytes": c.Int("heap_max_bytes"), - "used_in_bytes": c.Int("heap_used_bytes"), + "event_loop_delay": c.Float("event_loop_delay"), + "memory": c.Dict("memory", s.Schema{ + "heap": c.Dict("heap", s.Schema{ + "total_in_bytes": c.Int("total_bytes"), + "used_in_bytes": c.Int("used_bytes"), "size_limit": c.Int("size_limit"), - }, + }), }), "uptime_in_millis": c.Int("uptime_ms"), }), @@ -81,8 +81,8 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error } process := data["process"].(map[string]interface{}) - mem := process["mem"].(map[string]interface{}) - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(mem["resident_set_size_bytes"].(float64))) + memory := process["memory"].(map[string]interface{}) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_bytes"].(float64))) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From e99d40cf1c293048000ee67f136e49aed5f34008 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Jul 2018 09:01:28 -0700 Subject: [PATCH 21/66] Adjust usage structure to meet X-Pack Monitoring docs expectations --- metricbeat/module/kibana/stats/data.go | 14 +++++----- metricbeat/module/kibana/stats/data_xpack.go | 27 ++++++++++++++++++-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index ba7df08ff6f..786c468d73c 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -85,7 +85,10 @@ var ( }), "sockets": SocketsDict, "kibana": KibanaDict, - "usage": UsageDict, + "usage": c.Dict("usage", s.Schema{ + "kibana": kibanaUsageDict, + "reporting": ReportingUsageDict, + }), } // RequestsDict defines how to convert the requests field @@ -116,12 +119,6 @@ var ( "status": c.Str("status"), }) - // UsageDict defines how to convert the usage field - UsageDict = c.Dict("usage", s.Schema{ - "kibana": kibanaUsageDict, - "reporting": reportingUsageDict, - }) - kibanaUsageDict = c.Dict("kibana", s.Schema{ "index": c.Str("index"), "dashboard": c.Dict("dashboard", s.Schema{ @@ -144,7 +141,8 @@ var ( }, c.DictOptional), }) - reportingUsageDict = c.Dict("reporting", s.Schema{ + // ReportingUsageDict defines how to convert the usage.reporting field + ReportingUsageDict = c.Dict("reporting", s.Schema{ "available": c.Bool("available"), "enabled": c.Bool("enabled"), "browser_type": c.Str("browser_type"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 960681c8c4c..ade7fda3c6a 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -59,10 +59,33 @@ var ( "response_times": c.Dict("response_times", s.Schema{ "average": c.Float("avg_ms"), "max": c.Float("max_ms"), - }), + }, c.DictOptional), "sockets": SocketsDict, "kibana": KibanaDict, - "usage": UsageDict, + "usage": c.Dict("usage", s.Schema{ + "index": c.Str("kibana.index"), + "index_pattern": c.Dict("kibana.index_pattern", s.Schema{ + "total": c.Int("total"), + }), + "search": c.Dict("kibana.search", s.Schema{ + "total": c.Int("total"), + }), + "visualization": c.Dict("kibana.visualization", s.Schema{ + "total": c.Int("total"), + }), + "dashboard": c.Dict("kibana.dashboard", s.Schema{ + "total": c.Int("total"), + }), + "timelion_sheet": c.Dict("kibana.timelion_sheet", s.Schema{ + "total": c.Int("total"), + }), + "graph_workspace": c.Dict("kibana.graph_workspace", s.Schema{ + "total": c.Int("total"), + }), + "xpack": s.Object{ + "reporting": ReportingUsageDict, + }, + }), } ) From b993ebedd513dc43f9e0e32c39f47588db262978 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Jul 2018 09:02:03 -0700 Subject: [PATCH 22/66] Better naming of field --- metricbeat/module/kibana/stats/stats.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 6be313c9091..4f7ff6c950e 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -72,7 +72,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } config := struct { - xpack bool `config:"xpack.enabled"` + XPackEnabled bool `config:"xpack.enabled"` }{false} if err := m.Module().UnpackConfig(&config); err != nil { @@ -80,7 +80,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - if config.xpack { + if config.XPackEnabled { cfgwarn.Experimental("The experimental xpack.enabled flag in kibana/stats metricset is enabled.") intervalMs := m.Module().Config().Period.Nanoseconds() / 1000 / 1000 eventMappingXPack(r, intervalMs, content) From 7adc668d1814d8c3fe85aac4118f6c22188907ee Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Jul 2018 17:08:20 -0700 Subject: [PATCH 23/66] Removing sockets field --- metricbeat/module/kibana/stats/_meta/fields.yml | 13 ------------- metricbeat/module/kibana/stats/data.go | 13 +------------ metricbeat/module/kibana/stats/data_xpack.go | 3 +-- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index f2997d86844..29c1ee44c40 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -117,19 +117,6 @@ type: scaled_float description: > Maximum response time in milliseconds - - name: sockets - type: group - description: > - Socket connections metrics - fields: - - name: http.total - type: long - description: > - Number of HTTP connections - - name: https.total - type: long - description: > - Number of HTTPS connections - name: kibana type: group description: > diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 786c468d73c..3ad399c49c1 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -83,8 +83,7 @@ var ( "ms": c.Float("max_ms"), }, }), - "sockets": SocketsDict, - "kibana": KibanaDict, + "kibana": KibanaDict, "usage": c.Dict("usage", s.Schema{ "kibana": kibanaUsageDict, "reporting": ReportingUsageDict, @@ -97,16 +96,6 @@ var ( "total": c.Int("total", s.Optional), }) - // SocketsDict defines how to convert the sockets field - SocketsDict = c.Dict("sockets", s.Schema{ - "http": c.Dict("http", s.Schema{ - "total": c.Int("total"), - }), - "https": c.Dict("https", s.Schema{ - "total": c.Int("total"), - }), - }) - // KibanaDict defines how to convert the kibana field KibanaDict = c.Dict("kibana", s.Schema{ "uuid": c.Str("uuid"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index ade7fda3c6a..8ed9c560a9e 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -60,8 +60,7 @@ var ( "average": c.Float("avg_ms"), "max": c.Float("max_ms"), }, c.DictOptional), - "sockets": SocketsDict, - "kibana": KibanaDict, + "kibana": KibanaDict, "usage": c.Dict("usage", s.Schema{ "index": c.Str("kibana.index"), "index_pattern": c.Dict("kibana.index_pattern", s.Schema{ From 5da8e34bb9d6a873be556240af77d34d9d8396e9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Jul 2018 17:08:38 -0700 Subject: [PATCH 24/66] Rename _all field to all --- metricbeat/module/kibana/stats/_meta/fields.yml | 6 +++--- metricbeat/module/kibana/stats/data.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 29c1ee44c40..90df7b2d728 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -209,7 +209,7 @@ type: keyword description: > Type of browser being used by Reporting for screenshots - - name: _all + - name: all type: long description: > Total number of reports generated across all types of reports @@ -265,7 +265,7 @@ description: > Metrics for the last day fields: - - name: _all + - name: all type: long description: > Total number of reports generated across all types of reports @@ -321,7 +321,7 @@ description: > Metrics for the last seven days fields: - - name: _all + - name: all type: long description: > Total number of reports generated across all types of reports diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 3ad399c49c1..cc1d30903fd 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -135,7 +135,7 @@ var ( "available": c.Bool("available"), "enabled": c.Bool("enabled"), "browser_type": c.Str("browser_type"), - "_all": c.Int("_all"), + "all": c.Int("all"), "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, @@ -161,7 +161,7 @@ var ( }, c.DictOptional) reportingPeriodSchema = s.Schema{ - "_all": c.Int("_all"), + "all": c.Int("all"), "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, From 6a23350cf276383b1cb0730a3554cb83ce5af798 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Jul 2018 17:08:57 -0700 Subject: [PATCH 25/66] Updating test fixture --- .../kibana/stats/_meta/test/stats.700.json | 98 +++++++------------ 1 file changed, 34 insertions(+), 64 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 2ae7c77f5cf..51196401fec 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -9,54 +9,48 @@ "snapshot":false, "status":"green" }, - "last_updated":"2018-07-13T23:24:06.884Z", + "last_updated":"2018-07-17T00:01:00.572Z", "collection_interval_ms":5000, "process":{ "memory":{ "heap":{ - "total_bytes":205746176, - "used_bytes":186202032, + "total_in_bytes":338665472, + "used_in_bytes":196976728, "size_limit":1501560832 }, - "resident_set_size_bytes":197615616 + "resident_set_size_in_bytes":420392960 }, - "event_loop_delay":0.32519304752349854, - "pid":5355, - "uptime_ms":438358 + "event_loop_delay":0.36263203620910645, + "pid":20345, + "uptime_ms":25647 }, "os":{ "load":{ - "1m":2.58544921875, - "5m":3.12646484375, - "15m":3.509765625 + "1m":3.21630859375, + "5m":3.32568359375, + "15m":3.42431640625 }, "memory":{ - "total_bytes":17179869184, - "free_bytes":869421056, - "used_bytes":16310448128 + "total_in_bytes":17179869184, + "free_in_bytes":1907470336, + "used_in_bytes":15272398848 }, - "uptime_ms":1837583000 + "uptime_ms":2098991000 }, "response_times":{ - "avg_ms":14.25, - "max_ms":22 + "avg_ms":34, + "max_ms":133 }, "requests":{ - "total":0, + "total":9, "disconnects":0, "status_codes":{ - - } - }, - "concurrent_connections":7, - "sockets":{ - "http":{ - "total":0 - }, - "https":{ - "total":0 + "200":1, + "302":1, + "304":3 } }, + "concurrent_connections":6, "usage":{ "kibana":{ "index":".kibana", @@ -70,7 +64,7 @@ "total":0 }, "index_pattern":{ - "total":1 + "total":0 }, "graph_workspace":{ "total":0 @@ -83,65 +77,41 @@ "available":true, "enabled":true, "browser_type":"phantom", - "_all":0, + "all":0, "csv":{ "available":true, "total":0 }, "printable_pdf":{ - "available":true, - "total":0, - "app":{ - "visualization":0, - "dashboard":0 - }, - "layout":{ - "print":0, - "preserve_layout":0 - } + "available":false, + "total":0 }, "status":{ }, - "lastDay":{ - "_all":0, + "last_day":{ + "all":0, "csv":{ "available":true, "total":0 }, "printable_pdf":{ - "available":true, - "total":0, - "app":{ - "visualization":0, - "dashboard":0 - }, - "layout":{ - "print":0, - "preserve_layout":0 - } + "available":false, + "total":0 }, "status":{ } }, - "last7Days":{ - "_all":0, + "last_7_days":{ + "all":0, "csv":{ "available":true, "total":0 }, "printable_pdf":{ - "available":true, - "total":0, - "app":{ - "visualization":0, - "dashboard":0 - }, - "layout":{ - "print":0, - "preserve_layout":0 - } + "available":false, + "total":0 }, "status":{ @@ -149,5 +119,5 @@ } } }, - "cluster_uuid":"cGwLbjF8SyygI5eNqTX6qw" + "cluster_uuid":"iwT1u1zRQBeRab1oo8KtsA" } From a395b92c7044b2a222e722ea6cef428e06f6ad2a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 03:52:11 -0700 Subject: [PATCH 26/66] Updating field names --- metricbeat/docs/fields.asciidoc | 45 ++++--------------- metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/fields.yml | 4 +- metricbeat/module/kibana/stats/data.go | 4 +- 4 files changed, 14 insertions(+), 41 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fd54c7b4bde..33c9801b639 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6869,13 +6869,13 @@ Uptime of process in milliseconds -- [float] -== requests fields +== request fields Request count metrics -*`kibana.stats.requests.disconnects`*:: +*`kibana.stats.request.disconnects`*:: + -- type: long @@ -6885,7 +6885,7 @@ Number of requests that were disconnected -- -*`kibana.stats.requests.total`*:: +*`kibana.stats.request.total`*:: + -- type: long @@ -6896,13 +6896,13 @@ Total number of requests -- [float] -== response_times fields +== response_time fields Response times metrics -*`kibana.stats.response_times.avg.ms`*:: +*`kibana.stats.response_time.avg.ms`*:: + -- type: scaled_float @@ -6912,7 +6912,7 @@ Average response time in milliseconds -- -*`kibana.stats.response_times.max.ms`*:: +*`kibana.stats.response_time.max.ms`*:: + -- type: scaled_float @@ -6920,33 +6920,6 @@ type: scaled_float Maximum response time in milliseconds --- - -[float] -== sockets fields - -Socket connections metrics - - - -*`kibana.stats.sockets.http.total`*:: -+ --- -type: long - -Number of HTTP connections - - --- - -*`kibana.stats.sockets.https.total`*:: -+ --- -type: long - -Number of HTTPS connections - - -- [float] @@ -7157,7 +7130,7 @@ Type of browser being used by Reporting for screenshots -- -*`kibana.stats.usage.reporting._all`*:: +*`kibana.stats.usage.reporting.all`*:: + -- type: long @@ -7275,7 +7248,7 @@ Metrics for the last day -*`kibana.stats.usage.reporting.lastDay._all`*:: +*`kibana.stats.usage.reporting.lastDay.all`*:: + -- type: long @@ -7393,7 +7366,7 @@ Metrics for the last seven days -*`kibana.stats.usage.reporting.last7Days._all`*:: +*`kibana.stats.usage.reporting.last7Days.all`*:: + -- type: long diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index cf087507156..b08e9f78c44 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm89v67gRx+/5KwbvksvGQA6LAj4UKDZbbFFs+vCS1x6KwqClscWGIlUO5cT71xekfliWSYn+IScPiA+LrK033w9nhsMhJd3BC27n8MKXTLIbAMONwDl8+bv74ssNQIqUaF4YruQc/nwDAFD9CLlKS4E3AJQpbRaJkiu+nsOKCbLfahTICOewRGNNExrD5Zrm8O8vROLLf24AVhxFSnNn9Q4ky7HDYj9mW+Ac1lqVRf2Nh2ffTtcWGWao/dZnLmiy+tRDdXaAyRR0Ke8MzxFyNJonNOtc3h9x8+nTdQkTUZJBvShLnu5d0MC+4PZV6f5vA8j28/373x5ArcBkCL8KRoYnhEwnWaMHRsFrxpOsGWGipMTE7I2nQ6lkUmqN0oXZXsiVJC+vUHJ9HOxjmS9RW9xEcJQGOhKQsxQtrB0Jod6gnsGjMggmYwaWWr0SaoKESSCUKeSlMLwQCMTtn0yiKmnPolGg8X8lkrEXc+MudpaBEaGNswElE/zJBdxkuHXmNd6VhIBk2FJwyjDtmvX7Tfl91E/ACCf9o0DN7PwB2pLBvMm/3oW+VNvhCMXSGdv04zOENQrmQbMywDao2Rr7gCHEHeR97tGoAClhAtPFSihmPBeNkgLcQ85laXCPMYjy85QoPx+Fcj8py30MTIOSY670duIcqkQCWT6eREYZJmbLrfEk4EClcqaVzpmZQ+gfR7jz2aqDCoyJbRgXbCkQuAyoNMNYacR3G0UoJhZqHL0kTD8cuoUKobfghV3fZ/mh9AD18QleyVgYwkTJlLwLSKFVgnSxVeRrZe6kxQM3dvkXShWLFAXbhj00WI9GPfWr1QGrA07HuijnQnCfn/pVaZYhOyw/Z5WmxmfW8g9dj9wAmBAqYQZT2wfV2fXBJ/NvltvN3OU2Hpn4H7gQPOfm3cB/Z28zUCIFKliCjmjf/48qxdl/qRnUTxGBCBankeFE4H6vKpJadZ0cnHkNUd1LX6xEAXyru/NEldIcVap2VCmnZkczgad2O5Zm9NV25BU1dqSxv2vr1YMJyKqZLg/4AqGjQknChY37xQL4rbYKzuqJ8WOb9VCSn9/x/qVqcVsfQLMaDyw1ncWGvU2L9zt743mZH4HX1j2VvODlpuOTM7e/JT8popkxxWyqtN9NyN+en79C+Ixin4euBPQUJPIeeu1DnBC05oAODUuZ6Zsdi5TnKApGjqMivVJzcUmGyQTdMVUQw/43iGHw7cSZ1WfwyjQMXKb4No0vHlm11lY8t3axNaglEwHRNm8V+UY+QXSs0qB3jGaSCqXNgqWpPtymXJarOqO7pZbLHc9Z+SDgBjVxJSfFCmm09ViygjIVjtlSKYHMBxkB8a8MTYbaHY/WQMuSixQ4AWu1q+/CiIaZctrgNUl16/ZRwmQ+0d1u4/Dw6dxaWIhyzSXdVsZP2gB7a/SZm8v29F1jHOJ4+f6ANStllC0V0+lk622/720VCYhtBhrxDaeSCf4Hs9avhrenOoZY3bm5Glslh2NULuKLghmbAVeDc6pQq44hrjUrssWr0i9uA341SKcLre4Ypm3tBVdyQRmiuRplIwtO1g+52ynadZZ7pC9R/b411usSeEtnlsD2hH36RbfP7hbeoHx7lirtz+Eue1K8kHgDV99UXViUadaR523h1pFaCZZoCZsjvh3zSmmgRCNK28WE93ILJq5zmGLBCNYoUbszPJZoRQRMCCdGnauCsAltgqz+eRR3YlDNFeezX57+GcQIz564+TOeolG4uzTdwdqQD06f8fOzkaBHovUD3/HnLvjBABeaS2P5F0W6mjrUXx/++uOEegf7YUPd8WdEqEd2TmfEuCqCtUBwIYyJcKLyQqBvFHAhHz4eVMghzfbeNuP+JXAqqKBg72arr82Zjqp+rkpsh+VbRpTpdQFfGXfT1SibjdoMcbYPGjEyD+zwGZELFz/b4VgpSL1a43Mj0DhMU1rOax/GGogx10ZCx7cRQ/6NXV9iVphI8BMbiph1ZjQdohFPayziW4sJkmC4wfiwSXBUq/G+SRDXckQ0HReKfmzrERf74fbjcl49rgmJakOmhBuQjWxHJnVdXFMS15ZMCXpqc/KnB7adpG/3tieEG5S2STmtg//sUvqfzy4lTPnZpXg+HzQJPruUkwk/u5QrwH12KRcBjelSjvojOMmOfsHxqZo/1Rzp/HLKu4yeh4XOeIfR96CW/y27y7472de11v269aM/s6pIXpygNl/XYD9ClQAztUHNhJjZ/718CGrrTiwQAn/9PfmhnXZp76clRNz9jXpZFS5wf/CXSqUjGH4jFMYfmR/2WCRT8/C8jRV37/0ecsQticOP0MOErUb3Cfphj8I7N0V7cP8PAAD//+fPaFU=" + return "eJzsm8+P67YRx+/7VwzeZS9ZA3sICvhQoMimaFFkG7zktYeiMMbS2GKXIlUO5V3nrw+oH7YskxL9Q94NsD4EG1uP3w9nyOGXlPQAL7Sdw4tYosI7ACuspDl8+Uf1xZc7gJQ4MaKwQqs5/PkOAKD+EXKdlpLuADjTxi4SrVZiPYcVSnbfGpKETHNYknVNM1kr1Jrn8J8vzPLLf+8AVoJkyvOq1QdQmFOHxX3stqA5rI0ui+YbD89hO9222KLl3be+5oJN1p+mq1U7gCoFU6oHK3KCnKwRCc86l/d73H76dF3CRJZsySzKUqQHF7SwL7R91ab/2wCy+3z79vcn0CuwGcGPEtmKhAlNkrV6YDW8ZiLJ2h4mWilK7EF/OpRaJaUxpKo0uwuFVuzllVqtT4N9LvMlGYebSEHKQkcCckzJwbqeMJkNmRk8a0tgM7SwNPqVyTAkqIBJpZCX0opCErBwf6IiXfJBi1aDof+XxNZdLGx1cdUyIDO5PFvQKqHvqoTbjLZV84YeSiYgtriUgjNKu83646b9MeoPwIgg/bMgg27+AG/ZUt6Ov96FvqG2x5Ea0xlu+vkZwhoF86A5GcANGVxTHzCEuId8zD0aNSAnKCldrKRG67lolBTgEXKhSksHjEGU76dE+f4klMdJWR5jYFqUnHJtthOPoVokMMrHB5HVFuVsubWeAThQqaqmtcnRziH0jyPC+atTBx3oE25QSFxKAqECKm03Vobo3XoRyomDGkcvmdIPh+6gQug78MKt77P8WHqA+vQBXss4GKZEq5S9C0hhdEJ8tVXk57q5sxYP2rjlX2pdLFKSuA1HaLAejUbqR6cDTgcqHReiXEgpfHHqV6VZRnhcfi4qTW3MXMt/6HpUdQCl1AlaSp0PakbXB5/Mf3Pc1cxdbuORWfxGCylyYd8N/Cd8m4GWKXCBCVVEh/F/1inN/sdtp76LSESwOI10JwL3W12R9Kob5ODMa4kaL32tCgXwtTHniS6VPalS7aFSwe2GZoJA7TcsTee53o28kqGONPU3bb1yMAFZPdHVEV8gc1xoxbRwab9W/r42jYJr9LSFZk+Gm/XQEL/c7/6lNri7EEC7Fg8sNJ2lBt+mxfsJ30Re5ifgeQ9PDrHOSGZ70EMWU7T9Zsey6DnSgJFjjcj4NFxCsUWVUHXcEcRw/w1iWHo7M0d9Bq9MyyBUSm/TxOIZ65pd89y7om3JKJQB0RYp00dF+0pE/cg4pcHoWIOKC23sAtPUHNvd63LVZz33vOOqjnmcfBBwQ4aFVpNihTR2fkZhwZkO52yptST0QUZA/Dsjm5GpjtkaoGUpZAqCAXfa9XdhRIu2nDZ57aC6r/y4tJlPdO9ajw8xLq2FhSzXQvF93fhZGylvjb5wk7I7xTUUhzhevj9gzUqRs6VGk85uZaB2igyMmwFHtxFcohS/oWv9ZngHqmOI9R2Am7HVcjRGVWV8UaB1I+BmcJUqNKpjiGuDRbZ41eal2sjdDLLShZ3uGKYziVJoteCMyN6MspWFStYPud9yuHVWeKSvUf2+tq03JfCeLyyBu5Pa6RfdPnu18Abld2dyyv0cdtmT4oXEW7jm5tzCoUyzjvy6Lap1pFGCJTnC9qhoz7zSBjgxRMq5mPDGDuVtNuWOi2FNikx1FISJ0cxOvhLjzlVB1oQ3QVb/NIrbetZTpQrZD7/8K4gRnjxx02d8hEbh7kfpHtZlfHD2jJ/DjCQ9Eq2f+E4898kPJrgwQlnHvyjS1dSp/vnpr3+cVO9hP2yqO/GMSPXIxumCHNc1sBEIroMxGU50Xkjy9QKuFMPnowo5pLm7RYrCvwJOBRUU7N2z87mc6aiax3Pkdlh+x0gqvS3gK4pqulrtRqOxQ5y751WQ7RMeP2pw5eLnDI6TgtSrFVH9vL5hmspymXsY8w9jkY2EjncRQ+GNXV5iFphI8DP9RMwyMzocohHP8xXxzmKCQTDsLz7sIDjJabzvIIhzHBGe40rZj3Uecbkfdh/Xi+ppHiTKhUwJNyAb6UYmDV2cJ4lzJVOCnutN/vSE20lsu9edMG1IOY9y5hbt06Qcfj5NSpjy06R4Ph90EHyalLMJP03KDeA+TcpVQGNMykl/BCfZyW/J/VLPn3qOdH4554U4z5NCF7wI53tKy/+q1nVfwOvrutb9us1zP7O6SF6doGm+qcF+hHoAzPSGDEo5c/97/RQ0rVdigRT46+/ZT+zslvb+sISIW79RbzzCFe4O/lCrdATDrxXC8XPXE901qB/BdrkS1cujxxxxS+Lwg9gwodXoPoc9HFF4Z1N0APd7AAAA//9ryfy4" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 90df7b2d728..9249c9609ea 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -91,7 +91,7 @@ type: long description: > Uptime of process in milliseconds - - name: requests + - name: request type: group description: > Request count metrics @@ -104,7 +104,7 @@ type: long description: > Total number of requests - - name: response_times + - name: response_time type: group description: > Response times metrics diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index cc1d30903fd..370a0124273 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -74,8 +74,8 @@ var ( "ms": c.Int("uptime_ms"), }, }), - "requests": RequestsDict, - "response_times": c.Dict("response_times", s.Schema{ + "request": RequestsDict, + "response_time": c.Dict("response_times", s.Schema{ "avg": s.Object{ "ms": c.Float("avg_ms"), }, From a506aa9ca9015a224f8a952ccf274e76e220ec35 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 04:10:23 -0700 Subject: [PATCH 27/66] Fixing config parsing + logging --- metricbeat/module/kibana/stats/stats.go | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 4f7ff6c950e..056d876ec8b 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/kibana" ) // init registers the MetricSet with the central registry. @@ -44,20 +45,32 @@ var ( // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP + xPackEnabled bool } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The kibana stats metricset is experimental") + config := kibana.DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + if config.XPackEnabled { + cfgwarn.Experimental("The experimental xpack.enabled flag in kibana/stats metricset is enabled.") + } + http, err := helper.NewHTTP(base) if err != nil { return nil, err } + return &MetricSet{ base, http, + config.XPackEnabled, }, nil } @@ -71,17 +84,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - config := struct { - XPackEnabled bool `config:"xpack.enabled"` - }{false} - - if err := m.Module().UnpackConfig(&config); err != nil { - r.Error(err) - return - } - - if config.XPackEnabled { - cfgwarn.Experimental("The experimental xpack.enabled flag in kibana/stats metricset is enabled.") + if m.xPackEnabled { intervalMs := m.Module().Config().Period.Nanoseconds() / 1000 / 1000 eventMappingXPack(r, intervalMs, content) } else { From c2292bdcd19ca7a2fd575d23ec99c133418805a4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 04:12:30 -0700 Subject: [PATCH 28/66] Adding file --- metricbeat/module/kibana/config.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 metricbeat/module/kibana/config.go diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go new file mode 100644 index 00000000000..748ba6bb5e3 --- /dev/null +++ b/metricbeat/module/kibana/config.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kibana + +// Config defines the structure for the Kibana module configuration options +type Config struct { + XPackEnabled bool `config:"xpack.enabled"` +} + +// DefaultConfig returns the default configuration for the Kibana module +func DefaultConfig() Config { + return Config{ + XPackEnabled: false, + } +} From 9315247fa9c5e7f98b6b9e9a798b0c361d34ea91 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 04:55:24 -0700 Subject: [PATCH 29/66] Fixing index name --- metricbeat/helper/xpack/xpack.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metricbeat/helper/xpack/xpack.go b/metricbeat/helper/xpack/xpack.go index 4c7363c8f40..f876be3d826 100644 --- a/metricbeat/helper/xpack/xpack.go +++ b/metricbeat/helper/xpack/xpack.go @@ -19,7 +19,6 @@ package xpack import ( "fmt" - "time" ) // Product supported by X-Pack Monitoring @@ -57,8 +56,7 @@ func (p Product) String() string { // MakeMonitoringIndexName method returns the name of the monitoring index for // a given product { elasticsearch, kibana, logstash, beats } func MakeMonitoringIndexName(product Product) string { - today := time.Now().UTC().Format("2006.01.02") const version = "6" - return fmt.Sprintf(".monitoring-%v-%v-mb-%v", product, version, today) + return fmt.Sprintf(".monitoring-%v-%v-mb", product, version) } From 8760c9175ed5618393fccce443505281db3cd725 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 04:58:11 -0700 Subject: [PATCH 30/66] Updating test fixture and corresponding code --- .../kibana/stats/_meta/test/stats.700.json | 40 +++++++++---------- metricbeat/module/kibana/stats/data.go | 10 ++--- metricbeat/module/kibana/stats/data_xpack.go | 12 +++--- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 51196401fec..24a25d53b98 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -9,45 +9,43 @@ "snapshot":false, "status":"green" }, - "last_updated":"2018-07-17T00:01:00.572Z", + "last_updated":"2018-07-17T11:47:04.629Z", "collection_interval_ms":5000, "process":{ "memory":{ "heap":{ - "total_in_bytes":338665472, - "used_in_bytes":196976728, + "total_in_bytes":253456384, + "used_in_bytes":190699632, "size_limit":1501560832 }, - "resident_set_size_in_bytes":420392960 + "resident_set_size_in_bytes":341807104 }, - "event_loop_delay":0.36263203620910645, - "pid":20345, - "uptime_ms":25647 + "event_loop_delay":0.2823359966278076, + "pid":27037, + "uptime_ms":518674.99999999994 }, "os":{ "load":{ - "1m":3.21630859375, - "5m":3.32568359375, - "15m":3.42431640625 + "1m":3.24658203125, + "5m":3.30322265625, + "15m":3.4365234375 }, "memory":{ "total_in_bytes":17179869184, - "free_in_bytes":1907470336, - "used_in_bytes":15272398848 + "free_in_bytes":1046061056, + "used_in_bytes":16133808128 }, - "uptime_ms":2098991000 + "uptime_ms":2141353000 }, "response_times":{ - "avg_ms":34, - "max_ms":133 + "avg_ms":7, + "max_ms":7 }, "requests":{ - "total":9, + "total":1, "disconnects":0, "status_codes":{ - "200":1, - "302":1, - "304":3 + "200":1 } }, "concurrent_connections":6, @@ -64,7 +62,7 @@ "total":0 }, "index_pattern":{ - "total":0 + "total":1 }, "graph_workspace":{ "total":0 @@ -119,5 +117,5 @@ } } }, - "cluster_uuid":"iwT1u1zRQBeRab1oo8KtsA" + "cluster_uuid":"t-xQbfmPS5CZUS8yVjgLtw" } diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 370a0124273..2687e7c641b 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -40,13 +40,13 @@ var ( }), "memory": c.Dict("memory", s.Schema{ "total": s.Object{ - "bytes": c.Int("total_bytes"), + "bytes": c.Int("total_in_bytes"), }, "free": s.Object{ - "bytes": c.Int("free_bytes"), + "bytes": c.Int("free_in_bytes"), }, "used": s.Object{ - "bytes": c.Int("used_bytes"), + "bytes": c.Int("used_in_bytes"), }, }), "uptime": s.Object{ @@ -60,10 +60,10 @@ var ( "memory": c.Dict("memory", s.Schema{ "heap": c.Dict("heap", s.Schema{ "total": s.Object{ - "bytes": c.Int("total_bytes"), + "bytes": c.Int("total_in_bytes"), }, "used": s.Object{ - "bytes": c.Int("used_bytes"), + "bytes": c.Int("used_in_bytes"), }, "size_limit": s.Object{ "bytes": c.Int("size_limit"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 8ed9c560a9e..640db69707f 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -38,9 +38,9 @@ var ( "15m": c.Float("15m"), }), "memory": c.Dict("memory", s.Schema{ - "total_in_bytes": c.Int("total_bytes"), - "free_in_bytes": c.Int("free_bytes"), - "used_in_bytes": c.Int("used_bytes"), + "total_in_bytes": c.Int("total_in_bytes"), + "free_in_bytes": c.Int("free_in_bytes"), + "used_in_bytes": c.Int("used_in_bytes"), }), "uptime_in_millis": c.Int("uptime_ms"), }), @@ -48,8 +48,8 @@ var ( "event_loop_delay": c.Float("event_loop_delay"), "memory": c.Dict("memory", s.Schema{ "heap": c.Dict("heap", s.Schema{ - "total_in_bytes": c.Int("total_bytes"), - "used_in_bytes": c.Int("used_bytes"), + "total_in_bytes": c.Int("total_in_bytes"), + "used_in_bytes": c.Int("used_in_bytes"), "size_limit": c.Int("size_limit"), }), }), @@ -104,7 +104,7 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error process := data["process"].(map[string]interface{}) memory := process["memory"].(map[string]interface{}) - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_bytes"].(float64))) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_in_bytes"].(float64))) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From aa6ef8ec43d6b6c2ed9f8fe92325425dab651732 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 05:21:18 -0700 Subject: [PATCH 31/66] Making response_times.avg_ms optional as it is sometimes not reported --- metricbeat/module/kibana/stats/data.go | 2 +- metricbeat/module/kibana/stats/data_xpack.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 2687e7c641b..435769cba9b 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -77,7 +77,7 @@ var ( "request": RequestsDict, "response_time": c.Dict("response_times", s.Schema{ "avg": s.Object{ - "ms": c.Float("avg_ms"), + "ms": c.Float("avg_ms", s.Optional), }, "max": s.Object{ "ms": c.Float("max_ms"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 640db69707f..5d63a173b1b 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -57,7 +57,7 @@ var ( }), "requests": RequestsDict, "response_times": c.Dict("response_times", s.Schema{ - "average": c.Float("avg_ms"), + "average": c.Float("avg_ms", s.Optional), "max": c.Float("max_ms"), }, c.DictOptional), "kibana": KibanaDict, From 6652ba5cc296b739a7e5f5ed955444739e5e7c67 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 05:22:30 -0700 Subject: [PATCH 32/66] Snake casing API response keys --- metricbeat/module/kibana/stats/_meta/fields.yml | 4 ++-- metricbeat/module/kibana/stats/data.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 9249c9609ea..66601288bb4 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -260,7 +260,7 @@ type: long description: > Number of reports waiting to start processing - - name: lastDay + - name: last_day type: group description: > Metrics for the last day @@ -316,7 +316,7 @@ type: long description: > Number of reports waiting to start processing - - name: last7Days + - name: last_7_days type: group description: > Metrics for the last seven days diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 435769cba9b..0f7e42b61cf 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -139,8 +139,8 @@ var ( "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, - "lastDay": c.Dict("lastDay", reportingPeriodSchema, c.DictOptional), - "last7Days": c.Dict("last7Days", reportingPeriodSchema, c.DictOptional), + "last_day": c.Dict("last_day", reportingPeriodSchema, c.DictOptional), + "last_7_days": c.Dict("last_7_days", reportingPeriodSchema, c.DictOptional), }, c.DictOptional) reportingCsvDict = c.Dict("csv", s.Schema{ From 94170718740dc0078ad772f2610826e6ec24f599 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 08:33:11 -0700 Subject: [PATCH 33/66] Removing os field (since they can be covered by the system module) --- metricbeat/docs/fields.asciidoc | 137 +++--------------- metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/fields.yml | 46 ------ metricbeat/module/kibana/stats/data.go | 23 --- 4 files changed, 21 insertions(+), 187 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 33c9801b639..4224f637722 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6699,103 +6699,6 @@ type: long Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. --- - -[float] -== os fields - -Operating system metrics - - - -[float] -== load.avg fields - -Operating system load averages - - - -*`kibana.stats.os.load.avg.1m`*:: -+ --- -type: scaled_float - -1 minute load average - - --- - -*`kibana.stats.os.load.avg.5m`*:: -+ --- -type: scaled_float - -5 minute load average - - --- - -*`kibana.stats.os.load.avg.15m`*:: -+ --- -type: scaled_float - -15 minute load average - - --- - -[float] -== memory fields - -Operating system memory metrics - - - -*`kibana.stats.os.memory.total.bytes`*:: -+ --- -type: long - -format: bytes - -Total operating system memory available in bytes - - --- - -*`kibana.stats.os.memory.free.bytes`*:: -+ --- -type: long - -format: bytes - -Operating system memory free in bytes - - --- - -*`kibana.stats.os.memory.used.bytes`*:: -+ --- -type: long - -format: bytes - -Operating system memory used in bytes - - --- - -*`kibana.stats.os.uptime.ms`*:: -+ --- -type: long - -Operating system uptime in seconds - - -- [float] @@ -7242,13 +7145,13 @@ Number of reports waiting to start processing -- [float] -== lastDay fields +== last_day fields Metrics for the last day -*`kibana.stats.usage.reporting.lastDay.all`*:: +*`kibana.stats.usage.reporting.last_day.all`*:: + -- type: long @@ -7265,7 +7168,7 @@ Metrics for CSV reports -*`kibana.stats.usage.reporting.lastDay.csv.available`*:: +*`kibana.stats.usage.reporting.last_day.csv.available`*:: + -- type: boolean @@ -7275,7 +7178,7 @@ Whether CSV reporting is available -- -*`kibana.stats.usage.reporting.lastDay.csv.total`*:: +*`kibana.stats.usage.reporting.last_day.csv.total`*:: + -- type: long @@ -7292,7 +7195,7 @@ Metrics for PDF reports -*`kibana.stats.usage.reporting.lastDay.printable_pdf.available`*:: +*`kibana.stats.usage.reporting.last_day.printable_pdf.available`*:: + -- type: boolean @@ -7302,7 +7205,7 @@ Whether PDF reporting is available -- -*`kibana.stats.usage.reporting.lastDay.printable_pdf.total`*:: +*`kibana.stats.usage.reporting.last_day.printable_pdf.total`*:: + -- type: long @@ -7319,7 +7222,7 @@ Report status metrics -*`kibana.stats.usage.reporting.lastDay.status.completed`*:: +*`kibana.stats.usage.reporting.last_day.status.completed`*:: + -- type: long @@ -7329,7 +7232,7 @@ Number of reports completed -- -*`kibana.stats.usage.reporting.lastDay.status.failed`*:: +*`kibana.stats.usage.reporting.last_day.status.failed`*:: + -- type: long @@ -7339,7 +7242,7 @@ Number of reports failed -- -*`kibana.stats.usage.reporting.lastDay.status.processing`*:: +*`kibana.stats.usage.reporting.last_day.status.processing`*:: + -- type: long @@ -7349,7 +7252,7 @@ Number of reports currently processing -- -*`kibana.stats.usage.reporting.lastDay.status.pending`*:: +*`kibana.stats.usage.reporting.last_day.status.pending`*:: + -- type: long @@ -7360,13 +7263,13 @@ Number of reports waiting to start processing -- [float] -== last7Days fields +== last_7_days fields Metrics for the last seven days -*`kibana.stats.usage.reporting.last7Days.all`*:: +*`kibana.stats.usage.reporting.last_7_days.all`*:: + -- type: long @@ -7383,7 +7286,7 @@ Metrics for CSV reports -*`kibana.stats.usage.reporting.last7Days.csv.available`*:: +*`kibana.stats.usage.reporting.last_7_days.csv.available`*:: + -- type: boolean @@ -7393,7 +7296,7 @@ Whether CSV reporting is available -- -*`kibana.stats.usage.reporting.last7Days.csv.total`*:: +*`kibana.stats.usage.reporting.last_7_days.csv.total`*:: + -- type: long @@ -7410,7 +7313,7 @@ Metrics for PDF reports -*`kibana.stats.usage.reporting.last7Days.printable_pdf.available`*:: +*`kibana.stats.usage.reporting.last_7_days.printable_pdf.available`*:: + -- type: boolean @@ -7420,7 +7323,7 @@ Whether PDF reporting is available -- -*`kibana.stats.usage.reporting.last7Days.printable_pdf.total`*:: +*`kibana.stats.usage.reporting.last_7_days.printable_pdf.total`*:: + -- type: long @@ -7437,7 +7340,7 @@ Report status metrics -*`kibana.stats.usage.reporting.last7Days.status.completed`*:: +*`kibana.stats.usage.reporting.last_7_days.status.completed`*:: + -- type: long @@ -7447,7 +7350,7 @@ Number of reports completed -- -*`kibana.stats.usage.reporting.last7Days.status.failed`*:: +*`kibana.stats.usage.reporting.last_7_days.status.failed`*:: + -- type: long @@ -7457,7 +7360,7 @@ Number of reports failed -- -*`kibana.stats.usage.reporting.last7Days.status.processing`*:: +*`kibana.stats.usage.reporting.last_7_days.status.processing`*:: + -- type: long @@ -7467,7 +7370,7 @@ Number of reports currently processing -- -*`kibana.stats.usage.reporting.last7Days.status.pending`*:: +*`kibana.stats.usage.reporting.last_7_days.status.pending`*:: + -- type: long diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index b08e9f78c44..71d6b361c3b 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm8+P67YRx+/7VwzeZS9ZA3sICvhQoMimaFFkG7zktYeiMMbS2GKXIlUO5V3nrw+oH7YskxL9Q94NsD4EG1uP3w9nyOGXlPQAL7Sdw4tYosI7ACuspDl8+Uf1xZc7gJQ4MaKwQqs5/PkOAKD+EXKdlpLuADjTxi4SrVZiPYcVSnbfGpKETHNYknVNM1kr1Jrn8J8vzPLLf+8AVoJkyvOq1QdQmFOHxX3stqA5rI0ui+YbD89hO9222KLl3be+5oJN1p+mq1U7gCoFU6oHK3KCnKwRCc86l/d73H76dF3CRJZsySzKUqQHF7SwL7R91ab/2wCy+3z79vcn0CuwGcGPEtmKhAlNkrV6YDW8ZiLJ2h4mWilK7EF/OpRaJaUxpKo0uwuFVuzllVqtT4N9LvMlGYebSEHKQkcCckzJwbqeMJkNmRk8a0tgM7SwNPqVyTAkqIBJpZCX0opCErBwf6IiXfJBi1aDof+XxNZdLGx1cdUyIDO5PFvQKqHvqoTbjLZV84YeSiYgtriUgjNKu83646b9MeoPwIgg/bMgg27+AG/ZUt6Ov96FvqG2x5Ea0xlu+vkZwhoF86A5GcANGVxTHzCEuId8zD0aNSAnKCldrKRG67lolBTgEXKhSksHjEGU76dE+f4klMdJWR5jYFqUnHJtthOPoVokMMrHB5HVFuVsubWeAThQqaqmtcnRziH0jyPC+atTBx3oE25QSFxKAqECKm03Vobo3XoRyomDGkcvmdIPh+6gQug78MKt77P8WHqA+vQBXss4GKZEq5S9C0hhdEJ8tVXk57q5sxYP2rjlX2pdLFKSuA1HaLAejUbqR6cDTgcqHReiXEgpfHHqV6VZRnhcfi4qTW3MXMt/6HpUdQCl1AlaSp0PakbXB5/Mf3Pc1cxdbuORWfxGCylyYd8N/Cd8m4GWKXCBCVVEh/F/1inN/sdtp76LSESwOI10JwL3W12R9Kob5ODMa4kaL32tCgXwtTHniS6VPalS7aFSwe2GZoJA7TcsTee53o28kqGONPU3bb1yMAFZPdHVEV8gc1xoxbRwab9W/r42jYJr9LSFZk+Gm/XQEL/c7/6lNri7EEC7Fg8sNJ2lBt+mxfsJ30Re5ifgeQ9PDrHOSGZ70EMWU7T9Zsey6DnSgJFjjcj4NFxCsUWVUHXcEcRw/w1iWHo7M0d9Bq9MyyBUSm/TxOIZ65pd89y7om3JKJQB0RYp00dF+0pE/cg4pcHoWIOKC23sAtPUHNvd63LVZz33vOOqjnmcfBBwQ4aFVpNihTR2fkZhwZkO52yptST0QUZA/Dsjm5GpjtkaoGUpZAqCAXfa9XdhRIu2nDZ57aC6r/y4tJlPdO9ajw8xLq2FhSzXQvF93fhZGylvjb5wk7I7xTUUhzhevj9gzUqRs6VGk85uZaB2igyMmwFHtxFcohS/oWv9ZngHqmOI9R2Am7HVcjRGVWV8UaB1I+BmcJUqNKpjiGuDRbZ41eal2sjdDLLShZ3uGKYziVJoteCMyN6MspWFStYPud9yuHVWeKSvUf2+tq03JfCeLyyBu5Pa6RfdPnu18Abld2dyyv0cdtmT4oXEW7jm5tzCoUyzjvy6Lap1pFGCJTnC9qhoz7zSBjgxRMq5mPDGDuVtNuWOi2FNikx1FISJ0cxOvhLjzlVB1oQ3QVb/NIrbetZTpQrZD7/8K4gRnjxx02d8hEbh7kfpHtZlfHD2jJ/DjCQ9Eq2f+E4898kPJrgwQlnHvyjS1dSp/vnpr3+cVO9hP2yqO/GMSPXIxumCHNc1sBEIroMxGU50Xkjy9QKuFMPnowo5pLm7RYrCvwJOBRUU7N2z87mc6aiax3Pkdlh+x0gqvS3gK4pqulrtRqOxQ5y751WQ7RMeP2pw5eLnDI6TgtSrFVH9vL5hmspymXsY8w9jkY2EjncRQ+GNXV5iFphI8DP9RMwyMzocohHP8xXxzmKCQTDsLz7sIDjJabzvIIhzHBGe40rZj3Uecbkfdh/Xi+ppHiTKhUwJNyAb6UYmDV2cJ4lzJVOCnutN/vSE20lsu9edMG1IOY9y5hbt06Qcfj5NSpjy06R4Ph90EHyalLMJP03KDeA+TcpVQGNMykl/BCfZyW/J/VLPn3qOdH4554U4z5NCF7wI53tKy/+q1nVfwOvrutb9us1zP7O6SF6doGm+qcF+hHoAzPSGDEo5c/97/RQ0rVdigRT46+/ZT+zslvb+sISIW79RbzzCFe4O/lCrdATDrxXC8XPXE901qB/BdrkS1cujxxxxS+Lwg9gwodXoPoc9HFF4Z1N0APd7AAAA//9ryfy4" + return "eJzsm01v4zgShu/5FYVccun4uoAPCyy6e7GLRQeN/tg9LAZGWSpbnFCkhkU5cf/6AfVhyzIp0Y7lZID40Ejbcr2PqsjiS1m6h0fazuFRLFHhDYAVVtIcbv9TvXF7A5ASJ0YUVmg1h7/fAADUH0Ku01LSDQBn2thFotVKrOewQsnuXUOSkGkOS7IuNJO1Qq15Dv+/ZZa3v90ArATJlOdV1HtQmFOHxb3stqA5rI0ui+YdD89hnG4stmh5964vXDBk/WpOtYoDqFIwpbq3IifIyRqR8KxzeP+M21efrkuYyJItmUVZivTggBb2kbZP2vQ/G0B2r58///0J9ApsRvBZIluRMKFJslYPrIanTCRZe4aJVooSe3A+HUqtktIYUlWZ3YFCK/bySq3Wp8E+lPmSjMNNpCBloSMBOabkYN2ZMJkNmRk8aEtgM7SwNPqJyTAkqIBJpZCX0opCErBwf6IiXfJBRKvB0B8lsXUHC1sdXEUGZCZXZwtaJfShKrjNaFuFN3RfMgGxxaUUnFHaDevPW2F0QuxPVH8URmTqax2uHXu9z33DbI9CG1c9qXWxSEnidpb3v9+CcYKS0sVKarRHh4wQAnx2OuB0oNIBoSAXUgqmRKu0L9ri5ZRrs51lhP2cDOUrgqfNmYscSFwodXs6qy3K2XJr6fi7AwO/Cq1NjnYOoS+PngDAD6denwBKqRO0lLph3Iwul2F/9Ba/ZEpfjf5fjtsRwHIbj8ziFy2kyIV9NfAv+DwDLVPgAhOqiA7z/6BTmv3O7Ul9iChE4dYO39QbOZ0I3J9VbNdGO0kOzryWqGmFl+pQAN+a3proUtmTOtUeKhXcrkcTJGq/3jQnz/Vi8kSGOtLUX3N77WACsnqiqyO+QOW40Ipp4cp+qfp9a4KCC3raQrMnw816aIgPri9RefrHhgyuaZeCinZkoeksNfg8Ld4XfBZ5mZ+A5/W+h1hnFLP16WQxRdsPO1ZFjyOFEVcamZ+GSyi2qBKq3GoQw/0bxLD0fGaN+gxemZZBqJSep8nFA9Y9u+a5c03bklEoA6ItUqaPmvaFiPqZcUqD2bEGFRduC4hpao7t7mW5aqt+xzuuyqU7+SDghgwLrSbFCmns/IzCgjMdrtlSa0nog4yA+F9GNnO7uoxaoGUpZAqCAXfa9XthRIu2nLZ47aC6q/y4tJlPdO9acX2xha0BKGS5Forv6uBnbaS8PfqFm5TdJtxQHOJ4+36DPStFzpYaTTq7loHaKTIwbgYc3UZwiVL8Qhf9angHqmOI9QWcq7HVcjRGVVV8UaB1I+BqcJUqNKpjiGuDRbZ40uax2shdDbLShZ3uGKYziVJoteCMyF6NspWFStYPud9yuHVWeKQv0f2+tdGbFnjHL2yBuEEhcSnD/vFii26fvVp4g/K7a3LKfRx22ZPihcRbuOba6sKhTLOO/NgW1TrSKMGSHGF7qWjPvNIGODFEyrmY8MYO5XU25Y6LYU2KTHUpCBOjmZ18Jcado4KsCW+CrP5pFLf1rKdKlbKP3/8bxAhPnrjpMz5Co3D3o3QP6yo+OHvGr8OMFD0SrV/4Tj73xQ8WuDBCWce/KNLV1KX++umff51S72HfbKk7+Ywo9cjG6QU1rntgIxBcB2MqnOi8kOQ7C7hQDh+OOuSQZsu1QuFfAaeCCgr2frPzuZzpqJpfV+V2WH7HSCq9LuATimq6Wu1Go7FDnC2lRLaLFLdTdz/ncJwW+LUi2p/XOEzTWl5mH8YMxFhmI6HjbcRQemPXl5gVJhL8TEMRs86MDodoxPOMRby1mGAQDBuMNzsITrIarzsI4ixHhOm4UPVjrUdc7Yftx+WyepoJibIhU8INyEbakUlTF2dK4mzJlKBnm5O/OXsyiXP3+hOmDSkISL7blJOh321KmPLdpnheb3QQvNuUswnfbcoV4N5tykVAY2zKSX8EJ9nJzzl8r+dPPUc6n5zzSIPnZqEXPMrgu1HLf7P9ZR+h6Ou66H7d5tafWd0kL07QhG96sB+hHgAzvSGDUs7cfy9fgiZ6JRYogb//nn3Tzm5p7w9LiPj1N+qZFbjAD4Qfa5WOYPjBEDi+9XqiHw7qu7BdrUT1+M8xR9ySOHwvNkxoNbq3Yg9nFF7ZFB3A/RkAAP//pld6TA==" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 66601288bb4..76f65a7d747 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -12,52 +12,6 @@ type: long description: > Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. - - name: os - type: group - description: > - Operating system metrics - fields: - - name: load.avg - type: group - description: > - Operating system load averages - fields: - - name: 1m - type: scaled_float - description: > - 1 minute load average - - name: 5m - type: scaled_float - description: > - 5 minute load average - - name: 15m - type: scaled_float - description: > - 15 minute load average - - name: memory - type: group - description: > - Operating system memory metrics - fields: - - name: total.bytes - type: long - format: bytes - description: > - Total operating system memory available in bytes - - name: free.bytes - type: long - format: bytes - description: > - Operating system memory free in bytes - - name: used.bytes - type: long - format: bytes - description: > - Operating system memory used in bytes - - name: uptime.ms - type: long - description: > - Operating system uptime in seconds - name: process type: group description: > diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 0f7e42b61cf..1369eae4d04 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -30,29 +30,6 @@ var ( schema = s.Schema{ "cluster_uuid": c.Str("cluster_uuid"), "concurrent_connections": c.Int("concurrent_connections"), - "os": c.Dict("os", s.Schema{ - "load": c.Dict("load", s.Schema{ - "avg": s.Object{ - "1m": c.Float("1m"), - "5m": c.Float("5m"), - "15m": c.Float("15m"), - }, - }), - "memory": c.Dict("memory", s.Schema{ - "total": s.Object{ - "bytes": c.Int("total_in_bytes"), - }, - "free": s.Object{ - "bytes": c.Int("free_in_bytes"), - }, - "used": s.Object{ - "bytes": c.Int("used_in_bytes"), - }, - }), - "uptime": s.Object{ - "ms": c.Int("uptime_ms"), - }, - }), "process": c.Dict("process", s.Schema{ "event_loop_delay": s.Object{ "ms": c.Float("event_loop_delay"), From 55da81cf214d7e5b9c6354508949126c0f62c71e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 08:37:03 -0700 Subject: [PATCH 34/66] Removing usage fields from non-monitoring events --- metricbeat/docs/fields.asciidoc | 468 ------------------ metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/fields.yml | 223 --------- metricbeat/module/kibana/stats/data.go | 63 --- metricbeat/module/kibana/stats/data_xpack.go | 36 +- 5 files changed, 36 insertions(+), 756 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4224f637722..b5ade08a986 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6910,474 +6910,6 @@ type: keyword Kibana instance's health status --- - -[float] -== usage fields - -Kibana plugins' usage metrics - - - -[float] -== kibana fields - -Kibana core plugins' usage metrics - - - -*`kibana.stats.usage.kibana.index`*:: -+ --- -type: keyword - -Name of Kibana's internal index - - --- - -*`kibana.stats.usage.kibana.dashboard.total`*:: -+ --- -type: long - -Total number of dashboards saved - - --- - -*`kibana.stats.usage.kibana.visualization.total`*:: -+ --- -type: long - -Total number of visualizations saved - - --- - -*`kibana.stats.usage.kibana.search.total`*:: -+ --- -type: long - -Total number of searches saved - - --- - -*`kibana.stats.usage.kibana.index_pattern.total`*:: -+ --- -type: long - -Total number of index patterns saved - - --- - -*`kibana.stats.usage.kibana.graph_workspace.total`*:: -+ --- -type: long - -Total number of graph workspaces saved - - --- - -*`kibana.stats.usage.kibana.timelion_sheet.total`*:: -+ --- -type: long - -Total number of timelion sheets saved - - --- - -[float] -== reporting fields - -Kibana Reporting plugin's usage metrics - - - -*`kibana.stats.usage.reporting.available`*:: -+ --- -type: boolean - -Whether the Reporting plugin is available - - --- - -*`kibana.stats.usage.reporting.enabled`*:: -+ --- -type: boolean - -Whether the Reporting plugin is enabled - - --- - -*`kibana.stats.usage.reporting.browser_type`*:: -+ --- -type: keyword - -Type of browser being used by Reporting for screenshots - - --- - -*`kibana.stats.usage.reporting.all`*:: -+ --- -type: long - -Total number of reports generated across all types of reports - - --- - -[float] -== csv fields - -Metrics for CSV reports - - - -*`kibana.stats.usage.reporting.csv.available`*:: -+ --- -type: boolean - -Whether CSV reporting is available - - --- - -*`kibana.stats.usage.reporting.csv.total`*:: -+ --- -type: long - -Total number of CSV reports generated - - --- - -[float] -== printable_pdf fields - -Metrics for PDF reports - - - -*`kibana.stats.usage.reporting.printable_pdf.available`*:: -+ --- -type: boolean - -Whether PDF reporting is available - - --- - -*`kibana.stats.usage.reporting.printable_pdf.total`*:: -+ --- -type: long - -Total number of PDF reports generated - - --- - -[float] -== status fields - -Report status metrics - - - -*`kibana.stats.usage.reporting.status.completed`*:: -+ --- -type: long - -Number of reports completed - - --- - -*`kibana.stats.usage.reporting.status.failed`*:: -+ --- -type: long - -Number of reports failed - - --- - -*`kibana.stats.usage.reporting.status.processing`*:: -+ --- -type: long - -Number of reports currently processing - - --- - -*`kibana.stats.usage.reporting.status.pending`*:: -+ --- -type: long - -Number of reports waiting to start processing - - --- - -[float] -== last_day fields - -Metrics for the last day - - - -*`kibana.stats.usage.reporting.last_day.all`*:: -+ --- -type: long - -Total number of reports generated across all types of reports - - --- - -[float] -== csv fields - -Metrics for CSV reports - - - -*`kibana.stats.usage.reporting.last_day.csv.available`*:: -+ --- -type: boolean - -Whether CSV reporting is available - - --- - -*`kibana.stats.usage.reporting.last_day.csv.total`*:: -+ --- -type: long - -Total number of CSV reports generated - - --- - -[float] -== printable_pdf fields - -Metrics for PDF reports - - - -*`kibana.stats.usage.reporting.last_day.printable_pdf.available`*:: -+ --- -type: boolean - -Whether PDF reporting is available - - --- - -*`kibana.stats.usage.reporting.last_day.printable_pdf.total`*:: -+ --- -type: long - -Total number of PDF reports generated - - --- - -[float] -== status fields - -Report status metrics - - - -*`kibana.stats.usage.reporting.last_day.status.completed`*:: -+ --- -type: long - -Number of reports completed - - --- - -*`kibana.stats.usage.reporting.last_day.status.failed`*:: -+ --- -type: long - -Number of reports failed - - --- - -*`kibana.stats.usage.reporting.last_day.status.processing`*:: -+ --- -type: long - -Number of reports currently processing - - --- - -*`kibana.stats.usage.reporting.last_day.status.pending`*:: -+ --- -type: long - -Number of reports waiting to start processing - - --- - -[float] -== last_7_days fields - -Metrics for the last seven days - - - -*`kibana.stats.usage.reporting.last_7_days.all`*:: -+ --- -type: long - -Total number of reports generated across all types of reports - - --- - -[float] -== csv fields - -Metrics for CSV reports - - - -*`kibana.stats.usage.reporting.last_7_days.csv.available`*:: -+ --- -type: boolean - -Whether CSV reporting is available - - --- - -*`kibana.stats.usage.reporting.last_7_days.csv.total`*:: -+ --- -type: long - -Total number of CSV reports generated - - --- - -[float] -== printable_pdf fields - -Metrics for PDF reports - - - -*`kibana.stats.usage.reporting.last_7_days.printable_pdf.available`*:: -+ --- -type: boolean - -Whether PDF reporting is available - - --- - -*`kibana.stats.usage.reporting.last_7_days.printable_pdf.total`*:: -+ --- -type: long - -Total number of PDF reports generated - - --- - -[float] -== status fields - -Report status metrics - - - -*`kibana.stats.usage.reporting.last_7_days.status.completed`*:: -+ --- -type: long - -Number of reports completed - - --- - -*`kibana.stats.usage.reporting.last_7_days.status.failed`*:: -+ --- -type: long - -Number of reports failed - - --- - -*`kibana.stats.usage.reporting.last_7_days.status.processing`*:: -+ --- -type: long - -Number of reports currently processing - - --- - -*`kibana.stats.usage.reporting.last_7_days.status.pending`*:: -+ --- -type: long - -Number of reports waiting to start processing - - -- [float] diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 71d6b361c3b..403837c127d 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm01v4zgShu/5FYVccun4uoAPCyy6e7GLRQeN/tg9LAZGWSpbnFCkhkU5cf/6AfVhyzIp0Y7lZID40Ejbcr2PqsjiS1m6h0fazuFRLFHhDYAVVtIcbv9TvXF7A5ASJ0YUVmg1h7/fAADUH0Ku01LSDQBn2thFotVKrOewQsnuXUOSkGkOS7IuNJO1Qq15Dv+/ZZa3v90ArATJlOdV1HtQmFOHxb3stqA5rI0ui+YdD89hnG4stmh5964vXDBk/WpOtYoDqFIwpbq3IifIyRqR8KxzeP+M21efrkuYyJItmUVZivTggBb2kbZP2vQ/G0B2r58///0J9ApsRvBZIluRMKFJslYPrIanTCRZe4aJVooSe3A+HUqtktIYUlWZ3YFCK/bySq3Wp8E+lPmSjMNNpCBloSMBOabkYN2ZMJkNmRk8aEtgM7SwNPqJyTAkqIBJpZCX0opCErBwf6IiXfJBRKvB0B8lsXUHC1sdXEUGZCZXZwtaJfShKrjNaFuFN3RfMgGxxaUUnFHaDevPW2F0QuxPVH8URmTqax2uHXu9z33DbI9CG1c9qXWxSEnidpb3v9+CcYKS0sVKarRHh4wQAnx2OuB0oNIBoSAXUgqmRKu0L9ri5ZRrs51lhP2cDOUrgqfNmYscSFwodXs6qy3K2XJr6fi7AwO/Cq1NjnYOoS+PngDAD6denwBKqRO0lLph3Iwul2F/9Ba/ZEpfjf5fjtsRwHIbj8ziFy2kyIV9NfAv+DwDLVPgAhOqiA7z/6BTmv3O7Ul9iChE4dYO39QbOZ0I3J9VbNdGO0kOzryWqGmFl+pQAN+a3proUtmTOtUeKhXcrkcTJGq/3jQnz/Vi8kSGOtLUX3N77WACsnqiqyO+QOW40Ipp4cp+qfp9a4KCC3raQrMnw816aIgPri9RefrHhgyuaZeCinZkoeksNfg8Ld4XfBZ5mZ+A5/W+h1hnFLP16WQxRdsPO1ZFjyOFEVcamZ+GSyi2qBKq3GoQw/0bxLD0fGaN+gxemZZBqJSep8nFA9Y9u+a5c03bklEoA6ItUqaPmvaFiPqZcUqD2bEGFRduC4hpao7t7mW5aqt+xzuuyqU7+SDghgwLrSbFCmns/IzCgjMdrtlSa0nog4yA+F9GNnO7uoxaoGUpZAqCAXfa9XthRIu2nLZ47aC6q/y4tJlPdO9acX2xha0BKGS5Forv6uBnbaS8PfqFm5TdJtxQHOJ4+36DPStFzpYaTTq7loHaKTIwbgYc3UZwiVL8Qhf9angHqmOI9QWcq7HVcjRGVVV8UaB1I+BqcJUqNKpjiGuDRbZ40uax2shdDbLShZ3uGKYziVJoteCMyF6NspWFStYPud9yuHVWeKQv0f2+tdGbFnjHL2yBuEEhcSnD/vFii26fvVp4g/K7a3LKfRx22ZPihcRbuOba6sKhTLOO/NgW1TrSKMGSHGF7qWjPvNIGODFEyrmY8MYO5XU25Y6LYU2KTHUpCBOjmZ18Jcado4KsCW+CrP5pFLf1rKdKlbKP3/8bxAhPnrjpMz5Co3D3o3QP6yo+OHvGr8OMFD0SrV/4Tj73xQ8WuDBCWce/KNLV1KX++umff51S72HfbKk7+Ywo9cjG6QU1rntgIxBcB2MqnOi8kOQ7C7hQDh+OOuSQZsu1QuFfAaeCCgr2frPzuZzpqJpfV+V2WH7HSCq9LuATimq6Wu1Go7FDnC2lRLaLFLdTdz/ncJwW+LUi2p/XOEzTWl5mH8YMxFhmI6HjbcRQemPXl5gVJhL8TEMRs86MDodoxPOMRby1mGAQDBuMNzsITrIarzsI4ixHhOm4UPVjrUdc7Yftx+WyepoJibIhU8INyEbakUlTF2dK4mzJlKBnm5O/OXsyiXP3+hOmDSkISL7blJOh321KmPLdpnheb3QQvNuUswnfbcoV4N5tykVAY2zKSX8EJ9nJzzl8r+dPPUc6n5zzSIPnZqEXPMrgu1HLf7P9ZR+h6Ou66H7d5tafWd0kL07QhG96sB+hHgAzvSGDUs7cfy9fgiZ6JRYogb//nn3Tzm5p7w9LiPj1N+qZFbjAD4Qfa5WOYPjBEDi+9XqiHw7qu7BdrUT1+M8xR9ySOHwvNkxoNbq3Yg9nFF7ZFB3A/RkAAP//pld6TA==" + return "eJzEmM2O4zYMx+95CmIuc9nxA+RQoGgXaFHMoNh20ENRBIzExOrIkivSmaRPv5A/EseRE2c2wfqw2HHkP38iaZH0E7zRbg5vZokOZwBixNIcHn6rbzzMADSxCqYU490cfpgBADQ/QuF1ZWkGwLkPslDercx6Diu0HO8GsoRMc1iSRGkmEePWPIe/H5jtwz8zgJUhq3leqz6Bw4J6LPGSXUlzWAdfle2dBM+xTl+LBYX3d1Nyo5LN1W611gF0GkLlnsQUBAVJMIqz3vLhjrtrSNcnVLZiobCoKqOPFnSwb7R792H42xnkeL2+/voz+BVITvDZIotRTBhU3tkD8fCeG5V3O1TeOVJytJ8epXeqCoFcHea40HjHSV7r3fo62JeqWFKIuMoacgI9E1Cgpggbd8IUNhQyePFCIDkKLIN/ZwoMCh0wOQ1FZcWUloBN/C868hUfKYqHQP9VxBIXG6kX18qAzBTjLOCdok91wCWnXS0f6KliAmLBpTWck+7Lpv1WBq+I044aZuEET/3eyHW5N/g9lWYHFNrE6Fnvy4Umi7usGD7fgbFCS3qxsh7lZMkFQoDP0Q5EO1DbAeOgMNYaJuWdHhrt8AoqfNhlOeHQJ+f8NYGn81lUHnHcmOsOdOIFbbbcCZ0+eybxa2kfCpQ5jD18cQMAf0brzQbQWq9QSMc0brMrejit3uFXTPq70f8SuSMBLHfTkdn8TwtrCiPfDfwZtxl4q4FLVFQTHfv/xWvK/uVuU58mBKKMtSP16l3YzgTc11o7HqM9J4++eR1RexTe6oQC+NKercpXTq46qQ5Q2nBXj+7gqEO9aTfPTTF5p0A90zSsuYPj4A5kzYvuTvhGIseld0yLGPZbxe9LKwpR9LpCcyDDzfpcip+tL5P89OOGAq5p74Ka9kKh6ZUa3N4X7xm3pqiKK/CSve8x1geC2fXpJKhRhrKXopjoSOFCVzrRPy2XcSzoFNXd6ihG/HcUQ2j7wRgNGZJmOgbjNG3v44sXbM7shucxHtpCwaEdMdoh5f7k0L4R0dAz0dJZ70hAx2UcAVHrcNru3paradUfec9Vd+nR/CjghgIb7+6KNWZj3884LDn34zFbem8JU5ATIP7KSfI41eXUAS0rYzUYBtzbbu6NIwpKdd/gdUn1WPfjVvKh0VGWqyf3P2qF9pT7xiE9kf7fMJynjp70+HjbjwJDu1E9bbdN5qzpRG5O0Mq3jU4aoUmAzMdKb20W/7x9CFr12thICNId0IdL8nMjd5qWMKEiT/oKAzfoQ39qrPQMjn/qgNNhYhwpPcNPYurmihgrU3/QOuUYdyFMni7gkv8m8kKin+8PF+c9ChOmjXuSHsF9DQAA//+Rupdj" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 76f65a7d747..fc13b17396c 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -108,226 +108,3 @@ type: keyword description: > Kibana instance's health status - - name: usage - type: group - description: > - Kibana plugins' usage metrics - fields: - - name: kibana - type: group - description: > - Kibana core plugins' usage metrics - fields: - - name: index - type: keyword - description: > - Name of Kibana's internal index - - name: dashboard.total - type: long - description: > - Total number of dashboards saved - - name: visualization.total - type: long - description: > - Total number of visualizations saved - - name: search.total - type: long - description: > - Total number of searches saved - - name: index_pattern.total - type: long - description: > - Total number of index patterns saved - - name: graph_workspace.total - type: long - description: > - Total number of graph workspaces saved - - name: timelion_sheet.total - type: long - description: > - Total number of timelion sheets saved - - name: reporting - type: group - description: > - Kibana Reporting plugin's usage metrics - fields: - - name: available - type: boolean - description: > - Whether the Reporting plugin is available - - name: enabled - type: boolean - description: > - Whether the Reporting plugin is enabled - - name: browser_type - type: keyword - description: > - Type of browser being used by Reporting for screenshots - - name: all - type: long - description: > - Total number of reports generated across all types of reports - - name: csv - type: group - description: > - Metrics for CSV reports - fields: - - name: available - type: boolean - description: > - Whether CSV reporting is available - - name: total - type: long - description: > - Total number of CSV reports generated - - name: printable_pdf - type: group - description: > - Metrics for PDF reports - fields: - - name: available - type: boolean - description: > - Whether PDF reporting is available - - name: total - type: long - description: > - Total number of PDF reports generated - - name: status - type: group - description: > - Report status metrics - fields: - - name: completed - type: long - description: > - Number of reports completed - - name: failed - type: long - description: > - Number of reports failed - - name: processing - type: long - description: > - Number of reports currently processing - - name: pending - type: long - description: > - Number of reports waiting to start processing - - name: last_day - type: group - description: > - Metrics for the last day - fields: - - name: all - type: long - description: > - Total number of reports generated across all types of reports - - name: csv - type: group - description: > - Metrics for CSV reports - fields: - - name: available - type: boolean - description: > - Whether CSV reporting is available - - name: total - type: long - description: > - Total number of CSV reports generated - - name: printable_pdf - type: group - description: > - Metrics for PDF reports - fields: - - name: available - type: boolean - description: > - Whether PDF reporting is available - - name: total - type: long - description: > - Total number of PDF reports generated - - name: status - type: group - description: > - Report status metrics - fields: - - name: completed - type: long - description: > - Number of reports completed - - name: failed - type: long - description: > - Number of reports failed - - name: processing - type: long - description: > - Number of reports currently processing - - name: pending - type: long - description: > - Number of reports waiting to start processing - - name: last_7_days - type: group - description: > - Metrics for the last seven days - fields: - - name: all - type: long - description: > - Total number of reports generated across all types of reports - - name: csv - type: group - description: > - Metrics for CSV reports - fields: - - name: available - type: boolean - description: > - Whether CSV reporting is available - - name: total - type: long - description: > - Total number of CSV reports generated - - name: printable_pdf - type: group - description: > - Metrics for PDF reports - fields: - - name: available - type: boolean - description: > - Whether PDF reporting is available - - name: total - type: long - description: > - Total number of PDF reports generated - - name: status - type: group - description: > - Report status metrics - fields: - - name: completed - type: long - description: > - Number of reports completed - - name: failed - type: long - description: > - Number of reports failed - - name: processing - type: long - description: > - Number of reports currently processing - - name: pending - type: long - description: > - Number of reports waiting to start processing - - - - - diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 1369eae4d04..9a08f948f5b 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -61,10 +61,6 @@ var ( }, }), "kibana": KibanaDict, - "usage": c.Dict("usage", s.Schema{ - "kibana": kibanaUsageDict, - "reporting": ReportingUsageDict, - }), } // RequestsDict defines how to convert the requests field @@ -84,65 +80,6 @@ var ( "snapshot": c.Bool("snapshot"), "status": c.Str("status"), }) - - kibanaUsageDict = c.Dict("kibana", s.Schema{ - "index": c.Str("index"), - "dashboard": c.Dict("dashboard", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "visualization": c.Dict("visualization", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "search": c.Dict("search", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "index_pattern": c.Dict("index_pattern", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "graph_workspace": c.Dict("graph_workspace", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - "timelion_sheet": c.Dict("timelion_sheet", s.Schema{ - "total": c.Int("total"), - }, c.DictOptional), - }) - - // ReportingUsageDict defines how to convert the usage.reporting field - ReportingUsageDict = c.Dict("reporting", s.Schema{ - "available": c.Bool("available"), - "enabled": c.Bool("enabled"), - "browser_type": c.Str("browser_type"), - "all": c.Int("all"), - "csv": reportingCsvDict, - "printable_pdf": reportingPrintablePdfDict, - "status": reportingStatusDict, - "last_day": c.Dict("last_day", reportingPeriodSchema, c.DictOptional), - "last_7_days": c.Dict("last_7_days", reportingPeriodSchema, c.DictOptional), - }, c.DictOptional) - - reportingCsvDict = c.Dict("csv", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional) - - reportingPrintablePdfDict = c.Dict("printable_pdf", s.Schema{ - "available": c.Bool("available"), - "total": c.Int("total"), - }, c.DictOptional) - - reportingStatusDict = c.Dict("status", s.Schema{ - "completed": c.Int("completed", s.Optional), - "failed": c.Int("failed", s.Optional), - "processing": c.Int("processing", s.Optional), - "pending": c.Int("pending", s.Optional), - }, c.DictOptional) - - reportingPeriodSchema = s.Schema{ - "all": c.Int("all"), - "csv": reportingCsvDict, - "printable_pdf": reportingPrintablePdfDict, - "status": reportingStatusDict, - } ) func eventMapping(r mb.ReporterV2, content []byte) error { diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 5d63a173b1b..bee6e2c25fd 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -82,10 +82,44 @@ var ( "total": c.Int("total"), }), "xpack": s.Object{ - "reporting": ReportingUsageDict, + "reporting": c.Dict("reporting", s.Schema{ + "available": c.Bool("available"), + "enabled": c.Bool("enabled"), + "browser_type": c.Str("browser_type"), + "all": c.Int("all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + "last_day": c.Dict("last_day", reportingPeriodSchema, c.DictOptional), + "last_7_days": c.Dict("last_7_days", reportingPeriodSchema, c.DictOptional), + }, c.DictOptional), }, }), } + + reportingCsvDict = c.Dict("csv", s.Schema{ + "available": c.Bool("available"), + "total": c.Int("total"), + }, c.DictOptional) + + reportingPrintablePdfDict = c.Dict("printable_pdf", s.Schema{ + "available": c.Bool("available"), + "total": c.Int("total"), + }, c.DictOptional) + + reportingStatusDict = c.Dict("status", s.Schema{ + "completed": c.Int("completed", s.Optional), + "failed": c.Int("failed", s.Optional), + "processing": c.Int("processing", s.Optional), + "pending": c.Int("pending", s.Optional), + }, c.DictOptional) + + reportingPeriodSchema = s.Schema{ + "all": c.Int("all"), + "csv": reportingCsvDict, + "printable_pdf": reportingPrintablePdfDict, + "status": reportingStatusDict, + } ) func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error { From 24774e6a429fce2b48646bf7f52a7fd0f71c13fc Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 09:15:01 -0700 Subject: [PATCH 35/66] Regenerating data.json --- .../module/kibana/stats/_meta/data.json | 64 ++++++++----------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index db704021485..7de7dfe32e4 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -6,64 +6,54 @@ }, "elasticsearch": { "cluster": { - "id": "PKEQ1V5kT4yPng_sgiqF9g" + "id": "NkfU5AinRnyFnqBD36zhEw" } }, "kibana": { "stats": { - "concurrent_connections": 17, - "event_loop_delay": 718.5184001922607, - "name": "kibana", + "concurrent_connections": 3, + "kibana": { + "host": "localhost", + "index": ".kibana", + "name": "Shaunaks-MBP-2", + "snapshot": false, + "status": "green", + "transport_address": "localhost:5601", + "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d", + "version": "7.0.0-alpha1" + }, "process": { + "event_loop_delay": { + "ms": 0.7881790399551392 + }, "memory": { - "external": { - "bytes": 1969666 - }, "heap": { - "max": { - "bytes": 200028160 + "size_limit": { + "bytes": 1501560832 + }, + "total": { + "bytes": 477569024 }, "used": { - "bytes": 138875320 + "bytes": 205526576 } - }, - "resident_set_size": { - "bytes": 247865344 } }, - "pid": 1, "uptime": { - "ms": 260863 + "ms": 24803 } }, - "requests": { + "request": { "disconnects": 0, - "total": 91 + "total": 3 }, - "response_times": { + "response_time": { "avg": { - "ms": 1347.2500000000002 + "ms": 39 }, "max": { - "ms": 4293 + "ms": 48 } - }, - "sockets": { - "http": { - "total": 56 - }, - "https": { - "total": 0 - } - }, - "status": { - "overall": { - "state": "green" - } - }, - "uuid": "b4b34609-03b3-463c-8699-17109e72df70", - "version": { - "number": "7.0.0-alpha1" } } }, From bcab5596f987d6afa663084faafff30ba0fb5c22 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 09:18:33 -0700 Subject: [PATCH 36/66] Updating unit tests for xpack helper --- metricbeat/helper/xpack/xpack_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/metricbeat/helper/xpack/xpack_test.go b/metricbeat/helper/xpack/xpack_test.go index 841e6730c66..c6aa2cd5f1f 100644 --- a/metricbeat/helper/xpack/xpack_test.go +++ b/metricbeat/helper/xpack/xpack_test.go @@ -20,14 +20,11 @@ package xpack import ( "fmt" "testing" - "time" "github.com/stretchr/testify/assert" ) func TestMakeMonitoringIndexName(t *testing.T) { - today := time.Now().UTC().Format("2006.01.02") - tests := []struct { Name string Product Product @@ -36,22 +33,22 @@ func TestMakeMonitoringIndexName(t *testing.T) { { "Elasticsearch monitoring index", Elasticsearch, - fmt.Sprintf(".monitoring-es-6-mb-%v", today), + ".monitoring-es-6-mb", }, { "Kibana monitoring index", Kibana, - fmt.Sprintf(".monitoring-kibana-6-mb-%v", today), + ".monitoring-kibana-6-mb", }, { "Logstash monitoring index", Logstash, - fmt.Sprintf(".monitoring-logstash-6-mb-%v", today), + ".monitoring-logstash-6-mb", }, { "Beats monitoring index", Beats, - fmt.Sprintf(".monitoring-beats-6-mb-%v", today), + ".monitoring-beats-6-mb", }, } From 16b953ded6ed3524fd68e9f9ccef73d7f4ea450e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 10:12:47 -0700 Subject: [PATCH 37/66] Updating test fixture and corresponding code --- .../kibana/stats/_meta/test/stats.700.json | 42 +++++++++---------- metricbeat/module/kibana/stats/data.go | 4 +- metricbeat/module/kibana/stats/data_xpack.go | 12 +++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 24a25d53b98..137065034b2 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -9,46 +9,46 @@ "snapshot":false, "status":"green" }, - "last_updated":"2018-07-17T11:47:04.629Z", + "last_updated":"2018-07-17T17:09:23.926Z", "collection_interval_ms":5000, "process":{ "memory":{ "heap":{ - "total_in_bytes":253456384, - "used_in_bytes":190699632, + "total_bytes":342585344, + "used_bytes":192264704, "size_limit":1501560832 }, - "resident_set_size_in_bytes":341807104 + "resident_set_size_bytes":431456256 }, - "event_loop_delay":0.2823359966278076, - "pid":27037, - "uptime_ms":518674.99999999994 + "event_loop_delay":0.37530696392059326, + "pid":32307, + "uptime_ms":27353 }, "os":{ "load":{ - "1m":3.24658203125, - "5m":3.30322265625, - "15m":3.4365234375 + "1m":2.01513671875, + "5m":2.0283203125, + "15m":2.041015625 }, "memory":{ - "total_in_bytes":17179869184, - "free_in_bytes":1046061056, - "used_in_bytes":16133808128 + "total_bytes":17179869184, + "free_bytes":66318336, + "used_bytes":17113550848 }, - "uptime_ms":2141353000 + "uptime_ms":2160690000 }, "response_times":{ - "avg_ms":7, - "max_ms":7 + "avg_ms":15, + "max_ms":18 }, "requests":{ - "total":1, + "total":4, "disconnects":0, "status_codes":{ - "200":1 + "200":4 } }, - "concurrent_connections":6, + "concurrent_connections":3, "usage":{ "kibana":{ "index":".kibana", @@ -62,7 +62,7 @@ "total":0 }, "index_pattern":{ - "total":1 + "total":0 }, "graph_workspace":{ "total":0 @@ -117,5 +117,5 @@ } } }, - "cluster_uuid":"t-xQbfmPS5CZUS8yVjgLtw" + "cluster_uuid":"NkfU5AinRnyFnqBD36zhEw" } diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 9a08f948f5b..52f47af2ed0 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -37,10 +37,10 @@ var ( "memory": c.Dict("memory", s.Schema{ "heap": c.Dict("heap", s.Schema{ "total": s.Object{ - "bytes": c.Int("total_in_bytes"), + "bytes": c.Int("total_bytes"), }, "used": s.Object{ - "bytes": c.Int("used_in_bytes"), + "bytes": c.Int("used_bytes"), }, "size_limit": s.Object{ "bytes": c.Int("size_limit"), diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index bee6e2c25fd..e29b3bb88c5 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -38,9 +38,9 @@ var ( "15m": c.Float("15m"), }), "memory": c.Dict("memory", s.Schema{ - "total_in_bytes": c.Int("total_in_bytes"), - "free_in_bytes": c.Int("free_in_bytes"), - "used_in_bytes": c.Int("used_in_bytes"), + "total_in_bytes": c.Int("total_bytes"), + "free_in_bytes": c.Int("free_bytes"), + "used_in_bytes": c.Int("used_bytes"), }), "uptime_in_millis": c.Int("uptime_ms"), }), @@ -48,8 +48,8 @@ var ( "event_loop_delay": c.Float("event_loop_delay"), "memory": c.Dict("memory", s.Schema{ "heap": c.Dict("heap", s.Schema{ - "total_in_bytes": c.Int("total_in_bytes"), - "used_in_bytes": c.Int("used_in_bytes"), + "total_in_bytes": c.Int("total_bytes"), + "used_in_bytes": c.Int("used_bytes"), "size_limit": c.Int("size_limit"), }), }), @@ -138,7 +138,7 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error process := data["process"].(map[string]interface{}) memory := process["memory"].(map[string]interface{}) - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_in_bytes"].(float64))) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_bytes"].(float64))) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From 97f43a23e1944e718518c8f473e9e497355d03b8 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 15:07:50 -0700 Subject: [PATCH 38/66] Adding config setting for base path --- metricbeat/helper/http.go | 18 ++++++++ metricbeat/helper/http_test.go | 43 +++++++++++++++++++ .../module/kibana/_meta/config.reference.yml | 1 + metricbeat/module/kibana/_meta/config.yml | 1 + metricbeat/module/kibana/config.go | 4 +- metricbeat/module/kibana/stats/stats.go | 4 ++ 6 files changed, 70 insertions(+), 1 deletion(-) diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index b01fa781885..3f52cdb6b3d 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -25,6 +25,8 @@ import ( "io" "io/ioutil" "net/http" + "regexp" + "strings" "time" "github.com/pkg/errors" @@ -148,6 +150,22 @@ func (h *HTTP) SetBody(body []byte) { h.body = body } +// AddBasePath adds the given base path to the URI +func (h *HTTP) AddBasePath(basePath string) { + if basePath == "" { + return + } + + basePath = strings.Trim(basePath, "/") + basePath = "/" + basePath + "/" + + re := regexp.MustCompile("(.*//[^/]+)(/)(.*)") + uri := h.GetURI() + uri = re.ReplaceAllString(uri, "$1"+basePath+"$3") + + h.SetURI(uri) +} + // FetchContent makes an HTTP request to the configured url and returns the body content. func (h *HTTP) FetchContent() ([]byte, error) { resp, err := h.FetchResponse() diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index a3e5d615e4d..74129288450 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -69,3 +69,46 @@ func TestGetAuthHeaderFromTokenNoFile(t *testing.T) { assert.Equal(t, "", header) assert.Error(t, err) } + +func TestAddBasePath(t *testing.T) { + tests := []struct { + Name, Content, Expected string + }{ + { + "Test with empty basepath", + "", + "http://localhost:9999/some/path", + }, + { + "Test with basepath with no leading or trailing slashes", + "foobar", + "http://localhost:9999/foobar/some/path", + }, + { + "Test with basepath with a leading slash", + "/foobar", + "http://localhost:9999/foobar/some/path", + }, + { + "Test with basepath with a trailing slash", + "foobar/", + "http://localhost:9999/foobar/some/path", + }, + { + "Test with basepath with leading and trailing slashes", + "/foobar/", + "http://localhost:9999/foobar/some/path", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + http := HTTP{ + uri: "http://localhost:9999/some/path", + } + + http.AddBasePath(test.Content) + assert.Equal(t, test.Expected, http.GetURI()) + }) + } +} diff --git a/metricbeat/module/kibana/_meta/config.reference.yml b/metricbeat/module/kibana/_meta/config.reference.yml index 05d0b4d3dbd..328a5e0b57a 100644 --- a/metricbeat/module/kibana/_meta/config.reference.yml +++ b/metricbeat/module/kibana/_meta/config.reference.yml @@ -2,4 +2,5 @@ metricsets: ["status"] period: 10s hosts: ["localhost:5601"] + basepath: "" enabled: true diff --git a/metricbeat/module/kibana/_meta/config.yml b/metricbeat/module/kibana/_meta/config.yml index 838670f8b62..f997985d3c2 100644 --- a/metricbeat/module/kibana/_meta/config.yml +++ b/metricbeat/module/kibana/_meta/config.yml @@ -3,5 +3,6 @@ # - status period: 10s hosts: ["localhost:5601"] + #basepath: "" #username: "user" #password: "secret" diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go index 748ba6bb5e3..2a8e55008c6 100644 --- a/metricbeat/module/kibana/config.go +++ b/metricbeat/module/kibana/config.go @@ -19,12 +19,14 @@ package kibana // Config defines the structure for the Kibana module configuration options type Config struct { - XPackEnabled bool `config:"xpack.enabled"` + XPackEnabled bool `config:"xpack.enabled"` + BasePath string `config:"basepath"` } // DefaultConfig returns the default configuration for the Kibana module func DefaultConfig() Config { return Config{ XPackEnabled: false, + BasePath: "", } } diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 056d876ec8b..9e72dd5a8a1 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -67,6 +67,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + if config.BasePath != "" { + http.AddBasePath(config.BasePath) + } + return &MetricSet{ base, http, From ddb848ec8eac28a31d89c418d60f044b88fc2cb1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 15:18:30 -0700 Subject: [PATCH 39/66] More robust implementation of AddBasePath method --- metricbeat/helper/http.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index 3f52cdb6b3d..44bb7c25088 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -25,7 +25,7 @@ import ( "io" "io/ioutil" "net/http" - "regexp" + "net/url" "strings" "time" @@ -151,19 +151,23 @@ func (h *HTTP) SetBody(body []byte) { } // AddBasePath adds the given base path to the URI -func (h *HTTP) AddBasePath(basePath string) { +func (h *HTTP) AddBasePath(basePath string) error { if basePath == "" { - return + return nil } + // Normalize basepath basePath = strings.Trim(basePath, "/") - basePath = "/" + basePath + "/" - re := regexp.MustCompile("(.*//[^/]+)(/)(.*)") - uri := h.GetURI() - uri = re.ReplaceAllString(uri, "$1"+basePath+"$3") + url, err := url.Parse(h.GetURI()) + if err != nil { + return err + } + + url.Path = basePath + url.Path - h.SetURI(uri) + h.SetURI(url.String()) + return nil } // FetchContent makes an HTTP request to the configured url and returns the body content. From c7c2caea45dd303a039c0c1e211c73444bc38d4f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 15:19:40 -0700 Subject: [PATCH 40/66] Updating docs --- metricbeat/docs/modules/kibana.asciidoc | 1 + metricbeat/metricbeat.reference.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/metricbeat/docs/modules/kibana.asciidoc b/metricbeat/docs/modules/kibana.asciidoc index 14bd52bc9cc..0aa7ce4c242 100644 --- a/metricbeat/docs/modules/kibana.asciidoc +++ b/metricbeat/docs/modules/kibana.asciidoc @@ -25,6 +25,7 @@ metricbeat.modules: metricsets: ["status"] period: 10s hosts: ["localhost:5601"] + basepath: "" enabled: true ---- diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index d9f84bdd57a..4acd3a0ffb3 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -351,6 +351,7 @@ metricbeat.modules: metricsets: ["status"] period: 10s hosts: ["localhost:5601"] + basepath: "" enabled: true #----------------------------- Kubernetes Module ----------------------------- From 3ee0678e6ce8db815dd06cd7a3679b6a15199eb5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 15:34:13 -0700 Subject: [PATCH 41/66] Updating kibana module yaml file --- metricbeat/modules.d/kibana.yml.disabled | 1 + 1 file changed, 1 insertion(+) diff --git a/metricbeat/modules.d/kibana.yml.disabled b/metricbeat/modules.d/kibana.yml.disabled index a2476f1b37a..78f769cd65e 100644 --- a/metricbeat/modules.d/kibana.yml.disabled +++ b/metricbeat/modules.d/kibana.yml.disabled @@ -6,5 +6,6 @@ # - status period: 10s hosts: ["localhost:5601"] + #basepath: "" #username: "user" #password: "secret" From 7e4a20d1e11a414f63e4cde57d5a4559457b8ada Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 15:55:32 -0700 Subject: [PATCH 42/66] Fixing up field names --- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index e29b3bb88c5..bb84ad1e9f9 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -90,8 +90,8 @@ var ( "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, - "last_day": c.Dict("last_day", reportingPeriodSchema, c.DictOptional), - "last_7_days": c.Dict("last_7_days", reportingPeriodSchema, c.DictOptional), + "lastDay": c.Dict("last_day", reportingPeriodSchema, c.DictOptional), + "last7Days": c.Dict("last_7_days", reportingPeriodSchema, c.DictOptional), }, c.DictOptional), }, }), From 0f5f90b8be29aaf258f3f64563c956561c04983a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 17:08:48 -0700 Subject: [PATCH 43/66] Fixing field names --- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index bb84ad1e9f9..8d8b48a3936 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -86,7 +86,7 @@ var ( "available": c.Bool("available"), "enabled": c.Bool("enabled"), "browser_type": c.Str("browser_type"), - "all": c.Int("all"), + "_all": c.Int("all"), "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, @@ -115,7 +115,7 @@ var ( }, c.DictOptional) reportingPeriodSchema = s.Schema{ - "all": c.Int("all"), + "_all": c.Int("all"), "csv": reportingCsvDict, "printable_pdf": reportingPrintablePdfDict, "status": reportingStatusDict, From ec62c12fa9a916cd922db45d59adbc93ea36262f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 17:14:18 -0700 Subject: [PATCH 44/66] Adding new reporting usage fields --- metricbeat/module/kibana/stats/data_xpack.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 8d8b48a3936..4a4618ce0c3 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -105,6 +105,14 @@ var ( reportingPrintablePdfDict = c.Dict("printable_pdf", s.Schema{ "available": c.Bool("available"), "total": c.Int("total"), + "app": c.Dict("app", s.Schema{ + "visualization": c.Int("visualization"), + "dashboard": c.Int("dashboard"), + }), + "layout": c.Dict("layout", s.Schema{ + "print": c.Int("print"), + "preserve_layout": c.Int("preserve_layout"), + }), }, c.DictOptional) reportingStatusDict = c.Dict("status", s.Schema{ From 490b07694b1b9a9e39925b714dac74e00f936e33 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Jul 2018 17:32:28 -0700 Subject: [PATCH 45/66] Forgot to update the test fixture --- .../kibana/stats/_meta/test/stats.700.json | 69 ++++++++++++------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/metricbeat/module/kibana/stats/_meta/test/stats.700.json b/metricbeat/module/kibana/stats/_meta/test/stats.700.json index 137065034b2..0f74c8b8dbd 100644 --- a/metricbeat/module/kibana/stats/_meta/test/stats.700.json +++ b/metricbeat/module/kibana/stats/_meta/test/stats.700.json @@ -9,43 +9,42 @@ "snapshot":false, "status":"green" }, - "last_updated":"2018-07-17T17:09:23.926Z", + "last_updated":"2018-07-18T00:32:00.948Z", "collection_interval_ms":5000, "process":{ "memory":{ "heap":{ - "total_bytes":342585344, - "used_bytes":192264704, + "total_bytes":223391744, + "used_bytes":198413592, "size_limit":1501560832 }, - "resident_set_size_bytes":431456256 + "resident_set_size_bytes":347242496 }, - "event_loop_delay":0.37530696392059326, - "pid":32307, - "uptime_ms":27353 + "event_loop_delay":0.25226891040802, + "pid":46426, + "uptime_ms":1753889 }, "os":{ "load":{ - "1m":2.01513671875, - "5m":2.0283203125, - "15m":2.041015625 + "1m":3.50634765625, + "5m":3.76904296875, + "15m":3.54833984375 }, "memory":{ "total_bytes":17179869184, - "free_bytes":66318336, - "used_bytes":17113550848 + "free_bytes":31711232, + "used_bytes":17148157952 }, - "uptime_ms":2160690000 + "uptime_ms":2187246000 }, "response_times":{ - "avg_ms":15, - "max_ms":18 + "max_ms":0 }, "requests":{ - "total":4, + "total":0, "disconnects":0, "status_codes":{ - "200":4 + } }, "concurrent_connections":3, @@ -81,8 +80,16 @@ "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ @@ -94,8 +101,16 @@ "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ @@ -108,8 +123,16 @@ "total":0 }, "printable_pdf":{ - "available":false, - "total":0 + "available":true, + "total":0, + "app":{ + "visualization":0, + "dashboard":0 + }, + "layout":{ + "print":0, + "preserve_layout":0 + } }, "status":{ From 43881aa0181174ab3ca80bb352bedf11e0932ff1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 06:39:06 -0700 Subject: [PATCH 46/66] Alternative basepath implementation --- metricbeat/helper/http.go | 22 ------------- metricbeat/helper/http_test.go | 43 ------------------------- metricbeat/mb/parse/url.go | 34 ++++++++++++++----- metricbeat/mb/parse/url_test.go | 1 + metricbeat/module/kibana/stats/stats.go | 11 +++---- 5 files changed, 31 insertions(+), 80 deletions(-) diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index 44bb7c25088..b01fa781885 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -25,8 +25,6 @@ import ( "io" "io/ioutil" "net/http" - "net/url" - "strings" "time" "github.com/pkg/errors" @@ -150,26 +148,6 @@ func (h *HTTP) SetBody(body []byte) { h.body = body } -// AddBasePath adds the given base path to the URI -func (h *HTTP) AddBasePath(basePath string) error { - if basePath == "" { - return nil - } - - // Normalize basepath - basePath = strings.Trim(basePath, "/") - - url, err := url.Parse(h.GetURI()) - if err != nil { - return err - } - - url.Path = basePath + url.Path - - h.SetURI(url.String()) - return nil -} - // FetchContent makes an HTTP request to the configured url and returns the body content. func (h *HTTP) FetchContent() ([]byte, error) { resp, err := h.FetchResponse() diff --git a/metricbeat/helper/http_test.go b/metricbeat/helper/http_test.go index 74129288450..a3e5d615e4d 100644 --- a/metricbeat/helper/http_test.go +++ b/metricbeat/helper/http_test.go @@ -69,46 +69,3 @@ func TestGetAuthHeaderFromTokenNoFile(t *testing.T) { assert.Equal(t, "", header) assert.Error(t, err) } - -func TestAddBasePath(t *testing.T) { - tests := []struct { - Name, Content, Expected string - }{ - { - "Test with empty basepath", - "", - "http://localhost:9999/some/path", - }, - { - "Test with basepath with no leading or trailing slashes", - "foobar", - "http://localhost:9999/foobar/some/path", - }, - { - "Test with basepath with a leading slash", - "/foobar", - "http://localhost:9999/foobar/some/path", - }, - { - "Test with basepath with a trailing slash", - "foobar/", - "http://localhost:9999/foobar/some/path", - }, - { - "Test with basepath with leading and trailing slashes", - "/foobar/", - "http://localhost:9999/foobar/some/path", - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - http := HTTP{ - uri: "http://localhost:9999/some/path", - } - - http.AddBasePath(test.Content) - assert.Equal(t, test.Expected, http.GetURI()) - }) - } -} diff --git a/metricbeat/mb/parse/url.go b/metricbeat/mb/parse/url.go index af3eb172640..b601cc50665 100644 --- a/metricbeat/mb/parse/url.go +++ b/metricbeat/mb/parse/url.go @@ -31,12 +31,13 @@ import ( // URLHostParserBuilder builds a tailored HostParser for used with host strings // that are URLs. type URLHostParserBuilder struct { - PathConfigKey string - DefaultPath string - DefaultUsername string - DefaultPassword string - DefaultScheme string - QueryParams string + BasePathConfigKey string + PathConfigKey string + DefaultPath string + DefaultUsername string + DefaultPassword string + DefaultScheme string + QueryParams string } // Build returns a new HostParser function whose behavior is influenced by the @@ -49,7 +50,7 @@ func (b URLHostParserBuilder) Build() mb.HostParser { return mb.HostData{}, err } - var user, pass, path string + var user, pass, path, basePath string t, ok := conf["username"] if ok { user, ok = t.(string) @@ -77,8 +78,25 @@ func (b URLHostParserBuilder) Build() mb.HostParser { } else { path = b.DefaultPath } + // Normalize path + path = strings.Trim(path, "/") - return ParseURL(host, b.DefaultScheme, user, pass, path, b.QueryParams) + t, ok = conf[b.BasePathConfigKey] + if ok { + basePath, ok = t.(string) + if !ok { + return mb.HostData{}, errors.Errorf("'%v' config for module %v is not a string", b.BasePathConfigKey, module.Name()) + } + } else { + basePath = "" + } + // Normalize basepath + basePath = strings.Trim(basePath, "/") + + // Combine paths and normalize + fullPath := strings.Trim(basePath+"/"+path, "/") + + return ParseURL(host, b.DefaultScheme, user, pass, fullPath, b.QueryParams) } } diff --git a/metricbeat/mb/parse/url_test.go b/metricbeat/mb/parse/url_test.go index 4f08aa23697..2a4cef4032f 100644 --- a/metricbeat/mb/parse/url_test.go +++ b/metricbeat/mb/parse/url_test.go @@ -114,6 +114,7 @@ func TestURLHostParserBuilder(t *testing.T) { {map[string]interface{}{}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/default"}, {map[string]interface{}{"username": "guest"}, URLHostParserBuilder{}, "http://guest@example.com"}, {map[string]interface{}{"username": "guest", "password": "secret"}, URLHostParserBuilder{}, "http://guest:secret@example.com"}, + {map[string]interface{}{"basepath": "/foo"}, URLHostParserBuilder{BasePathConfigKey: "basepath", DefaultPath: "/default"}, "http://example.com/foo/default"}, } for _, test := range cases { diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 9e72dd5a8a1..51b56579611 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -35,10 +35,10 @@ func init() { var ( hostParser = parse.URLHostParserBuilder{ - DefaultScheme: "http", - PathConfigKey: "path", - DefaultPath: "api/stats", - QueryParams: "extended=true", // make Kibana fetch the cluster_uuid + DefaultScheme: "http", + BasePathConfigKey: "basepath", + DefaultPath: "api/stats", + QueryParams: "extended=true", // make Kibana fetch the cluster_uuid }.Build() ) @@ -67,9 +67,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } - if config.BasePath != "" { - http.AddBasePath(config.BasePath) - } return &MetricSet{ base, From 986fc61101973039ba2fe11aeabd23b4d8b9fef5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 06:55:06 -0700 Subject: [PATCH 47/66] Fixing type of response time fields --- metricbeat/docs/fields.asciidoc | 4 ++-- metricbeat/module/kibana/fields.go | 2 +- metricbeat/module/kibana/stats/_meta/fields.yml | 4 ++-- metricbeat/module/kibana/stats/data.go | 4 ++-- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b5ade08a986..7aa4cc3f7c9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6808,7 +6808,7 @@ Response times metrics *`kibana.stats.response_time.avg.ms`*:: + -- -type: scaled_float +type: long Average response time in milliseconds @@ -6818,7 +6818,7 @@ Average response time in milliseconds *`kibana.stats.response_time.max.ms`*:: + -- -type: scaled_float +type: long Maximum response time in milliseconds diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 403837c127d..094ba66f373 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEmM2O4zYMx+95CmIuc9nxA+RQoGgXaFHMoNh20ENRBIzExOrIkivSmaRPv5A/EseRE2c2wfqw2HHkP38iaZH0E7zRbg5vZokOZwBixNIcHn6rbzzMADSxCqYU490cfpgBADQ/QuF1ZWkGwLkPslDercx6Diu0HO8GsoRMc1iSRGkmEePWPIe/H5jtwz8zgJUhq3leqz6Bw4J6LPGSXUlzWAdfle2dBM+xTl+LBYX3d1Nyo5LN1W611gF0GkLlnsQUBAVJMIqz3vLhjrtrSNcnVLZiobCoKqOPFnSwb7R792H42xnkeL2+/voz+BVITvDZIotRTBhU3tkD8fCeG5V3O1TeOVJytJ8epXeqCoFcHea40HjHSV7r3fo62JeqWFKIuMoacgI9E1Cgpggbd8IUNhQyePFCIDkKLIN/ZwoMCh0wOQ1FZcWUloBN/C868hUfKYqHQP9VxBIXG6kX18qAzBTjLOCdok91wCWnXS0f6KliAmLBpTWck+7Lpv1WBq+I044aZuEET/3eyHW5N/g9lWYHFNrE6Fnvy4Umi7usGD7fgbFCS3qxsh7lZMkFQoDP0Q5EO1DbAeOgMNYaJuWdHhrt8AoqfNhlOeHQJ+f8NYGn81lUHnHcmOsOdOIFbbbcCZ0+eybxa2kfCpQ5jD18cQMAf0brzQbQWq9QSMc0brMrejit3uFXTPq70f8SuSMBLHfTkdn8TwtrCiPfDfwZtxl4q4FLVFQTHfv/xWvK/uVuU58mBKKMtSP16l3YzgTc11o7HqM9J4++eR1RexTe6oQC+NKercpXTq46qQ5Q2nBXj+7gqEO9aTfPTTF5p0A90zSsuYPj4A5kzYvuTvhGIseld0yLGPZbxe9LKwpR9LpCcyDDzfpcip+tL5P89OOGAq5p74Ka9kKh6ZUa3N4X7xm3pqiKK/CSve8x1geC2fXpJKhRhrKXopjoSOFCVzrRPy2XcSzoFNXd6ihG/HcUQ2j7wRgNGZJmOgbjNG3v44sXbM7shucxHtpCwaEdMdoh5f7k0L4R0dAz0dJZ70hAx2UcAVHrcNru3paradUfec9Vd+nR/CjghgIb7+6KNWZj3884LDn34zFbem8JU5ATIP7KSfI41eXUAS0rYzUYBtzbbu6NIwpKdd/gdUn1WPfjVvKh0VGWqyf3P2qF9pT7xiE9kf7fMJynjp70+HjbjwJDu1E9bbdN5qzpRG5O0Mq3jU4aoUmAzMdKb20W/7x9CFr12thICNId0IdL8nMjd5qWMKEiT/oKAzfoQ39qrPQMjn/qgNNhYhwpPcNPYurmihgrU3/QOuUYdyFMni7gkv8m8kKin+8PF+c9ChOmjXuSHsF9DQAA//+Rupdj" + return "eJzEmM2O4zYMx+95CmIuc9nxA+RQoGgXaFHMoNh20ENRBIzExOrIkivSmaRPv5A/EseRE2c2wfqw2HHkP38iaZH0E7zRbg5vZokOZwBixNIcHn6rbzzMADSxCqYU490cfpgBADQ/QuF1ZWkGwLkPslDercx6Diu0HO8GsoRMc1iSRGkmEePWPIe/H5jtwz8zgJUhq3leqz6Bw4J6LPGSXUlzWAdfle2dBM+xTl+LBYX3d1Nyo5LN1W611gF0GkLlnsQUBAVJMIqz3vLhjrtrSNcnVLZiobCoKqOPFnSwb7R792H42xnkeL2+/voz+BVITvDZIotRTBhU3tkD8fCeG5V3O1TeOVJytJ8epXeqCoFcHea40HjHSV7r3fo62JeqWFKIuMoacgI9E1Cgpggbd8IUNhQyePFCIDkKLIN/ZwoMCh0wOQ1FZcWUloBN/C868hUfKYqHQP9VxBIXG6kX18qAzBTjLOCdok91wCWnXS0f6KliAmLBpTWck+7Lpv1WBq+I044aZuEET/3eyHW5N/g9lWYHFNrE6Fnvy4Umi7usGD7fgbFCS3qxsh7lZMkFQoDP0Q5EO1DbAeOgMNYaJuWdHhrt8AoqfNhlOeHQJ+f8NYGn81lUHnHcmOsOdOIFbbbcCZ0+eybxa2kfCpQ5jD18cQMAf0brzQbQWq9QSMc0brMrejit3uFXTPq70f8SuSMBLHfTkdn8TwtrCiPfDfwZtxl4q4FLVFQTHfv/xWvK/uVuU58mBKKMtSP16l3YzgTc11o7HqM9J4++eR1RexTe6oQC+NKercpXTq46qQ5Q2nBXj+7gqEO9aTfPTTF5p0A90zSsuYPj4A5kzYvuTvhGIseld0yLGPZbxe9LKwpR9LpCcyDDzfo+Kf7jhgKuab/1mvJCgemVGNzeB+sZt6aoiiuwkr3uMc4Hgtf15SSoUYayl6KW6EDhQhc60T8tl3Es6BTV3ekoRvx3FENoe9qSfIghaaZjME7T9j6+eMHmjG54HuMhLRQc2hGjHVLuTw7pGxENPRMtnfWOBHRcxpEPtQ6n7e1tuZrW/JH3XHVXHs2PAm4osPHurlhjNvb9i8OScz8es6X3ljAFOQHir5wkj1NcTh3QsjJWg2HAve3m3jiioFT3DV6XVI91/20lHxodZbl6Uv+jVmhPuW8cyhPp/w3DeOroSY+Lt/0IMLQb1dN222TOms7j5gStfNvYpBGaBMh8rPDWZvHP24egVa+NjYQg3fF8uCQ/N3KnaQkTKvKkry5wgwbmp8ZKz+D4pw04HR7GkdIz+ySmbo6IsTL1B6xTjnEXwuRpAi75byIvJPr3/jBx3qMwYbq4J+kR3NcAAAD///XhkQE=" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index fc13b17396c..c1a7241fd64 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -64,11 +64,11 @@ Response times metrics fields: - name: avg.ms - type: scaled_float + type: long description: > Average response time in milliseconds - name: max.ms - type: scaled_float + type: long description: > Maximum response time in milliseconds - name: kibana diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 52f47af2ed0..e5bfb7f9df5 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -54,10 +54,10 @@ var ( "request": RequestsDict, "response_time": c.Dict("response_times", s.Schema{ "avg": s.Object{ - "ms": c.Float("avg_ms", s.Optional), + "ms": c.Int("avg_ms", s.Optional), }, "max": s.Object{ - "ms": c.Float("max_ms"), + "ms": c.Int("max_ms"), }, }), "kibana": KibanaDict, diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 4a4618ce0c3..1db11af24a5 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -57,8 +57,8 @@ var ( }), "requests": RequestsDict, "response_times": c.Dict("response_times", s.Schema{ - "average": c.Float("avg_ms", s.Optional), - "max": c.Float("max_ms"), + "average": c.Int("avg_ms", s.Optional), + "max": c.Int("max_ms"), }, c.DictOptional), "kibana": KibanaDict, "usage": c.Dict("usage", s.Schema{ From 60fa99d7ae1d909aaf9b42b9f71b84873180ed16 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 09:19:49 -0700 Subject: [PATCH 48/66] Add error checking for type assertions --- metricbeat/module/kibana/stats/data_xpack.go | 25 +++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 1db11af24a5..3a4ac8ee92d 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -19,6 +19,7 @@ package stats import ( "encoding/json" + "fmt" "time" "github.com/elastic/beats/libbeat/common" @@ -130,6 +131,12 @@ var ( } ) +func reportErrorForMissingField(field string, r mb.ReporterV2) error { + err := fmt.Errorf("Could not find field '%v' in Kibana stats API response", field) + r.Error(err) + return err +} + func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) @@ -144,9 +151,21 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error return err } - process := data["process"].(map[string]interface{}) - memory := process["memory"].(map[string]interface{}) - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(memory["resident_set_size_bytes"].(float64))) + process, ok := data["process"].(map[string]interface{}) + if !ok { + return reportErrorForMissingField("process", r) + } + memory, ok := process["memory"].(map[string]interface{}) + if !ok { + return reportErrorForMissingField("process.memory", r) + } + + rss, ok := memory["resident_set_size_bytes"].(float64) + if !ok { + return reportErrorForMissingField("process.memory.resident_set_size_bytes", r) + } + + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(rss)) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From 03b3b3527c565605d2be80384ca9796fea9dd63e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 09:30:59 -0700 Subject: [PATCH 49/66] Moving Kibana metadata fields into top-level --- metricbeat/docs/fields.asciidoc | 167 +++++++++--------- metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/data.json | 40 ++--- .../module/kibana/stats/_meta/fields.yml | 69 ++++---- metricbeat/module/kibana/stats/data.go | 21 +-- metricbeat/module/kibana/stats/data_xpack.go | 11 +- 6 files changed, 149 insertions(+), 161 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7aa4cc3f7c9..42ca7fde09f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6681,233 +6681,226 @@ Kibana stats and run-time metrics. -*`kibana.stats.cluster_uuid`*:: +*`kibana.stats.uuid`*:: + -- type: keyword -UUID of the Elasticsearch cluster to which Kibana connects. +Kibana instance UUID -- -*`kibana.stats.concurrent_connections`*:: +*`kibana.stats.name`*:: + -- -type: long +type: text -Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. +Kibana instance name -- -[float] -== process fields - -Process metrics - - - -*`kibana.stats.process.event_loop_delay.ms`*:: +*`kibana.stats.index`*:: + -- -type: scaled_float +type: keyword -Event loop delay in milliseconds +Name of Kibana's internal index -- -[float] -== memory.heap fields +*`kibana.stats.host`*:: ++ +-- +type: keyword -Process heap metrics +Kibana instance hostname +-- -*`kibana.stats.process.memory.heap.total.bytes`*:: +*`kibana.stats.transport_address`*:: + -- -type: long - -format: bytes +type: keyword -Total heap allocated to process in bytes +Kibana server's hostname and port -- -*`kibana.stats.process.memory.heap.used.bytes`*:: +*`kibana.stats.version`*:: + -- -type: long - -format: bytes +type: keyword -Heap used by process in bytes +Kibana version -- -*`kibana.stats.process.memory.heap.size_limit.bytes`*:: +*`kibana.stats.snapshot`*:: + -- -type: long - -format: bytes +type: boolean -Max. old space size allocated to Node.js process, in bytes +Whether the Kibana build is a snapshot build -- -*`kibana.stats.process.memory.heap.uptime.ms`*:: +*`kibana.stats.status`*:: + -- -type: long +type: keyword -Uptime of process in milliseconds +Kibana instance's health status -- -[float] -== request fields +*`kibana.stats.cluster_uuid`*:: ++ +-- +type: keyword -Request count metrics +UUID of the Elasticsearch cluster to which Kibana connects. +-- -*`kibana.stats.request.disconnects`*:: +*`kibana.stats.concurrent_connections`*:: + -- type: long -Number of requests that were disconnected +Number of client connections made to the server. Note that browsers can send multiple simultaneous connections to request mulitple server assets at once, and they can re-use established connections. -- -*`kibana.stats.request.total`*:: +[float] +== process fields + +Process metrics + + + +*`kibana.stats.process.event_loop_delay.ms`*:: + -- -type: long +type: scaled_float -Total number of requests +Event loop delay in milliseconds -- [float] -== response_time fields +== memory.heap fields -Response times metrics +Process heap metrics -*`kibana.stats.response_time.avg.ms`*:: +*`kibana.stats.process.memory.heap.total.bytes`*:: + -- type: long -Average response time in milliseconds +format: bytes + +Total heap allocated to process in bytes -- -*`kibana.stats.response_time.max.ms`*:: +*`kibana.stats.process.memory.heap.used.bytes`*:: + -- type: long -Maximum response time in milliseconds - - --- - -[float] -== kibana fields +format: bytes -Kibana metadata +Heap used by process in bytes +-- -*`kibana.stats.kibana.uuid`*:: +*`kibana.stats.process.memory.heap.size_limit.bytes`*:: + -- -type: keyword +type: long -Kibana instance UUID +format: bytes + +Max. old space size allocated to Node.js process, in bytes -- -*`kibana.stats.kibana.name`*:: +*`kibana.stats.process.memory.heap.uptime.ms`*:: + -- -type: text +type: long -Kibana instance name +Uptime of process in milliseconds -- -*`kibana.stats.kibana.index`*:: -+ --- -type: keyword +[float] +== request fields -Name of Kibana's internal index +Request count metrics --- -*`kibana.stats.kibana.host`*:: +*`kibana.stats.request.disconnects`*:: + -- -type: keyword +type: long -Kibana instance hostname +Number of requests that were disconnected -- -*`kibana.stats.kibana.transport_address`*:: +*`kibana.stats.request.total`*:: + -- -type: keyword +type: long -Kibana server's hostname and port +Total number of requests -- -*`kibana.stats.kibana.version`*:: -+ --- -type: keyword +[float] +== response_time fields -Kibana version +Response times metrics --- -*`kibana.stats.kibana.snapshot`*:: +*`kibana.stats.response_time.avg.ms`*:: + -- -type: boolean +type: long -Whether the Kibana build is a snapshot build +Average response time in milliseconds -- -*`kibana.stats.kibana.status`*:: +*`kibana.stats.response_time.max.ms`*:: + -- -type: keyword +type: long -Kibana instance's health status +Maximum response time in milliseconds -- diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 094ba66f373..3081cefa301 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEmM2O4zYMx+95CmIuc9nxA+RQoGgXaFHMoNh20ENRBIzExOrIkivSmaRPv5A/EseRE2c2wfqw2HHkP38iaZH0E7zRbg5vZokOZwBixNIcHn6rbzzMADSxCqYU490cfpgBADQ/QuF1ZWkGwLkPslDercx6Diu0HO8GsoRMc1iSRGkmEePWPIe/H5jtwz8zgJUhq3leqz6Bw4J6LPGSXUlzWAdfle2dBM+xTl+LBYX3d1Nyo5LN1W611gF0GkLlnsQUBAVJMIqz3vLhjrtrSNcnVLZiobCoKqOPFnSwb7R792H42xnkeL2+/voz+BVITvDZIotRTBhU3tkD8fCeG5V3O1TeOVJytJ8epXeqCoFcHea40HjHSV7r3fo62JeqWFKIuMoacgI9E1Cgpggbd8IUNhQyePFCIDkKLIN/ZwoMCh0wOQ1FZcWUloBN/C868hUfKYqHQP9VxBIXG6kX18qAzBTjLOCdok91wCWnXS0f6KliAmLBpTWck+7Lpv1WBq+I044aZuEET/3eyHW5N/g9lWYHFNrE6Fnvy4Umi7usGD7fgbFCS3qxsh7lZMkFQoDP0Q5EO1DbAeOgMNYaJuWdHhrt8AoqfNhlOeHQJ+f8NYGn81lUHnHcmOsOdOIFbbbcCZ0+eybxa2kfCpQ5jD18cQMAf0brzQbQWq9QSMc0brMrejit3uFXTPq70f8SuSMBLHfTkdn8TwtrCiPfDfwZtxl4q4FLVFQTHfv/xWvK/uVuU58mBKKMtSP16l3YzgTc11o7HqM9J4++eR1RexTe6oQC+NKercpXTq46qQ5Q2nBXj+7gqEO9aTfPTTF5p0A90zSsuYPj4A5kzYvuTvhGIseld0yLGPZbxe9LKwpR9LpCcyDDzfo+Kf7jhgKuab/1mvJCgemVGNzeB+sZt6aoiiuwkr3uMc4Hgtf15SSoUYayl6KW6EDhQhc60T8tl3Es6BTV3ekoRvx3FENoe9qSfIghaaZjME7T9j6+eMHmjG54HuMhLRQc2hGjHVLuTw7pGxENPRMtnfWOBHRcxpEPtQ6n7e1tuZrW/JH3XHVXHs2PAm4osPHurlhjNvb9i8OScz8es6X3ljAFOQHir5wkj1NcTh3QsjJWg2HAve3m3jiioFT3DV6XVI91/20lHxodZbl6Uv+jVmhPuW8cyhPp/w3DeOroSY+Lt/0IMLQb1dN222TOms7j5gStfNvYpBGaBMh8rPDWZvHP24egVa+NjYQg3fF8uCQ/N3KnaQkTKvKkry5wgwbmp8ZKz+D4pw04HR7GkdIz+ySmbo6IsTL1B6xTjnEXwuRpAi75byIvJPr3/jBx3qMwYbq4J+kR3NcAAAD///XhkQE=" + return "eJzEl91u6zYMx+/zFERvzs2pHyAXA4btABuGFsPZil0MQ0BLTKxVljyRTpM9/SB/pI4jN+6Og/qiaPzx54+kSEr38EzHNTybHB2uAMSIpTXc/dLcuFsBaGIVTCXGuzV8twIAaB9C6XVtaQXAhQ+yUd5tzW4NW7Qc7wayhExryEmiNJOIcTtew593zPburxXA1pDVvG5U78FhSQOWeMmxojXsgq+r7k6C51xnqMWCwqe7KblJyfbqXG10AJ2GULt7MSVBSRKM4mzw+tjj/hrTDQnr2uizBz3kMx1ffBg/ewN1gGscCzpF8PT0849Js/Fv0qzQQb7N5oV0b9M4TYflfH3EksBvO/ufGIwTCg5twlCPUHgee7dgtKP6pPcS0HEVCwW1DsS8OAdT2FP4xCeOZsFGk0mgPQU23i2OkdI91aPDigufzkHuvSUcA10x+kdBUlAAKagHyGtjNRgGPNlr76WRBKVePhn9oojpILRSpAz1DMrWLBQ2yzaDWPyxQGJovlhkMYoJgyp6eyAeXgqjih5aeedIyVlTG1B6p+oQyDW9Pr5ovEtHznq3e2c112VOIeIqa8gJDExAiZoibPSkXeUZPHohkAIF8uBfmAKDQgdMTkNZWzGVJWAT/0VHvuYzRfEQ6J+aWOLLRpqXG2VAZorNXsA7RZ+bIpKCjo18oPuaCYgFc2u4ID2UTcetCl5N1ft4FM2I1K+tXD+ARs9Ts+YVhfYxe9b7aqPJ4jErx9/3YKzQkt5srcdxtV4lBPgS7UC0A40dMA5KY61hUt7psdEer6TSh2NWEI5j8la8ZvD0MYvKE4GbCt2gg3tBm+VHoctv31j4jbQPJcoapj6+6gDA79F66wBa6xUK6biMu9UVI5xWP+00mPSH0f8UuSMB5Mf5yGz+pY01pZEPA3/AQwbeauAKFTVE5/F/9Jqyv7l36vOMRFRxA5kqvSvuzMB9arRjGx0EebLyeqKuFS7VoQC+dr1V+drJuzrVK5Q23M+jGwTqdd50znM7TF4o0MA0jWfuqB3cgKwtdHfBN5E5rrxj2sS0L5W/r50oRNH3DZpXMtzvbrPEv99TwB2dXG8orwyYwYjBw22wHvBgyrqciTW5+3z3CfW3RqFLxzceRidPhYscj6J6eod020NwVE/b7Y4qWVtstzoJdbWcRmgXQObjorY2iz+XT0Gn3hibSEG6yP93C3lo5S6XJcxoHbMOGrBAzf7QWhkYnN7Nw+W8nEZKb1NnMfWjM+bKNGe2S47pEMLsAQrX4jeTFxIjazg/344ozBiotyQ9g/svAAD//z5MYYQ=" } diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index 7de7dfe32e4..fce566d99fb 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -6,25 +6,18 @@ }, "elasticsearch": { "cluster": { - "id": "NkfU5AinRnyFnqBD36zhEw" + "id": "A0sjozJKQoSRmOe2LV9kWA" } }, "kibana": { "stats": { - "concurrent_connections": 3, - "kibana": { - "host": "localhost", - "index": ".kibana", - "name": "Shaunaks-MBP-2", - "snapshot": false, - "status": "green", - "transport_address": "localhost:5601", - "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d", - "version": "7.0.0-alpha1" - }, + "concurrent_connections": 1, + "host": "localhost", + "index": "Shaunaks-MBP-2", + "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.7881790399551392 + "ms": 0.3324570655822754 }, "memory": { "heap": { @@ -32,29 +25,32 @@ "bytes": 1501560832 }, "total": { - "bytes": 477569024 + "bytes": 348958720 }, "used": { - "bytes": 205526576 + "bytes": 191876376 } } }, "uptime": { - "ms": 24803 + "ms": 28184 } }, "request": { "disconnects": 0, - "total": 3 + "total": 0 }, "response_time": { - "avg": { - "ms": 39 - }, + "avg": {}, "max": { - "ms": 48 + "ms": 0 } - } + }, + "snapshot": false, + "status": "green", + "transport_address": "localhost:5601", + "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d", + "version": "7.0.0-alpha1" } }, "metricset": { diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index c1a7241fd64..21fceca4733 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -4,6 +4,38 @@ Kibana stats and run-time metrics. release: beta fields: + - name: uuid + type: keyword + description: > + Kibana instance UUID + - name: name + type: text + description: > + Kibana instance name + - name: index + type: keyword + description: > + Name of Kibana's internal index + - name: host + type: keyword + description: > + Kibana instance hostname + - name: transport_address + type: keyword + description: > + Kibana server's hostname and port + - name: version + type: keyword + description: > + Kibana version + - name: snapshot + type: boolean + description: > + Whether the Kibana build is a snapshot build + - name: status + type: keyword + description: > + Kibana instance's health status - name: cluster_uuid type: keyword description: > @@ -71,40 +103,3 @@ type: long description: > Maximum response time in milliseconds - - name: kibana - type: group - description: > - Kibana metadata - fields: - - name: uuid - type: keyword - description: > - Kibana instance UUID - - name: name - type: text - description: > - Kibana instance name - - name: index - type: keyword - description: > - Name of Kibana's internal index - - name: host - type: keyword - description: > - Kibana instance hostname - - name: transport_address - type: keyword - description: > - Kibana server's hostname and port - - name: version - type: keyword - description: > - Kibana version - - name: snapshot - type: boolean - description: > - Whether the Kibana build is a snapshot build - - name: status - type: keyword - description: > - Kibana instance's health status diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index e5bfb7f9df5..755e1c701c9 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -28,6 +28,14 @@ import ( var ( schema = s.Schema{ + "uuid": c.Str("kibana.uuid"), + "name": c.Str("kibana.name"), + "index": c.Str("kibana.name"), + "host": c.Str("kibana.host"), + "transport_address": c.Str("kibana.transport_address"), + "version": c.Str("kibana.version"), + "snapshot": c.Bool("kibana.snapshot"), + "status": c.Str("kibana.status"), "cluster_uuid": c.Str("cluster_uuid"), "concurrent_connections": c.Int("concurrent_connections"), "process": c.Dict("process", s.Schema{ @@ -60,7 +68,6 @@ var ( "ms": c.Int("max_ms"), }, }), - "kibana": KibanaDict, } // RequestsDict defines how to convert the requests field @@ -68,18 +75,6 @@ var ( "disconnects": c.Int("disconnects", s.Optional), "total": c.Int("total", s.Optional), }) - - // KibanaDict defines how to convert the kibana field - KibanaDict = c.Dict("kibana", s.Schema{ - "uuid": c.Str("uuid"), - "name": c.Str("name"), - "index": c.Str("index"), - "host": c.Str("host"), - "transport_address": c.Str("transport_address"), - "version": c.Str("version"), - "snapshot": c.Bool("snapshot"), - "status": c.Str("status"), - }) ) func eventMapping(r mb.ReporterV2, content []byte) error { diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 3a4ac8ee92d..6468175785e 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -61,7 +61,16 @@ var ( "average": c.Int("avg_ms", s.Optional), "max": c.Int("max_ms"), }, c.DictOptional), - "kibana": KibanaDict, + "kibana": c.Dict("kibana", s.Schema{ + "uuid": c.Str("uuid"), + "name": c.Str("name"), + "index": c.Str("index"), + "host": c.Str("host"), + "transport_address": c.Str("transport_address"), + "version": c.Str("version"), + "snapshot": c.Bool("snapshot"), + "status": c.Str("status"), + }), "usage": c.Dict("usage", s.Schema{ "index": c.Str("kibana.index"), "index_pattern": c.Dict("kibana.index_pattern", s.Schema{ From 8a3feeef4d3a973a22959316541c388787afaec7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 09:39:04 -0700 Subject: [PATCH 50/66] Adding process.pid field --- metricbeat/docs/fields.asciidoc | 10 ++++++++++ metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/data.json | 19 +++++++++++-------- .../module/kibana/stats/_meta/fields.yml | 4 ++++ metricbeat/module/kibana/stats/data.go | 1 + 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 42ca7fde09f..7779d770223 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6788,6 +6788,16 @@ Process metrics +*`kibana.stats.process.pid`*:: ++ +-- +type: long + +Process ID of the Kibana instance + + +-- + *`kibana.stats.process.event_loop_delay.ms`*:: + -- diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 3081cefa301..c5745572b83 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEl91u6zYMx+/zFERvzs2pHyAXA4btABuGFsPZil0MQ0BLTKxVljyRTpM9/SB/pI4jN+6Og/qiaPzx54+kSEr38EzHNTybHB2uAMSIpTXc/dLcuFsBaGIVTCXGuzV8twIAaB9C6XVtaQXAhQ+yUd5tzW4NW7Qc7wayhExryEmiNJOIcTtew593zPburxXA1pDVvG5U78FhSQOWeMmxojXsgq+r7k6C51xnqMWCwqe7KblJyfbqXG10AJ2GULt7MSVBSRKM4mzw+tjj/hrTDQnr2uizBz3kMx1ffBg/ewN1gGscCzpF8PT0849Js/Fv0qzQQb7N5oV0b9M4TYflfH3EksBvO/ufGIwTCg5twlCPUHgee7dgtKP6pPcS0HEVCwW1DsS8OAdT2FP4xCeOZsFGk0mgPQU23i2OkdI91aPDigufzkHuvSUcA10x+kdBUlAAKagHyGtjNRgGPNlr76WRBKVePhn9oojpILRSpAz1DMrWLBQ2yzaDWPyxQGJovlhkMYoJgyp6eyAeXgqjih5aeedIyVlTG1B6p+oQyDW9Pr5ovEtHznq3e2c112VOIeIqa8gJDExAiZoibPSkXeUZPHohkAIF8uBfmAKDQgdMTkNZWzGVJWAT/0VHvuYzRfEQ6J+aWOLLRpqXG2VAZorNXsA7RZ+bIpKCjo18oPuaCYgFc2u4ID2UTcetCl5N1ft4FM2I1K+tXD+ARs9Ts+YVhfYxe9b7aqPJ4jErx9/3YKzQkt5srcdxtV4lBPgS7UC0A40dMA5KY61hUt7psdEer6TSh2NWEI5j8la8ZvD0MYvKE4GbCt2gg3tBm+VHoctv31j4jbQPJcoapj6+6gDA79F66wBa6xUK6biMu9UVI5xWP+00mPSH0f8UuSMB5Mf5yGz+pY01pZEPA3/AQwbeauAKFTVE5/F/9Jqyv7l36vOMRFRxA5kqvSvuzMB9arRjGx0EebLyeqKuFS7VoQC+dr1V+drJuzrVK5Q23M+jGwTqdd50znM7TF4o0MA0jWfuqB3cgKwtdHfBN5E5rrxj2sS0L5W/r50oRNH3DZpXMtzvbrPEv99TwB2dXG8orwyYwYjBw22wHvBgyrqciTW5+3z3CfW3RqFLxzceRidPhYscj6J6eod020NwVE/b7Y4qWVtstzoJdbWcRmgXQObjorY2iz+XT0Gn3hibSEG6yP93C3lo5S6XJcxoHbMOGrBAzf7QWhkYnN7Nw+W8nEZKb1NnMfWjM+bKNGe2S47pEMLsAQrX4jeTFxIjazg/344ozBiotyQ9g/svAAD//z5MYYQ=" + return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpL/85tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJFGaScS4LS/hzydm+/TXAmBjyGpe1qrP4LCgHku85FDSErbBV2V7J8FzrtPXYkHh492U3Khkc7Wu1jqATkOo3LOYgqAgCUZx1nt96HF3Den6hFVl9NmDDvKNDu8+DJ9dQe3hGseCThG8vv78Y9Js/DdpVmgvX2fzQrqzaZym/Xy+vmBB4Det/Q8MxgkFhzZhqEPIPQ+9mzHaUX3UewnouIyNgloHYp6dgynsKHzgI0ddsNFkEmhHgY13s2OkdI/96LDk3KdzsPbeEg6Bbhj9IyfJKYDk1AGsK2M1GAY82mvupZEEpZo/GV1RxHQQWslThjoGZSsWCqt5h0Fs/tggMTSfLLIYxYRB5Z09EA/vuVF5B628c6TkbKj1KL1TVQjk6lkfXzTepSNnvdve2c1VsaYQcZU15AR6JqBATRE2etJUeQYvXggkR4F18O9MgUGhAyanoaismNISsIn/RUe+4jNF8RDon4pY4stG6pdrZUBmisNewDtFH+smkpwOtXyg54oJiAXX1nBOui+bjlsZvBrr9+EqmhCpXxu5bgENnqd2TQ/lor6uZOwmyYnlVGiD8h/hoF2sIut9udJk8ZAVQz86LlZoSa821uNwakzg+xTtQLQDtR0wDgpjrWFS3umh0Q6voMKHQ5YTDnNzLW93xCsqjyRwLIW9TeIFbbY+CF3+9mo6ATY+FChLGPvxTQcAfo/WGwfQWq9QSMd2aqs8RjitfvziYdLfjP6nyB0JYH2YjszmX1pZUxj5ZuCfcZ+Btxq4REU10Xn8X7ym7G/unPo4IRFl/JBNtd4NdybgvtbacSj0gjzaeR1RO5LnmpQAX9oZr3zl5K6JeYLShru9+IBAnfZe6zw3S+2dAvVM0+XcPhsHDyBrGt1d8I1kjkvvmFYx7XPl70srClH0voV3IsPd9jEl/v2OAm7p6HpNeWPB9FYM7h+D9Rn3pqiKiVijX8F3n5R/qxXadHzloXj0dDrLMS2qp7/UHnsYj+ppu+2RKWua7VEnsraX0whNAWQ+FrW1Wfxz/hS06rWxkRSkm/x/j5DPjdxlWcKE0THpwAMz9OwPjZWewfFTBVzuy3Gk9GfqJKZudcZcmfrseMkxHkKYvEDhVvwm8kJiZfX35/WIwoSF+kjSM7j/AgAA//9sJ4H2" } diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index fce566d99fb..639764d8323 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -11,13 +11,13 @@ }, "kibana": { "stats": { - "concurrent_connections": 1, + "concurrent_connections": 0, "host": "localhost", "index": "Shaunaks-MBP-2", "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.3324570655822754 + "ms": 0.36442995071411133 }, "memory": { "heap": { @@ -25,25 +25,28 @@ "bytes": 1501560832 }, "total": { - "bytes": 348958720 + "bytes": 225144832 }, "used": { - "bytes": 191876376 + "bytes": 181993128 } } }, + "pid": 59876, "uptime": { - "ms": 28184 + "ms": 69018 } }, "request": { "disconnects": 0, - "total": 0 + "total": 1 }, "response_time": { - "avg": {}, + "avg": { + "ms": 25 + }, "max": { - "ms": 0 + "ms": 25 } }, "snapshot": false, diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 21fceca4733..e9817c7eb88 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -49,6 +49,10 @@ description: > Process metrics fields: + - name: pid + type: long + description: > + Process ID of the Kibana instance - name: event_loop_delay.ms type: scaled_float description: > diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 755e1c701c9..b6e2aba95f9 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -39,6 +39,7 @@ var ( "cluster_uuid": c.Str("cluster_uuid"), "concurrent_connections": c.Int("concurrent_connections"), "process": c.Dict("process", s.Schema{ + "pid": c.Int("pid"), "event_loop_delay": s.Object{ "ms": c.Float("event_loop_delay"), }, From 97318e08032218abf809093d2a597e7ba3bc3341 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 09:39:38 -0700 Subject: [PATCH 51/66] Making response_times.max_ms optional to match response_times.avg_ms --- metricbeat/module/kibana/stats/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index b6e2aba95f9..1eb46993d09 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -66,7 +66,7 @@ var ( "ms": c.Int("avg_ms", s.Optional), }, "max": s.Object{ - "ms": c.Int("max_ms"), + "ms": c.Int("max_ms", s.Optional), }, }), } From b6ac52271d694d456fc74bf403bf9b2c717218b7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 09:54:34 -0700 Subject: [PATCH 52/66] Removing extraneous newline --- metricbeat/module/kibana/stats/stats.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 51b56579611..c2dcace0506 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -67,7 +67,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } - return &MetricSet{ base, http, From 6b4e8848f8dab97b3dc818167d8d375b8394537b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 12:03:25 -0700 Subject: [PATCH 53/66] Making response_times.max_ms optional to match response_times.avg_ms --- metricbeat/module/kibana/stats/data_xpack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 6468175785e..16288003ada 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -59,7 +59,7 @@ var ( "requests": RequestsDict, "response_times": c.Dict("response_times", s.Schema{ "average": c.Int("avg_ms", s.Optional), - "max": c.Int("max_ms"), + "max": c.Int("max_ms", s.Optional), }, c.DictOptional), "kibana": c.Dict("kibana", s.Schema{ "uuid": c.Str("uuid"), From d251761c5727681e44e533cebed9509f305459b1 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 18 Jul 2018 13:15:37 -0700 Subject: [PATCH 54/66] Marking optional fields as optional --- metricbeat/module/kibana/stats/data_xpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 16288003ada..4130a418b0c 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -118,11 +118,11 @@ var ( "app": c.Dict("app", s.Schema{ "visualization": c.Int("visualization"), "dashboard": c.Int("dashboard"), - }), + }, c.DictOptional), "layout": c.Dict("layout", s.Schema{ "print": c.Int("print"), "preserve_layout": c.Int("preserve_layout"), - }), + }, c.DictOptional), }, c.DictOptional) reportingStatusDict = c.Dict("status", s.Schema{ From 0b8dc9a5551ae90f17c185d32e2b4d351772afef Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 07:04:53 -0700 Subject: [PATCH 55/66] Remove redundant else --- metricbeat/mb/parse/url.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/mb/parse/url.go b/metricbeat/mb/parse/url.go index b601cc50665..6abb5240976 100644 --- a/metricbeat/mb/parse/url.go +++ b/metricbeat/mb/parse/url.go @@ -87,8 +87,6 @@ func (b URLHostParserBuilder) Build() mb.HostParser { if !ok { return mb.HostData{}, errors.Errorf("'%v' config for module %v is not a string", b.BasePathConfigKey, module.Name()) } - } else { - basePath = "" } // Normalize basepath basePath = strings.Trim(basePath, "/") From 482f5c7bf168dba974e4e237bb18d27b4a83f652 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 07:07:58 -0700 Subject: [PATCH 56/66] Remove BasePath from Kibana config struct --- metricbeat/module/kibana/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metricbeat/module/kibana/config.go b/metricbeat/module/kibana/config.go index 2a8e55008c6..748ba6bb5e3 100644 --- a/metricbeat/module/kibana/config.go +++ b/metricbeat/module/kibana/config.go @@ -19,14 +19,12 @@ package kibana // Config defines the structure for the Kibana module configuration options type Config struct { - XPackEnabled bool `config:"xpack.enabled"` - BasePath string `config:"basepath"` + XPackEnabled bool `config:"xpack.enabled"` } // DefaultConfig returns the default configuration for the Kibana module func DefaultConfig() Config { return Config{ XPackEnabled: false, - BasePath: "", } } From d3e54c3bf2d073afebe4edc58f76a9cabf0b1b4f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 07:11:10 -0700 Subject: [PATCH 57/66] Use path.Join instead of hardcoding "/" --- metricbeat/mb/parse/url.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/mb/parse/url.go b/metricbeat/mb/parse/url.go index 6abb5240976..c3d71936877 100644 --- a/metricbeat/mb/parse/url.go +++ b/metricbeat/mb/parse/url.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "net/url" + p "path" "strings" "github.com/elastic/beats/metricbeat/mb" @@ -92,7 +93,7 @@ func (b URLHostParserBuilder) Build() mb.HostParser { basePath = strings.Trim(basePath, "/") // Combine paths and normalize - fullPath := strings.Trim(basePath+"/"+path, "/") + fullPath := strings.Trim(p.Join(basePath, path), "/") return ParseURL(host, b.DefaultScheme, user, pass, fullPath, b.QueryParams) } From a9d629544aeb07b005f28005ab9138d05e81ed51 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 07:12:33 -0700 Subject: [PATCH 58/66] Adding variations of basepath to unit test --- metricbeat/mb/parse/url_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metricbeat/mb/parse/url_test.go b/metricbeat/mb/parse/url_test.go index 2a4cef4032f..3d4fede58ff 100644 --- a/metricbeat/mb/parse/url_test.go +++ b/metricbeat/mb/parse/url_test.go @@ -115,6 +115,9 @@ func TestURLHostParserBuilder(t *testing.T) { {map[string]interface{}{"username": "guest"}, URLHostParserBuilder{}, "http://guest@example.com"}, {map[string]interface{}{"username": "guest", "password": "secret"}, URLHostParserBuilder{}, "http://guest:secret@example.com"}, {map[string]interface{}{"basepath": "/foo"}, URLHostParserBuilder{BasePathConfigKey: "basepath", DefaultPath: "/default"}, "http://example.com/foo/default"}, + {map[string]interface{}{"basepath": "foo/"}, URLHostParserBuilder{BasePathConfigKey: "basepath", DefaultPath: "/default"}, "http://example.com/foo/default"}, + {map[string]interface{}{"basepath": "/foo/"}, URLHostParserBuilder{BasePathConfigKey: "basepath", DefaultPath: "/default"}, "http://example.com/foo/default"}, + {map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{BasePathConfigKey: "basepath", DefaultPath: "/default"}, "http://example.com/foo/default"}, } for _, test := range cases { From b75280dd47b99cfd15716c7f7fa0e9a6410df2fb Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 07:18:36 -0700 Subject: [PATCH 59/66] Renaming field host -> host.name --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/data.json | 29 ++++++++----------- .../module/kibana/stats/_meta/fields.yml | 2 +- metricbeat/module/kibana/stats/data.go | 10 ++++--- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7779d770223..de525493846 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6711,7 +6711,7 @@ Name of Kibana's internal index -- -*`kibana.stats.host`*:: +*`kibana.stats.host.name`*:: + -- type: keyword diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index c5745572b83..d53f8bab0f4 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpL/85tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJFGaScS4LS/hzydm+/TXAmBjyGpe1qrP4LCgHku85FDSErbBV2V7J8FzrtPXYkHh492U3Khkc7Wu1jqATkOo3LOYgqAgCUZx1nt96HF3Den6hFVl9NmDDvKNDu8+DJ9dQe3hGseCThG8vv78Y9Js/DdpVmgvX2fzQrqzaZym/Xy+vmBB4Det/Q8MxgkFhzZhqEPIPQ+9mzHaUX3UewnouIyNgloHYp6dgynsKHzgI0ddsNFkEmhHgY13s2OkdI/96LDk3KdzsPbeEg6Bbhj9IyfJKYDk1AGsK2M1GAY82mvupZEEpZo/GV1RxHQQWslThjoGZSsWCqt5h0Fs/tggMTSfLLIYxYRB5Z09EA/vuVF5B628c6TkbKj1KL1TVQjk6lkfXzTepSNnvdve2c1VsaYQcZU15AR6JqBATRE2etJUeQYvXggkR4F18O9MgUGhAyanoaismNISsIn/RUe+4jNF8RDon4pY4stG6pdrZUBmisNewDtFH+smkpwOtXyg54oJiAXX1nBOui+bjlsZvBrr9+EqmhCpXxu5bgENnqd2TQ/lor6uZOwmyYnlVGiD8h/hoF2sIut9udJk8ZAVQz86LlZoSa821uNwakzg+xTtQLQDtR0wDgpjrWFS3umh0Q6voMKHQ5YTDnNzLW93xCsqjyRwLIW9TeIFbbY+CF3+9mo6ATY+FChLGPvxTQcAfo/WGwfQWq9QSMd2aqs8RjitfvziYdLfjP6nyB0JYH2YjszmX1pZUxj5ZuCfcZ+Btxq4REU10Xn8X7ym7G/unPo4IRFl/JBNtd4NdybgvtbacSj0gjzaeR1RO5LnmpQAX9oZr3zl5K6JeYLShru9+IBAnfZe6zw3S+2dAvVM0+XcPhsHDyBrGt1d8I1kjkvvmFYx7XPl70srClH0voV3IsPd9jEl/v2OAm7p6HpNeWPB9FYM7h+D9Rn3pqiKiVijX8F3n5R/qxXadHzloXj0dDrLMS2qp7/UHnsYj+ppu+2RKWua7VEnsraX0whNAWQ+FrW1Wfxz/hS06rWxkRSkm/x/j5DPjdxlWcKE0THpwAMz9OwPjZWewfFTBVzuy3Gk9GfqJKZudcZcmfrseMkxHkKYvEDhVvwm8kJiZfX35/WIwoSF+kjSM7j/AgAA//9sJ4H2" + return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpLn/5tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJNE0k4hxW17Cn0/M9umvBcDGkNW8rK0+g8OCeizxkkNJS9gGX5XtnQTPuZ2+LRYUPt5NmRs12Vytq7UdQKchVO5ZTEFQkASjOOu9PvS4u4Z0fcKqMvrsQQf5Rod3H4bPrqD2cI1jQacIXl9//jEpG/9Nygrt5es0L0x3msZp2s/n6wsWBH7T6n9gME4oOLQJoQ4h9yzZqO+zhDxKjIZAAjouY7eg1oGYZ+dgCjsKH/jIUVdtlEwC7Siw8W52jJTdY1M6LDn3wzJrRNfeW8Ih0A3RP3KSnAJITh3AujJWg2HAo15zL40kKNX8yeiKIqaD0EqeEuoYlK1YKKzmnQhxAsQuiaH5ZJHFKCYMKu/0QDy850blHbTyzpGSs8nWo/ROVSGQqwd+fNF4l46c9W57Z0tXxZpCxFXWkBPoSUCBmiJs9KSp8gxevBBIjgLr4N+ZAoNCB0xOQ1FZMaUlYBP/i458xWcWxUOgfypiiS8bqV+uLQMyU5z4At4p+lg3keR0qM0Heq6YgFhwbQ3npPtm03Erg1dj/T7cRxMi9WtjrttCg+ephdNDuaivKxm7SXJiORXaoPxHOGgXq8h6X640WTxkxdCPjosVWtKrjfU4nBoT+D5FHYg6UOuAcVAYaw2T8k4PRTu8ggofDllOOMzNtbzdEa9oeSSBYynsbRIvaLP1Qejyt1fTCbDxoUBZwtiPbzoA8HtUbxxAa71CIR3bqa3yGOG09eNnD5P+ZvQ/Re5IAOvDdGQ2/9LKmsLINwP/jPsMvNXAJSqqic7j/+I1ZX9z59THCYko49dsqvVuuDMB97W2HYdCL8ijndcRtSN5rkkJ8KWd8cpXTu6amCcobbjbiw8I1Gnvtc5zs9TeKVBPmi7n9tk4eABZ0+jugm8kc1x6x7SKaZ8rf19aoxCN3rfwTmS42z6mxL/fUcAtHV2vKW8smN6Kwf1jsD7j3hRVMRFr9Cv47uPyb7WFNh1feTJ+7DEtWk9/qT32RB6tp3XbI1PWNNujTmRtL6cRmgLIfCxqa7P45/wpaK3XYiMpSDf5/x4hnxtzl2UJE0bHpAMPzNCzPzQqPcHxUwVc7stxpPRn6iSmbnXGXJn67HjJMR5CmLxA4Vb8JvJCYmX19+f1iMKEhfpI0jO4/wIAAP//Cp+DxQ==" } diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index 639764d8323..5d706eac8df 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -6,18 +6,20 @@ }, "elasticsearch": { "cluster": { - "id": "A0sjozJKQoSRmOe2LV9kWA" + "id": "njqU4EQaTROIDlWPeUMQyw" } }, "kibana": { "stats": { "concurrent_connections": 0, - "host": "localhost", + "host": { + "name": "localhost" + }, "index": "Shaunaks-MBP-2", "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.36442995071411133 + "ms": 0.2747599985450506 }, "memory": { "heap": { @@ -25,29 +27,22 @@ "bytes": 1501560832 }, "total": { - "bytes": 225144832 + "bytes": 235712512 }, "used": { - "bytes": 181993128 + "bytes": 186280616 } } }, - "pid": 59876, + "pid": 13671, "uptime": { - "ms": 69018 + "ms": 795301 } }, - "request": { - "disconnects": 0, - "total": 1 - }, + "request": {}, "response_time": { - "avg": { - "ms": 25 - }, - "max": { - "ms": 25 - } + "avg": {}, + "max": {} }, "snapshot": false, "status": "green", diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index e9817c7eb88..6773f4e53d5 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -16,7 +16,7 @@ type: keyword description: > Name of Kibana's internal index - - name: host + - name: host.name type: keyword description: > Kibana instance hostname diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 1eb46993d09..9b17c8cba42 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -28,10 +28,12 @@ import ( var ( schema = s.Schema{ - "uuid": c.Str("kibana.uuid"), - "name": c.Str("kibana.name"), - "index": c.Str("kibana.name"), - "host": c.Str("kibana.host"), + "uuid": c.Str("kibana.uuid"), + "name": c.Str("kibana.name"), + "index": c.Str("kibana.name"), + "host": s.Object{ + "name": c.Str("kibana.host"), + }, "transport_address": c.Str("kibana.transport_address"), "version": c.Str("kibana.version"), "snapshot": c.Bool("kibana.snapshot"), From ec954b9bb797ee9697d5fb8226a10d9f23475fae Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 08:05:09 -0700 Subject: [PATCH 60/66] Cast to int64 instead of int --- metricbeat/module/kibana/stats/data_xpack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 4130a418b0c..14c249e6066 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -173,8 +173,7 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error if !ok { return reportErrorForMissingField("process.memory.resident_set_size_bytes", r) } - - kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int(rss)) + kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int64(rss)) timestamp := time.Now() kibanaStatsFields.Put("timestamp", timestamp) From d0a6df7cf03c9ac4e4bafb2cde85a232dd23ed3b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 08:11:16 -0700 Subject: [PATCH 61/66] Renaming field cluster_uuid -> elasticsearch.cluster.id --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/kibana/fields.go | 2 +- .../module/kibana/stats/_meta/data.json | 35 +++++++++++-------- .../module/kibana/stats/_meta/fields.yml | 2 +- metricbeat/module/kibana/stats/data.go | 14 +++++--- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index de525493846..fe4f1816930 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6761,7 +6761,7 @@ Kibana instance's health status -- -*`kibana.stats.cluster_uuid`*:: +*`kibana.stats.elasticsearch.cluster.id`*:: + -- type: keyword diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index d53f8bab0f4..940649a14fe 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpLn/5tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJNE0k4hxW17Cn0/M9umvBcDGkNW8rK0+g8OCeizxkkNJS9gGX5XtnQTPuZ2+LRYUPt5NmRs12Vytq7UdQKchVO5ZTEFQkASjOOu9PvS4u4Z0fcKqMvrsQQf5Rod3H4bPrqD2cI1jQacIXl9//jEpG/9Nygrt5es0L0x3msZp2s/n6wsWBH7T6n9gME4oOLQJoQ4h9yzZqO+zhDxKjIZAAjouY7eg1oGYZ+dgCjsKH/jIUVdtlEwC7Siw8W52jJTdY1M6LDn3wzJrRNfeW8Ih0A3RP3KSnAJITh3AujJWg2HAo15zL40kKNX8yeiKIqaD0EqeEuoYlK1YKKzmnQhxAsQuiaH5ZJHFKCYMKu/0QDy850blHbTyzpGSs8nWo/ROVSGQqwd+fNF4l46c9W57Z0tXxZpCxFXWkBPoSUCBmiJs9KSp8gxevBBIjgLr4N+ZAoNCB0xOQ1FZMaUlYBP/i458xWcWxUOgfypiiS8bqV+uLQMyU5z4At4p+lg3keR0qM0Heq6YgFhwbQ3npPtm03Erg1dj/T7cRxMi9WtjrttCg+ephdNDuaivKxm7SXJiORXaoPxHOGgXq8h6X640WTxkxdCPjosVWtKrjfU4nBoT+D5FHYg6UOuAcVAYaw2T8k4PRTu8ggofDllOOMzNtbzdEa9oeSSBYynsbRIvaLP1Qejyt1fTCbDxoUBZwtiPbzoA8HtUbxxAa71CIR3bqa3yGOG09eNnD5P+ZvQ/Re5IAOvDdGQ2/9LKmsLINwP/jPsMvNXAJSqqic7j/+I1ZX9z59THCYko49dsqvVuuDMB97W2HYdCL8ijndcRtSN5rkkJ8KWd8cpXTu6amCcobbjbiw8I1Gnvtc5zs9TeKVBPmi7n9tk4eABZ0+jugm8kc1x6x7SKaZ8rf19aoxCN3rfwTmS42z6mxL/fUcAtHV2vKW8smN6Kwf1jsD7j3hRVMRFr9Cv47uPyb7WFNh1feTJ+7DEtWk9/qT32RB6tp3XbI1PWNNujTmRtL6cRmgLIfCxqa7P45/wpaK3XYiMpSDf5/x4hnxtzl2UJE0bHpAMPzNCzPzQqPcHxUwVc7stxpPRn6iSmbnXGXJn67HjJMR5CmLxA4Vb8JvJCYmX19+f1iMKEhfpI0jO4/wIAAP//Cp+DxQ==" + return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpLn/5tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJNE0k4hxW17Cn0/M9umvBcDGkNW8rK0+g8OCeizxkkNJS9gGX5XtnQTPuZ2+LRYUPt5NmRs12Vytq7UdQKchVO5ZTEFQkASjOOu9PvS4u4Z0fcKqMvrsQQf5Rod3H4bPrqD2cI1jQacIXl9//jEpG/9Nygrt5es0L0x3msZp2s/n6wsWBH7T6n9gME4oOLQJoQ4h9yzZqO+zhDxKjIZAAjouY7eg1oGYZ+dgCjsKH/jIUVdtlEwC7Siw8W52jJTdY1M6LDn3wzJrRNfeW8Ih0A3RP3KSnAJITh3AujJWg2HAo15zL40kKNX8yeiKIqaD0EqeEuoYyCKLUUwYVJ4pW7FQyOacDnEaxI6JYfrUV4NWDcTDe25U3jmgvHOk5GzKnYiVd6oKgVw9/OOLxrt0FK132zvbuyrWFCKusoacQE8CCtQUYaMnTcVn8OKFQHIUWAf/zhQYFDpgchqKyoopLQGb+F905Cs+sygeAv1TEUt82Uj9cm0ZkJni9BfwTtHHuqEkp0NtPtBzxQTEgmtrOCfdN5uOWxm8Guv94W6aEKlfG3PdRho8Ty2fHspFfV3J2E2SE8up0AatMMJBu1hF1vtypcniISuGfnRcrNCSXm2sx+EEmcD3KepA1IFaB4yDwlhrmJR3eija4RVU+HDIcsJhbq7l7Y54RcsjCRxLYW+reEGbrQ9Cl7+9mk6AjQ8FyhLGfnzTAYDfo3rjAFrrFQrp2E5tlccIp60fP4GY9Dej/ylyRwJYH6Yjs/mXVtYURr4Z+GfcZ+CtBi5RUU10Hv8Xryn7mzunPk5IRBm/bFOtd8OdCbivte04FHpBHu28jqgdyXNNSoAv7YxXvnJy18Q8QWnD3V58QKBOe691npul9k6BetJ0ObfPxsEDyJpGdxd8I5nj0jumVUz7XPn70hqFaPS+hXciw932MSX+/Y4Cbunoek15Y8H0VgzuH4P1GfemqIqJWKNfxHcfnX+rLbTp+MpT8mOPbNF6+kvtsafzaD2t2x6fsqbZHnU6a3s5jdAUQOZjUVubxT/nT0FrvRYbSUG6yf/3CPncmLssS5gwOiYdeGCGnv2hUekJjp8q4HJfjiOlP1MnMXWrM+bK1GfHS47xEMLkBQq34jeRFxIrq78/r0cUJizUR5Kewf0XAAD//7fGiDM=" } diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index 5d706eac8df..ee2fa8e0f68 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -4,14 +4,14 @@ "hostname": "host.example.com", "name": "host.example.com" }, - "elasticsearch": { - "cluster": { - "id": "njqU4EQaTROIDlWPeUMQyw" - } - }, "kibana": { "stats": { - "concurrent_connections": 0, + "concurrent_connections": 3, + "elasticsearch": { + "cluster": { + "id": "njqU4EQaTROIDlWPeUMQyw" + } + }, "host": { "name": "localhost" }, @@ -19,7 +19,7 @@ "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.2747599985450506 + "ms": 0.55409200116992 }, "memory": { "heap": { @@ -27,22 +27,29 @@ "bytes": 1501560832 }, "total": { - "bytes": 235712512 + "bytes": 213430272 }, "used": { - "bytes": 186280616 + "bytes": 184037288 } } }, - "pid": 13671, + "pid": 16920, "uptime": { - "ms": 795301 + "ms": 2414293 } }, - "request": {}, + "request": { + "disconnects": 0, + "total": 3 + }, "response_time": { - "avg": {}, - "max": {} + "avg": { + "ms": 9 + }, + "max": { + "ms": 12 + } }, "snapshot": false, "status": "green", diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 6773f4e53d5..38f15b5bfd2 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -36,7 +36,7 @@ type: keyword description: > Kibana instance's health status - - name: cluster_uuid + - name: elasticsearch.cluster.id type: keyword description: > UUID of the Elasticsearch cluster to which Kibana connects. diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 9b17c8cba42..aef0264a507 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -34,11 +34,15 @@ var ( "host": s.Object{ "name": c.Str("kibana.host"), }, - "transport_address": c.Str("kibana.transport_address"), - "version": c.Str("kibana.version"), - "snapshot": c.Bool("kibana.snapshot"), - "status": c.Str("kibana.status"), - "cluster_uuid": c.Str("cluster_uuid"), + "transport_address": c.Str("kibana.transport_address"), + "version": c.Str("kibana.version"), + "snapshot": c.Bool("kibana.snapshot"), + "status": c.Str("kibana.status"), + "elasticsearch": s.Object{ + "cluster": s.Object{ + "id": c.Str("cluster_uuid"), + }, + }, "concurrent_connections": c.Int("concurrent_connections"), "process": c.Dict("process", s.Schema{ "pid": c.Int("pid"), From 3bf76e553789217f946dcb1d704ee2102f7c3b91 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 08:34:46 -0700 Subject: [PATCH 62/66] Moving field elasticsearch.cluster.id to root level of event --- metricbeat/docs/fields.asciidoc | 10 ------ metricbeat/module/kibana/fields.go | 2 +- metricbeat/module/kibana/kibana.go | 32 +++++++++++++++++++ .../module/kibana/stats/_meta/data.json | 28 ++++++++-------- .../module/kibana/stats/_meta/fields.yml | 4 --- metricbeat/module/kibana/stats/data.go | 21 ++++++------ metricbeat/module/kibana/stats/data_xpack.go | 14 +++----- 7 files changed, 59 insertions(+), 52 deletions(-) create mode 100644 metricbeat/module/kibana/kibana.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index fe4f1816930..2ade267a399 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6759,16 +6759,6 @@ type: keyword Kibana instance's health status --- - -*`kibana.stats.elasticsearch.cluster.id`*:: -+ --- -type: keyword - -UUID of the Elasticsearch cluster to which Kibana connects. - - -- *`kibana.stats.concurrent_connections`*:: diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 940649a14fe..a52baa11727 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEl82O2zYQx+9+isFecsnqAXwoULQBWhRZFGkXPRSFMSbHFrsUqXJGXrtPX1AftixTttzIiA5BVpLn/5tv8Rne6LCEN7NGhwsAMWJpCU+/1DeeFgCaWAVTivFuCd8tAACah1B4XVlaAHDug6yUdxuzXcIGLce7gSwh0xLWJNE0k4hxW17Cn0/M9umvBcDGkNW8rK0+g8OCeizxkkNJS9gGX5XtnQTPuZ2+LRYUPt5NmRs12Vytq7UdQKchVO5ZTEFQkASjOOu9PvS4u4Z0fcKqMvrsQQf5Rod3H4bPrqD2cI1jQacIXl9//jEpG/9Nygrt5es0L0x3msZp2s/n6wsWBH7T6n9gME4oOLQJoQ4h9yzZqO+zhDxKjIZAAjouY7eg1oGYZ+dgCjsKH/jIUVdtlEwC7Siw8W52jJTdY1M6LDn3wzJrRNfeW8Ih0A3RP3KSnAJITh3AujJWg2HAo15zL40kKNX8yeiKIqaD0EqeEuoYyCKLUUwYVJ4pW7FQyOacDnEaxI6JYfrUV4NWDcTDe25U3jmgvHOk5GzKnYiVd6oKgVw9/OOLxrt0FK132zvbuyrWFCKusoacQE8CCtQUYaMnTcVn8OKFQHIUWAf/zhQYFDpgchqKyoopLQGb+F905Cs+sygeAv1TEUt82Uj9cm0ZkJni9BfwTtHHuqEkp0NtPtBzxQTEgmtrOCfdN5uOWxm8Guv94W6aEKlfG3PdRho8Ty2fHspFfV3J2E2SE8up0AatMMJBu1hF1vtypcniISuGfnRcrNCSXm2sx+EEmcD3KepA1IFaB4yDwlhrmJR3eija4RVU+HDIcsJhbq7l7Y54RcsjCRxLYW+reEGbrQ9Cl7+9mk6AjQ8FyhLGfnzTAYDfo3rjAFrrFQrp2E5tlccIp60fP4GY9Dej/ylyRwJYH6Yjs/mXVtYURr4Z+GfcZ+CtBi5RUU10Hv8Xryn7mzunPk5IRBm/bFOtd8OdCbivte04FHpBHu28jqgdyXNNSoAv7YxXvnJy18Q8QWnD3V58QKBOe691npul9k6BetJ0ObfPxsEDyJpGdxd8I5nj0jumVUz7XPn70hqFaPS+hXciw932MSX+/Y4Cbunoek15Y8H0VgzuH4P1GfemqIqJWKNfxHcfnX+rLbTp+MpT8mOPbNF6+kvtsafzaD2t2x6fsqbZHnU6a3s5jdAUQOZjUVubxT/nT0FrvRYbSUG6yf/3CPncmLssS5gwOiYdeGCGnv2hUekJjp8q4HJfjiOlP1MnMXWrM+bK1GfHS47xEMLkBQq34jeRFxIrq78/r0cUJizUR5Kewf0XAAD//7fGiDM=" + return "eJzEmM1u4zYQx+9+ikEue9noAXQoUHQLdFEkKLYNeigKYySOLTYUqXJGjt2nL6gPR5YpW96VsToEiWT//z/OF8U8wisdUnjVGVpcAYgWQyk8/NrceFgBKOLc60q0syn8sAIAaB9C6VRtaAXAhfOyzp3d6G0KGzQc7noyhEwpZCRBmklE2y2n8NcDs3n4ewWw0WQUp43qI1gsacASLjlUlMLWu7rq7kR4TnWGWiwofLwbk5uUbK9uqY0OoFXga/souiQoSbzOORl8fLzi/hrTDQnrWquTBz3kKx3enB8/u4A6wNWWBW1O8PLy+VPUNvyM2grt5ds8z6R7T20V7Zdb6zOWBG7T+X9g0FbIWzQRox6hcCzJ5NoXCXmwmAyBeLRchW5BpTwxL87B5HfkP/CRo6naYBkF2pFn7eziGDHdY1NarLhw4zJrTTPnDOEY6IrpnwVJQR6koB4gq7VRoBnw6NfeiyMJSr18MvqiCOkgNFLEjHqG3Nm89p5sM0ot5UE8zmSc3d7YLHWZkQ/tkhtNVmBgASUqAnFN+Nr6SeDZCYEUKJB598bkGXK0wGQVlLURXRkC1uFXtORqPlEUB57+rYklfFhL8+FGGZCZwiwVcDanj015SkGHRt7TY80ExIKZ0VyQGsom0bhV3uVTnTSe9DMi9Vsr18/30fPYKB+gnM3yCxm7SvLO8vlTyNyguvvCmuCgXagi41y1VmTwkJTjdfRcnKMhtd4Yh+N+nMH3c/CB4AOND2gLpTZGM+XOqrFpj1dS6fwhKQjHubmUtxviFZQnEjiVwsGMdoImyQ5C59+9mE6AjfMlSgpTX766AIA/gnu7ADTG5SikQjt1VR4iHFc/vlAwqe9G/0vgDgSQHeYjs/6P1kaXWr4b+BPuE3BGAVeYU0N0Gv9npyj5h/tFfZyRiCq8J8Za78pyZuC+NNphKAyCPNl5PVE3kpealABfuhmfu9rKTRPzHUpp7kb8PQL1vu91i+d2U3sjTwNrOp/bJ+PgDmRto9szvonMceUs0zqkfan8felEIYjetuG9k+Fue58S/3FHHrd0XHpDeWWDGWwxuL8P1hPudVmXM7Em3y9vPoj+3ih06fjGM+d9D0BBPf6mdt+zblCP+3aHkaRttnuddbpejiO0BZC4UNTGJOHP5VPQqTdmEymIN/lXj5CnVu68LGHG6Jh14IEFevan1mVgOH2qgPP9chop/po6i6nfOkOuNMvoP0n9NRVCmL2BwrX4zeSFyJY13D8vRxRmbKj3JD2B+z8AAP//Tw5Vjw==" } diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go new file mode 100644 index 00000000000..43f1f711a06 --- /dev/null +++ b/metricbeat/module/kibana/kibana.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kibana + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" +) + +// ReportErrorForMissingField reports and returns an error message for the given +// field being missing in API response received from Kibana +func ReportErrorForMissingField(field string, r mb.ReporterV2) error { + err := fmt.Errorf("Could not find field '%v' in Kibana stats API response", field) + r.Error(err) + return err +} diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index ee2fa8e0f68..7a8e3c54c8e 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -4,14 +4,14 @@ "hostname": "host.example.com", "name": "host.example.com" }, + "elasticsearch": { + "cluster": { + "id": "njqU4EQaTROIDlWPeUMQyw" + } + }, "kibana": { "stats": { "concurrent_connections": 3, - "elasticsearch": { - "cluster": { - "id": "njqU4EQaTROIDlWPeUMQyw" - } - }, "host": { "name": "localhost" }, @@ -19,7 +19,7 @@ "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.55409200116992 + "ms": 0.2830430008471012 }, "memory": { "heap": { @@ -27,28 +27,26 @@ "bytes": 1501560832 }, "total": { - "bytes": 213430272 + "bytes": 207663104 }, "used": { - "bytes": 184037288 + "bytes": 187193320 } } }, - "pid": 16920, + "pid": 20173, "uptime": { - "ms": 2414293 + "ms": 928511 } }, "request": { "disconnects": 0, - "total": 3 + "total": 0 }, "response_time": { - "avg": { - "ms": 9 - }, + "avg": {}, "max": { - "ms": 12 + "ms": 0 } }, "snapshot": false, diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 38f15b5bfd2..7715734528b 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -36,10 +36,6 @@ type: keyword description: > Kibana instance's health status - - name: elasticsearch.cluster.id - type: keyword - description: > - UUID of the Elasticsearch cluster to which Kibana connects. - name: concurrent_connections type: long description: > diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index aef0264a507..f5882a38566 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -24,6 +24,7 @@ import ( s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/kibana" ) var ( @@ -34,15 +35,10 @@ var ( "host": s.Object{ "name": c.Str("kibana.host"), }, - "transport_address": c.Str("kibana.transport_address"), - "version": c.Str("kibana.version"), - "snapshot": c.Bool("kibana.snapshot"), - "status": c.Str("kibana.status"), - "elasticsearch": s.Object{ - "cluster": s.Object{ - "id": c.Str("cluster_uuid"), - }, - }, + "transport_address": c.Str("kibana.transport_address"), + "version": c.Str("kibana.version"), + "snapshot": c.Bool("kibana.snapshot"), + "status": c.Str("kibana.status"), "concurrent_connections": c.Int("concurrent_connections"), "process": c.Dict("process", s.Schema{ "pid": c.Int("pid"), @@ -102,10 +98,11 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.RootFields.Put("service.name", "kibana") // Set elasticsearch cluster id - if clusterID, ok := dataFields["cluster_uuid"]; ok { - delete(dataFields, "cluster_uuid") - event.RootFields.Put("elasticsearch.cluster.id", clusterID) + elasticsearchClusterID, ok := data["cluster_uuid"] + if !ok { + return kibana.ReportErrorForMissingField("cluster_uuid", r) } + event.RootFields.Put("elasticsearch.cluster.id", elasticsearchClusterID) event.MetricSetFields = dataFields diff --git a/metricbeat/module/kibana/stats/data_xpack.go b/metricbeat/module/kibana/stats/data_xpack.go index 14c249e6066..d874091dad6 100644 --- a/metricbeat/module/kibana/stats/data_xpack.go +++ b/metricbeat/module/kibana/stats/data_xpack.go @@ -19,7 +19,6 @@ package stats import ( "encoding/json" - "fmt" "time" "github.com/elastic/beats/libbeat/common" @@ -27,6 +26,7 @@ import ( c "github.com/elastic/beats/libbeat/common/schema/mapstriface" "github.com/elastic/beats/metricbeat/helper/xpack" "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/kibana" ) var ( @@ -140,12 +140,6 @@ var ( } ) -func reportErrorForMissingField(field string, r mb.ReporterV2) error { - err := fmt.Errorf("Could not find field '%v' in Kibana stats API response", field) - r.Error(err) - return err -} - func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) @@ -162,16 +156,16 @@ func eventMappingXPack(r mb.ReporterV2, intervalMs int64, content []byte) error process, ok := data["process"].(map[string]interface{}) if !ok { - return reportErrorForMissingField("process", r) + return kibana.ReportErrorForMissingField("process", r) } memory, ok := process["memory"].(map[string]interface{}) if !ok { - return reportErrorForMissingField("process.memory", r) + return kibana.ReportErrorForMissingField("process.memory", r) } rss, ok := memory["resident_set_size_bytes"].(float64) if !ok { - return reportErrorForMissingField("process.memory.resident_set_size_bytes", r) + return kibana.ReportErrorForMissingField("process.memory.resident_set_size_bytes", r) } kibanaStatsFields.Put("process.memory.resident_set_size_in_bytes", int64(rss)) From 639870503cc5f0b33f248ea7ed6409e2c6d315f6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 08:42:55 -0700 Subject: [PATCH 63/66] Moving field process.pid to root level of event --- metricbeat/docs/fields.asciidoc | 10 ---------- metricbeat/module/kibana/fields.go | 2 +- metricbeat/module/kibana/stats/_meta/data.json | 14 ++++++++------ metricbeat/module/kibana/stats/_meta/fields.yml | 4 ---- metricbeat/module/kibana/stats/data.go | 12 +++++++++++- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2ade267a399..0b812bf5931 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -6778,16 +6778,6 @@ Process metrics -*`kibana.stats.process.pid`*:: -+ --- -type: long - -Process ID of the Kibana instance - - --- - *`kibana.stats.process.event_loop_delay.ms`*:: + -- diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index a52baa11727..68bf480e11b 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzEmM1u4zYQx+9+ikEue9noAXQoUHQLdFEkKLYNeigKYySOLTYUqXJGjt2nL6gPR5YpW96VsToEiWT//z/OF8U8wisdUnjVGVpcAYgWQyk8/NrceFgBKOLc60q0syn8sAIAaB9C6VRtaAXAhfOyzp3d6G0KGzQc7noyhEwpZCRBmklE2y2n8NcDs3n4ewWw0WQUp43qI1gsacASLjlUlMLWu7rq7kR4TnWGWiwofLwbk5uUbK9uqY0OoFXga/souiQoSbzOORl8fLzi/hrTDQnrWquTBz3kKx3enB8/u4A6wNWWBW1O8PLy+VPUNvyM2grt5ds8z6R7T20V7Zdb6zOWBG7T+X9g0FbIWzQRox6hcCzJ5NoXCXmwmAyBeLRchW5BpTwxL87B5HfkP/CRo6naYBkF2pFn7eziGDHdY1NarLhw4zJrTTPnDOEY6IrpnwVJQR6koB4gq7VRoBnw6NfeiyMJSr18MvqiCOkgNFLEjHqG3Nm89p5sM0ot5UE8zmSc3d7YLHWZkQ/tkhtNVmBgASUqAnFN+Nr6SeDZCYEUKJB598bkGXK0wGQVlLURXRkC1uFXtORqPlEUB57+rYklfFhL8+FGGZCZwiwVcDanj015SkGHRt7TY80ExIKZ0VyQGsom0bhV3uVTnTSe9DMi9Vsr18/30fPYKB+gnM3yCxm7SvLO8vlTyNyguvvCmuCgXagi41y1VmTwkJTjdfRcnKMhtd4Yh+N+nMH3c/CB4AOND2gLpTZGM+XOqrFpj1dS6fwhKQjHubmUtxviFZQnEjiVwsGMdoImyQ5C59+9mE6AjfMlSgpTX766AIA/gnu7ADTG5SikQjt1VR4iHFc/vlAwqe9G/0vgDgSQHeYjs/6P1kaXWr4b+BPuE3BGAVeYU0N0Gv9npyj5h/tFfZyRiCq8J8Za78pyZuC+NNphKAyCPNl5PVE3kpealABfuhmfu9rKTRPzHUpp7kb8PQL1vu91i+d2U3sjTwNrOp/bJ+PgDmRto9szvonMceUs0zqkfan8felEIYjetuG9k+Fue58S/3FHHrd0XHpDeWWDGWwxuL8P1hPudVmXM7Em3y9vPoj+3ih06fjGM+d9D0BBPf6mdt+zblCP+3aHkaRttnuddbpejiO0BZC4UNTGJOHP5VPQqTdmEymIN/lXj5CnVu68LGHG6Jh14IEFevan1mVgOH2qgPP9chop/po6i6nfOkOuNMvoP0n9NRVCmL2BwrX4zeSFyJY13D8vRxRmbKj3JD2B+z8AAP//Tw5Vjw==" + return "eJzEmM9u4zYQxu9+ikEue9noAXwoULQFWhQJim2DHorCGIljiw1FqpyRY/fpC1KSI8tUrOzKiA5BItnf9+Nw/oi5h2c6ruFZ52hxBSBaDK3h7td4424FoIgLr2vRzq7huxUAQPsQKqcaQysALp2XTeHsVu/WsEXD4a4nQ8i0hpwkSDOJaLvjNfx1x2zu/l4BbDUZxeuoeg8WKxqwhEuONa1h511Td3cSPOc6Qy0WFD7dTclNSrZXt9SoA2gV+Mbei64IKhKvC84GHx+vuL/GdEPCptHq7EEP+UzHF+fHz95AHeBqy4K2IHh6+uXHpG34mbQVOsi3eV5I957aKjost9ZHrAjctvP/xKCtkLdoEkY9QulYssm1LxLyYDEZAvFouQ7Vgkp5Yl6cg8nvyX/iE0fM2mCZBNqTZ+3s4hgp3VNRWqy5dOM0a01z5wzhGOiK6Z8lSUkepKQeIG+0UaAZ8OTX3ksjCUqz/Gb0SRG2g9BImTLqGQpni8Z7srGVWiqCeJrJOLt7Z7E0VU4+lEthNFmBgQVUqAjExfC1+ZPBoxMCKVEg9+6FyTMUaIHJKqgaI7o2BKzDr2jJNXymKA48/dsQS/iwlvjhqAzITKGXCjhb0OeYnlLSMcp7um+YgFgwN5pLUkPZLBm32rtiqpLGnX5GpH5r5fr+PnqeauWvKLQPu2ecqzeKDB6zavz9HowLNKQ2W+NwXAdXCQF+Cj4QfCD6gLZQaWM0U+GsGpv2eBVVzh+zknAck7fiNYOnj1lQngjcVOgGvdEJmiw/Cl1+943Ej9LOVyhrmPry1QUA/BHc2wWgMa5AIRXSuMuuEOG0+mmQM6kPo/85cAcCyI/zkVn/RxujKy0fBv6AhwycUcA1FhSJzuP/6BRl/3C/qM8zNqIO72ep0ruynBm4T1E7tNFBkCcrryfqWuFSHQrgS9dbC9dYeVeneoVSmrvWeotAvc6bbvHcDpMX8jSwpvFcHbWDG5C1hW4v+CZ2jmtnmTZh25favy+dKATR9w2aVzLc726T4t/vyeOOTkuPlFcGzGDE4OE2WA940FVTzcSafK979wHw96jQbcc3nvVue/AI6uk3pNueMYN62rc7BGRtsd3qjNHVchqhTYDMhaQ2Jgt/Lr8FnXo0m9iCdJF/dQt5aOUu0xJmtI5ZBw1YoGZ/aF0GhtNv83A5L6eR0q+ps5j60Rn2SrOM/oPTX1MhhNkDFK7FbyYvJEbWcH6+HVGYMVBvSXoG938AAAD//7lENR0=" } diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index 7a8e3c54c8e..7097d36ba42 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -11,7 +11,7 @@ }, "kibana": { "stats": { - "concurrent_connections": 3, + "concurrent_connections": 6, "host": { "name": "localhost" }, @@ -19,7 +19,7 @@ "name": "Shaunaks-MBP-2", "process": { "event_loop_delay": { - "ms": 0.2830430008471012 + "ms": 0.23628300055861473 }, "memory": { "heap": { @@ -27,16 +27,15 @@ "bytes": 1501560832 }, "total": { - "bytes": 207663104 + "bytes": 215003136 }, "used": { - "bytes": 187193320 + "bytes": 185343400 } } }, - "pid": 20173, "uptime": { - "ms": 928511 + "ms": 1343714 } }, "request": { @@ -62,6 +61,9 @@ "name": "stats", "rtt": 115 }, + "process": { + "pid": 20173 + }, "service": { "name": "kibana" } diff --git a/metricbeat/module/kibana/stats/_meta/fields.yml b/metricbeat/module/kibana/stats/_meta/fields.yml index 7715734528b..58da702cc9f 100644 --- a/metricbeat/module/kibana/stats/_meta/fields.yml +++ b/metricbeat/module/kibana/stats/_meta/fields.yml @@ -45,10 +45,6 @@ description: > Process metrics fields: - - name: pid - type: long - description: > - Process ID of the Kibana instance - name: event_loop_delay.ms type: scaled_float description: > diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index f5882a38566..1802cfcb631 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -41,7 +41,6 @@ var ( "status": c.Str("kibana.status"), "concurrent_connections": c.Int("concurrent_connections"), "process": c.Dict("process", s.Schema{ - "pid": c.Int("pid"), "event_loop_delay": s.Object{ "ms": c.Float("event_loop_delay"), }, @@ -104,6 +103,17 @@ func eventMapping(r mb.ReporterV2, content []byte) error { } event.RootFields.Put("elasticsearch.cluster.id", elasticsearchClusterID) + // Set process PID + process, ok := data["process"].(map[string]interface{}) + if !ok { + return kibana.ReportErrorForMissingField("process", r) + } + pid, ok := process["pid"].(float64) + if !ok { + return kibana.ReportErrorForMissingField("process.pid", r) + } + event.RootFields.Put("process.pid", int(pid)) + event.MetricSetFields = dataFields r.Event(event) From 8c54595ecdcf7cbcfba823e86246cbd9b071ed6c Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 09:17:00 -0700 Subject: [PATCH 64/66] Adding unit test for ReportErrorForMissingField helper function --- metricbeat/module/kibana/kibana_test.go | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 metricbeat/module/kibana/kibana_test.go diff --git a/metricbeat/module/kibana/kibana_test.go b/metricbeat/module/kibana/kibana_test.go new file mode 100644 index 00000000000..e12debfdb99 --- /dev/null +++ b/metricbeat/module/kibana/kibana_test.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kibana + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/metricbeat/mb" +) + +type MockReporterV2 struct { + mb.ReporterV2 + err error +} + +func (MockReporterV2) Event(event mb.Event) bool { + return true +} + +var currentErr error + +func (m MockReporterV2) Error(err error) bool { + currentErr = err + return true +} + +func TestReportErrorForMissingField(t *testing.T) { + field := "some.missing.field" + r := MockReporterV2{} + err := ReportErrorForMissingField(field, r) + + expectedError := fmt.Errorf("Could not find field '%v' in Kibana stats API response", field) + assert.Equal(t, expectedError, err) + assert.Equal(t, expectedError, currentErr) +} From a9b601f9f0d8fa76c13c7edf543a696aedf31660 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 11:38:07 -0700 Subject: [PATCH 65/66] Removing unused field --- metricbeat/module/kibana/kibana_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/module/kibana/kibana_test.go b/metricbeat/module/kibana/kibana_test.go index e12debfdb99..7e3f84fabf9 100644 --- a/metricbeat/module/kibana/kibana_test.go +++ b/metricbeat/module/kibana/kibana_test.go @@ -28,7 +28,6 @@ import ( type MockReporterV2 struct { mb.ReporterV2 - err error } func (MockReporterV2) Event(event mb.Event) bool { From f011ff70ca59d94cdabba7abdfe9c0bd5390dd45 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 19 Jul 2018 11:40:50 -0700 Subject: [PATCH 66/66] Adding comment about hack --- metricbeat/module/kibana/kibana_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/kibana/kibana_test.go b/metricbeat/module/kibana/kibana_test.go index 7e3f84fabf9..543aabda282 100644 --- a/metricbeat/module/kibana/kibana_test.go +++ b/metricbeat/module/kibana/kibana_test.go @@ -34,7 +34,7 @@ func (MockReporterV2) Event(event mb.Event) bool { return true } -var currentErr error +var currentErr error // This hack is necessary because the Error method below cannot receive the type *MockReporterV2 func (m MockReporterV2) Error(err error) bool { currentErr = err