From b35a7a0a8ce7425b6030e9648b8312117b653d58 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 25 Apr 2018 11:03:55 +0200 Subject: [PATCH] Update elasticsearch.node_stats to use ReporterV2 (#6917) To allow more flexibility the Elasticsearch node_stats metricset is updated to use the ReporverV2 interface. This is a subset of https://github.com/elastic/beats/pull/6807. * Test data set for ES 6.2.3 added * Reporting of error if json decoding fails * Update node_stats integration tests to use reporter interface * Update generator to also add metricset info * Update data.json for node_stats and index * Set namespace as part of registration * Add service.name to event --- metricbeat/mb/testing/data_generator.go | 2 +- .../elasticsearch/index/_meta/data.json | 18 +- .../elasticsearch/node_stats/_meta/data.json | 57 +- .../node_stats/_meta/test/node_stats.623.json | 516 ++++++++++++++++++ .../module/elasticsearch/node_stats/data.go | 44 +- .../elasticsearch/node_stats/data_test.go | 20 +- .../elasticsearch/node_stats/node_stats.go | 10 +- .../node_stats/node_stats_integration_test.go | 16 +- 8 files changed, 607 insertions(+), 76 deletions(-) create mode 100644 metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go index 8c38da39a30..a76b03502e6 100644 --- a/metricbeat/mb/testing/data_generator.go +++ b/metricbeat/mb/testing/data_generator.go @@ -73,7 +73,7 @@ func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error { return fmt.Errorf("no events were generated") } - e := StandardizeEvent(f, events[0]) + e := StandardizeEvent(f, events[0], mb.AddMetricSetInfo) WriteEventToDataJSON(t, e) return nil diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json index d9e8b1519a2..01d1bb4a00e 100644 --- a/metricbeat/module/elasticsearch/index/_meta/data.json +++ b/metricbeat/module/elasticsearch/index/_meta/data.json @@ -10,26 +10,32 @@ "name": "docker-cluster" }, "index": { - "name": ".watches", + "name": ".monitoring-es-6-2018.04.22", "total": { "docs": { - "count": 6, - "deleted": 0 + "count": 241933, + "deleted": 525 }, "segments": { - "count": 3, + "count": 16, "memory": { - "bytes": 11505 + "bytes": 565573 } }, "store": { "size": { - "bytes": 67980 + "bytes": 135541290 } } } } }, + "metricset": { + "host": "127.0.0.1:9200", + "module": "elasticsearch", + "name": "index", + "rtt": 115 + }, "service": { "name": "elasticsearch" } diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index 607c7db8aa6..a974350d3f3 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -9,35 +9,35 @@ "name": "docker-cluster" }, "node": { - "name": "ab1_1Gw3Rtax_2l78Gq_CQ", + "name": "VnS118cKS7-8mz8l2jZZKQ", "stats": { "fs": { "summary": { "available": { - "bytes": 50680381440 + "bytes": 36334804992 }, "free": { - "bytes": 54133080064 + "bytes": 39551520768 }, "total": { - "bytes": 67371577344 + "bytes": 62725623808 } } }, "indices": { "docs": { - "count": 52, - "deleted": 4 + "count": 1141228, + "deleted": 42042 }, "segments": { - "count": 3, + "count": 154, "memory": { - "bytes": 20655 + "bytes": 2862018 } }, "store": { "size": { - "bytes": 107559 + "bytes": 642756442 } } }, @@ -46,14 +46,14 @@ "collectors": { "old": { "collection": { - "count": 7, - "ms": 3801 + "count": 105, + "ms": 14100 } }, "young": { "collection": { - "count": 56, - "ms": 14782 + "count": 43895, + "ms": 423308 } } } @@ -62,44 +62,44 @@ "pools": { "old": { "max": { - "bytes": 62914560 + "bytes": 362414080 }, "peak": { - "bytes": 53816432 + "bytes": 275023840 }, "peak_max": { - "bytes": 62914560 + "bytes": 362414080 }, "used": { - "bytes": 40017472 + "bytes": 255660936 } }, "survivor": { "max": { - "bytes": 3145728 + "bytes": 17432576 }, "peak": { - "bytes": 3145728 + "bytes": 17432576 }, "peak_max": { - "bytes": 3145728 + "bytes": 17432576 }, "used": { - "bytes": 3145728 + "bytes": 4414672 } }, "young": { "max": { - "bytes": 25165824 + "bytes": 139591680 }, "peak": { - "bytes": 25165824 + "bytes": 139591680 }, "peak_max": { - "bytes": 25165824 + "bytes": 139591680 }, "used": { - "bytes": 21071064 + "bytes": 91758824 } } } @@ -108,11 +108,14 @@ } } }, + "event": { + "service": "elasticsearch" + }, "metricset": { - "host": "elasticsearch:9200", + "host": "127.0.0.1:9200", "module": "elasticsearch", "name": "node_stats", - "namespace": "node.stats", + "namespace": "elasticsearch.node.stats", "rtt": 115 } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json new file mode 100644 index 00000000000..627b117321f --- /dev/null +++ b/metricbeat/module/elasticsearch/node_stats/_meta/test/node_stats.623.json @@ -0,0 +1,516 @@ +{ + "_nodes": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "cluster_name": "842d2a3cdf39e9ae2e0b2c7ca7cea075", + "nodes": { + "r4XD9O8eTrCHyN_GJswZ5A": { + "timestamp": 1524464610026, + "name": "instance-0000000016", + "transport_address": "172.25.133.112:19608", + "host": "172.25.133.112", + "ip": "172.25.133.112:19608", + "roles": [ + "master", + "data", + "ingest" + ], + "attributes": { + "logical_availability_zone": "zone-0", + "availability_zone": "us-east-1e", + "region": "us-east-1" + }, + "indices": { + "docs": { + "count": 34283257, + "deleted": 25991 + }, + "store": { + "size_in_bytes": 11772256011 + }, + "indexing": { + "index_total": 95855239, + "index_time_in_millis": 14070223, + "index_current": 0, + "index_failed": 107, + "delete_total": 164420, + "delete_time_in_millis": 7426, + "delete_current": 0, + "noop_update_total": 127, + "is_throttled": false, + "throttle_time_in_millis": 31 + }, + "get": { + "total": 585810, + "time_in_millis": 104059, + "exists_total": 585742, + "exists_time_in_millis": 104059, + "missing_total": 68, + "missing_time_in_millis": 0, + "current": 0 + }, + "search": { + "open_contexts": 0, + "query_total": 1926212, + "query_time_in_millis": 7629899, + "query_current": 0, + "fetch_total": 1014673, + "fetch_time_in_millis": 146067, + "fetch_current": 0, + "scroll_total": 160754, + "scroll_time_in_millis": 43277, + "scroll_current": 0, + "suggest_total": 0, + "suggest_time_in_millis": 0, + "suggest_current": 0 + }, + "merges": { + "current": 0, + "current_docs": 0, + "current_size_in_bytes": 0, + "total": 127608, + "total_time_in_millis": 37529833, + "total_docs": 2330861529, + "total_size_in_bytes": 717037012447, + "total_stopped_time_in_millis": 0, + "total_throttled_time_in_millis": 281485, + "total_auto_throttle_in_bytes": 9324199696 + }, + "refresh": { + "total": 1402567, + "total_time_in_millis": 27376569, + "listeners": 0 + }, + "flush": { + "total": 299, + "total_time_in_millis": 7005 + }, + "warmer": { + "current": 0, + "total": 960058, + "total_time_in_millis": 155434 + }, + "query_cache": { + "memory_size_in_bytes": 1698624, + "total_count": 1222084, + "hit_count": 148958, + "miss_count": 1073126, + "cache_size": 131, + "cache_count": 2115, + "evictions": 1984 + }, + "fielddata": { + "memory_size_in_bytes": 8256, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 1661, + "memory_in_bytes": 39764743, + "terms_memory_in_bytes": 22195943, + "stored_fields_memory_in_bytes": 3200936, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 343040, + "points_memory_in_bytes": 4009732, + "doc_values_memory_in_bytes": 10015092, + "index_writer_memory_in_bytes": 3147663, + "version_map_memory_in_bytes": 2536, + "fixed_bit_set_memory_in_bytes": 1126224, + "max_unsafe_auto_id_timestamp": 1524441609889, + "file_sizes": {} + }, + "translog": { + "operations": 1838361, + "size_in_bytes": 1525215881, + "uncommitted_operations": 24969, + "uncommitted_size_in_bytes": 15857511 + }, + "request_cache": { + "memory_size_in_bytes": 15608, + "evictions": 0, + "hit_count": 461937, + "miss_count": 113 + }, + "recovery": { + "current_as_source": 0, + "current_as_target": 0, + "throttle_time_in_millis": 366610 + } + }, + "os": { + "timestamp": 1524464610097, + "cpu": { + "percent": 17, + "load_average": { + "1m": 7.15, + "5m": 4.67, + "15m": 3.5 + } + }, + "mem": { + "total_in_bytes": 257796820992, + "free_in_bytes": 5525663744, + "used_in_bytes": 252271157248, + "free_percent": 2, + "used_percent": 98 + }, + "swap": { + "total_in_bytes": 10737414144, + "free_in_bytes": 10246205440, + "used_in_bytes": 491208704 + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 292079722854156 + }, + "cpu": { + "control_group": "/", + "cfs_period_micros": 100000, + "cfs_quota_micros": 236945, + "stat": { + "number_of_elapsed_periods": 15278067, + "number_of_times_throttled": 155945, + "time_throttled_nanos": 14606886791788 + } + }, + "memory": { + "control_group": "/", + "limit_in_bytes": "2147483648", + "usage_in_bytes": "2145837056" + } + } + }, + "process": { + "timestamp": 1524464610098, + "open_file_descriptors": 1434, + "max_file_descriptors": 1048576, + "cpu": { + "percent": 2, + "total_in_millis": 291781030 + }, + "mem": { + "total_virtual_in_bytes": 14964170752 + } + }, + "jvm": { + "timestamp": 1524464610099, + "uptime_in_millis": 2026428424, + "mem": { + "heap_used_in_bytes": 581971648, + "heap_used_percent": 55, + "heap_committed_in_bytes": 1056309248, + "heap_max_in_bytes": 1056309248, + "non_heap_used_in_bytes": 247342320, + "non_heap_committed_in_bytes": 281911296, + "pools": { + "young": { + "used_in_bytes": 17064904, + "max_in_bytes": 139591680, + "peak_used_in_bytes": 139591680, + "peak_max_in_bytes": 139591680 + }, + "survivor": { + "used_in_bytes": 7227224, + "max_in_bytes": 17432576, + "peak_used_in_bytes": 17432576, + "peak_max_in_bytes": 17432576 + }, + "old": { + "used_in_bytes": 557679520, + "max_in_bytes": 899284992, + "peak_used_in_bytes": 707779408, + "peak_max_in_bytes": 899284992 + } + } + }, + "threads": { + "count": 95, + "peak_count": 130 + }, + "gc": { + "collectors": { + "young": { + "collection_count": 678990, + "collection_time_in_millis": 9599952 + }, + "old": { + "collection_count": 3424, + "collection_time_in_millis": 376853 + } + } + }, + "buffer_pools": { + "direct": { + "count": 71, + "used_in_bytes": 1364951, + "total_capacity_in_bytes": 1364949 + }, + "mapped": { + "count": 3729, + "used_in_bytes": 11713431232, + "total_capacity_in_bytes": 11713431232 + } + }, + "classes": { + "current_loaded_count": 19772, + "total_loaded_count": 28207, + "total_unloaded_count": 8435 + } + }, + "thread_pool": { + "bulk": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 2968468 + }, + "fetch_shard_started": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 4, + "completed": 17664 + }, + "fetch_shard_store": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 4, + "completed": 2531 + }, + "flush": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 652262 + }, + "force_merge": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "generic": { + "threads": 31, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 56, + "completed": 10701102 + }, + "get": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 422867 + }, + "index": { + "threads": 2, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 2, + "completed": 162943 + }, + "listener": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 3 + }, + "management": { + "threads": 5, + "queue": 0, + "active": 1, + "rejected": 0, + "largest": 5, + "completed": 10422508 + }, + "refresh": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 145364755 + }, + "search": { + "threads": 4, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 4, + "completed": 3248375 + }, + "security-token-key": { + "threads": 0, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 0, + "completed": 0 + }, + "snapshot": { + "threads": 1, + "queue": 66, + "active": 1, + "rejected": 0, + "largest": 1, + "completed": 156934 + }, + "warmer": { + "threads": 1, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 1, + "completed": 954865 + }, + "watcher": { + "threads": 10, + "queue": 0, + "active": 0, + "rejected": 0, + "largest": 10, + "completed": 164130 + } + }, + "fs": { + "timestamp": 1524464610101, + "total": { + "total_in_bytes": 55834574848, + "free_in_bytes": 42471211008, + "available_in_bytes": 42471211008 + }, + "data": [ + { + "path": "/app/data/nodes/0", + "mount": "QuotaAwareFileStore(/app (/dev/mapper/data))", + "type": "xfs", + "total_in_bytes": 55834574848, + "free_in_bytes": 42471211008, + "available_in_bytes": 42471211008 + } + ], + "io_stats": {} + }, + "transport": { + "server_open": 0, + "rx_count": 852163, + "rx_size_in_bytes": 37599508177, + "tx_count": 896154, + "tx_size_in_bytes": 39437993670 + }, + "http": { + "current_open": 85, + "total_opened": 53188 + }, + "breakers": { + "request": { + "limit_size_in_bytes": 633785548, + "limit_size": "604.4mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "fielddata": { + "limit_size_in_bytes": 633785548, + "limit_size": "604.4mb", + "estimated_size_in_bytes": 8256, + "estimated_size": "8kb", + "overhead": 1.03, + "tripped": 0 + }, + "in_flight_requests": { + "limit_size_in_bytes": 1056309248, + "limit_size": "1007.3mb", + "estimated_size_in_bytes": 0, + "estimated_size": "0b", + "overhead": 1, + "tripped": 0 + }, + "accounting": { + "limit_size_in_bytes": 1056309248, + "limit_size": "1007.3mb", + "estimated_size_in_bytes": 39764743, + "estimated_size": "37.9mb", + "overhead": 1, + "tripped": 0 + }, + "parent": { + "limit_size_in_bytes": 739416473, + "limit_size": "705.1mb", + "estimated_size_in_bytes": 39772999, + "estimated_size": "37.9mb", + "overhead": 1, + "tripped": 0 + } + }, + "script": { + "compilations": 31, + "cache_evictions": 0 + }, + "discovery": { + "cluster_state_queue": { + "total": 1, + "pending": 0, + "committed": 1 + }, + "published_cluster_states": { + "full_states": 7, + "incompatible_diffs": 6, + "compatible_diffs": 247 + } + }, + "ingest": { + "total": { + "count": 0, + "time_in_millis": 0, + "current": 0, + "failed": 0 + }, + "pipelines": { + "xpack_monitoring_2": { + "count": 0, + "time_in_millis": 0, + "current": 0, + "failed": 0 + }, + "xpack_monitoring_6": { + "count": 0, + "time_in_millis": 0, + "current": 0, + "failed": 0 + } + } + }, + "adaptive_selection": { + "r4XD9O8eTrCHyN_GJswZ5A": { + "outgoing_searches": 0, + "avg_queue_size": 0, + "avg_service_time_ns": 79338, + "avg_response_time_ns": 235770, + "rank": "0.2" + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index 5c316b39824..b977320806c 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -81,32 +81,40 @@ var ( } ) -func eventsMapping(content []byte) ([]common.MapStr, error) { - nodesStruct := struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]map[string]interface{} `json:"nodes"` - }{} +type nodesStruct struct { + ClusterName string `json:"cluster_name"` + Nodes map[string]map[string]interface{} `json:"nodes"` +} + +func eventsMapping(r mb.ReporterV2, content []byte) []error { + + nodeData := &nodesStruct{} + err := json.Unmarshal(content, nodeData) + if err != nil { + r.Error(err) + return nil + } - json.Unmarshal(content, &nodesStruct) + var errs []error + for name, node := range nodeData.Nodes { + event := mb.Event{} - var events []common.MapStr - errors := s.NewErrors() + event.MetricSetFields, err = schema.Apply(node) + if err != nil { + errs = append(errs, err) + } - for name, node := range nodesStruct.Nodes { - event, errs := schema.Apply(node) - // Write name here as full name only available as key - event[mb.ModuleDataKey] = common.MapStr{ + event.ModuleFields = common.MapStr{ "node": common.MapStr{ "name": name, }, "cluster": common.MapStr{ - "name": nodesStruct.ClusterName, + "name": nodeData.ClusterName, }, } - event[mb.NamespaceKey] = "node.stats" - events = append(events, event) - errors.AddErrors(errs) + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", "elasticsearch") + r.Event(event) } - - return events, errors + return errs } diff --git a/metricbeat/module/elasticsearch/node_stats/data_test.go b/metricbeat/module/elasticsearch/node_stats/data_test.go index 860ff9c0e09..a10a7efdf5c 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_test.go +++ b/metricbeat/module/elasticsearch/node_stats/data_test.go @@ -8,6 +8,7 @@ import ( "testing" s "github.com/elastic/beats/libbeat/common/schema" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/stretchr/testify/assert" ) @@ -17,18 +18,17 @@ func TestStats(t *testing.T) { assert.NoError(t, err) for _, f := range files { - content, err := ioutil.ReadFile(f) + input, err := ioutil.ReadFile(f) assert.NoError(t, err) - _, errs := eventsMapping(content) - if errs == nil { - continue - } - errors, ok := errs.(*s.Errors) - if ok { - assert.False(t, errors.HasRequiredErrors(), "mapping error: %s", errors) - } else { - t.Error(err) + reporter := &mbtest.CapturingReporterV2{} + errors := eventsMapping(reporter, input) + for _, errs := range errors { + if e, ok := errs.(*s.Errors); ok { + assert.False(t, e.HasRequiredErrors(), "mapping error: %s", e) + } } + assert.True(t, len(reporter.GetEvents()) >= 1) + assert.Equal(t, 0, len(reporter.GetErrors())) } } diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index db44f2246d3..f603d4f9857 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -1,7 +1,6 @@ package node_stats import ( - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" @@ -14,6 +13,7 @@ func init() { mb.Registry.MustAddMetricSet("elasticsearch", "node_stats", New, mb.WithHostParser(hostParser), mb.DefaultMetricSet(), + mb.WithNamespace("elasticsearch.node.stats"), ) } @@ -47,12 +47,12 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch methods implements the data gathering and data conversion to the right format -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { content, err := m.http.FetchContent() if err != nil { - return nil, err + r.Error(err) + return } - events, _ := eventsMapping(content) - return events, nil + eventsMapping(r, content) } diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go b/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go index 1832ec18f34..b9f544aed17 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go @@ -15,21 +15,19 @@ import ( func TestFetch(t *testing.T) { compose.EnsureUp(t, "elasticsearch") - f := mbtest.NewEventsFetcher(t, elasticsearch.GetConfig("node_stats")) - event, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() - } + f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node_stats")) + events, errs := mbtest.ReportingFetchV2(f) - assert.NotNil(t, event) - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + assert.NotNil(t, events) + assert.Nil(t, errs) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } func TestData(t *testing.T) { compose.EnsureUp(t, "elasticsearch") - f := mbtest.NewEventsFetcher(t, elasticsearch.GetConfig("node_stats")) - err := mbtest.WriteEvents(f, t) + f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node_stats")) + err := mbtest.WriteEventsReporterV2(f, t) if err != nil { t.Fatal("write", err) }