From 2354d1ca9e84c14fc503a26fc9f2cf19b21173d7 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 5 Feb 2019 12:58:45 +0200 Subject: [PATCH 01/24] Add coreDNS metricbeat module Signed-off-by: Chris Mark --- metricbeat/docs/fields.asciidoc | 42 ++++++- metricbeat/docs/modules/coredns.asciidoc | 37 ++++++ .../docs/modules/coredns/stats.asciidoc | 23 ++++ metricbeat/docs/modules_list.asciidoc | 6 +- metricbeat/include/list.go | 2 + metricbeat/metricbeat.reference.yml | 5 + .../module/coredns/_meta/config.reference.yml | 4 + metricbeat/module/coredns/_meta/config.yml | 4 + metricbeat/module/coredns/_meta/docs.asciidoc | 2 + metricbeat/module/coredns/_meta/fields.yml | 11 ++ metricbeat/module/coredns/doc.go | 19 +++ metricbeat/module/coredns/fields.go | 36 ++++++ .../module/coredns/stats/_meta/data.json | 23 ++++ .../module/coredns/stats/_meta/docs.asciidoc | 1 + .../module/coredns/stats/_meta/fields.yml | 9 ++ .../module/coredns/stats/_meta/test/metrics | 117 ++++++++++++++++++ metricbeat/module/coredns/stats/stats.go | 35 ++++++ metricbeat/modules.d/coredns.yml.disabled | 7 ++ x-pack/metricbeat/metricbeat.reference.yml | 6 + 19 files changed, 380 insertions(+), 9 deletions(-) create mode 100644 metricbeat/docs/modules/coredns.asciidoc create mode 100644 metricbeat/docs/modules/coredns/stats.asciidoc create mode 100644 metricbeat/module/coredns/_meta/config.reference.yml create mode 100644 metricbeat/module/coredns/_meta/config.yml create mode 100644 metricbeat/module/coredns/_meta/docs.asciidoc create mode 100644 metricbeat/module/coredns/_meta/fields.yml create mode 100644 metricbeat/module/coredns/doc.go create mode 100644 metricbeat/module/coredns/fields.go create mode 100644 metricbeat/module/coredns/stats/_meta/data.json create mode 100644 metricbeat/module/coredns/stats/_meta/docs.asciidoc create mode 100644 metricbeat/module/coredns/stats/_meta/fields.yml create mode 100644 metricbeat/module/coredns/stats/_meta/test/metrics create mode 100644 metricbeat/module/coredns/stats/stats.go create mode 100644 metricbeat/modules.d/coredns.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c62d528e624..be506be1fd0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20,6 +20,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2567,6 +2568,36 @@ Nanoseconds consumed by stop-the-world garbage collection pauses since Consul st -- +[[exported-fields-coredns]] +== coredns fields + +experimental[] +coredns module + + + +[float] +== coredns fields + + + + +[float] +== metrics fields + +metrics + + + +*`coredns.metrics.example`*:: ++ +-- +type: keyword + +Example field + +-- + [[exported-fields-couchbase]] == Couchbase fields @@ -21632,7 +21663,7 @@ type: long format: bytes -Used memory by the Lua engine. +Used memory by the Lua engine. -- @@ -21644,7 +21675,7 @@ type: long format: bytes -The size in bytes of the dataset +The size in bytes of the dataset -- @@ -21889,7 +21920,7 @@ type: long format: bytes -The size in bytes of copy-on-write allocations during the last RBD save operation +The size in bytes of copy-on-write allocations during the last RBD save operation -- @@ -22018,7 +22049,7 @@ type: long format: bytes -AOF current file size +AOF current file size -- @@ -22122,7 +22153,7 @@ Total size in bytes of the replication backlog buffer -- type: long -The master offset of the replication backlog buffer +The master offset of the replication backlog buffer -- @@ -26580,4 +26611,3 @@ type: long Epoch value of the Zookeeper transaction ID. An epoch signifies the period in which a server is a leader -- - diff --git a/metricbeat/docs/modules/coredns.asciidoc b/metricbeat/docs/modules/coredns.asciidoc new file mode 100644 index 00000000000..129a8fca1fa --- /dev/null +++ b/metricbeat/docs/modules/coredns.asciidoc @@ -0,0 +1,37 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-coredns]] +== coredns module + +experimental[] + +This is the coredns module. + + + +[float] +=== Example configuration + +The coredns module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::coredns/stats.asciidoc[] + diff --git a/metricbeat/docs/modules/coredns/stats.asciidoc b/metricbeat/docs/modules/coredns/stats.asciidoc new file mode 100644 index 00000000000..d341ddf7971 --- /dev/null +++ b/metricbeat/docs/modules/coredns/stats.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-coredns-stats]] +=== coredns stats metricset + +experimental[] + +include::../../../module/coredns/stats/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/coredns/stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 11108798182..69e3fa2bc54 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -4,7 +4,7 @@ This file is generated! See scripts/docs_collector.py [options="header"] |=================================== -|Modules |Dashboards |Metricsets +|Modules |Dashboards |Metricsets |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | @@ -167,8 +167,7 @@ This file is generated! See scripts/docs_collector.py |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .2+| .2+| |<> -|<> -|================================ +|<> -- @@ -177,6 +176,7 @@ include::modules/apache.asciidoc[] include::modules/aws.asciidoc[] include::modules/ceph.asciidoc[] include::modules/consul.asciidoc[] +include::modules/coredns.asciidoc[] include::modules/couchbase.asciidoc[] include::modules/couchdb.asciidoc[] include::modules/docker.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index eee014afda3..7caabc5c5cf 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -40,6 +40,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/ceph/pool_disk" _ "github.com/elastic/beats/metricbeat/module/consul" _ "github.com/elastic/beats/metricbeat/module/consul/agent" + _ "github.com/elastic/beats/metricbeat/module/coredns" + _ "github.com/elastic/beats/metricbeat/module/coredns/stats" _ "github.com/elastic/beats/metricbeat/module/couchbase" _ "github.com/elastic/beats/metricbeat/module/couchbase/bucket" _ "github.com/elastic/beats/metricbeat/module/couchbase/cluster" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index d34a25c95f4..2cd9a227efd 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -162,6 +162,11 @@ metricbeat.modules: period: 10s hosts: ["localhost:8500"] +#------------------------------- Coredns Module ------------------------------ +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] #------------------------------ Couchbase Module ----------------------------- - module: couchbase diff --git a/metricbeat/module/coredns/_meta/config.reference.yml b/metricbeat/module/coredns/_meta/config.reference.yml new file mode 100644 index 00000000000..d8155ff00e5 --- /dev/null +++ b/metricbeat/module/coredns/_meta/config.reference.yml @@ -0,0 +1,4 @@ +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] diff --git a/metricbeat/module/coredns/_meta/config.yml b/metricbeat/module/coredns/_meta/config.yml new file mode 100644 index 00000000000..d8155ff00e5 --- /dev/null +++ b/metricbeat/module/coredns/_meta/config.yml @@ -0,0 +1,4 @@ +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] diff --git a/metricbeat/module/coredns/_meta/docs.asciidoc b/metricbeat/module/coredns/_meta/docs.asciidoc new file mode 100644 index 00000000000..4444dcee1f0 --- /dev/null +++ b/metricbeat/module/coredns/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +This is the coredns module. + diff --git a/metricbeat/module/coredns/_meta/fields.yml b/metricbeat/module/coredns/_meta/fields.yml new file mode 100644 index 00000000000..8442fcdc129 --- /dev/null +++ b/metricbeat/module/coredns/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: coredns + title: "coredns" + description: > + experimental[] + + coredns module + fields: + - name: coredns + type: group + description: > + fields: diff --git a/metricbeat/module/coredns/doc.go b/metricbeat/module/coredns/doc.go new file mode 100644 index 00000000000..57b552972d8 --- /dev/null +++ b/metricbeat/module/coredns/doc.go @@ -0,0 +1,19 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package coredns is a Metricbeat module that contains MetricSets. +package coredns diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go new file mode 100644 index 00000000000..06d888ac864 --- /dev/null +++ b/metricbeat/module/coredns/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package coredns + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "coredns", asset.ModuleFieldsPri, AssetCoredns); err != nil { + panic(err) + } +} + +// AssetCoredns returns asset data. +// This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. +func AssetCoredns() string { + return "eJx8j02ugyAYRees4sa5G2DwZm8VTQdGbhsifwFMdfeNQhpqbe/wfOGc0GPiKjH6SOWSALLOhhJdJZ0AFNMYdcjaO4k/AQBcAqO2dHkwl6vYWX0B69VsKICbplFJ7scebrBsQ9vyGihxj34OlZy03kWtzDJHPaYXPxN+lZZ9Ko6xNshlsGH/W7sSnbg+fFSH24/0tv8iLFHxDAAA///GwWq1" +} diff --git a/metricbeat/module/coredns/stats/_meta/data.json b/metricbeat/module/coredns/stats/_meta/data.json new file mode 100644 index 00000000000..4ae76d5782d --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data.json @@ -0,0 +1,23 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"coredns", + "name":"stats", + "rtt":44269 + }, + "coredns": { + "stats": { + "panic": { + "count": { + "total": 0 + } + } + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/coredns/stats/_meta/docs.asciidoc b/metricbeat/module/coredns/stats/_meta/docs.asciidoc new file mode 100644 index 00000000000..5bf0e0a26e6 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the metrics metricset of the module coredns. diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml new file mode 100644 index 00000000000..0fe0994a782 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: metrics + type: group + description: > + metrics + fields: + - name: example + type: keyword + description: > + Example field diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics b/metricbeat/module/coredns/stats/_meta/test/metrics new file mode 100644 index 00000000000..e55fb82cde4 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/test/metrics @@ -0,0 +1,117 @@ +# HELP coredns_build_info A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built. +# TYPE coredns_build_info gauge +coredns_build_info{goversion="go1.11.4",revision="6b56a9c",version="1.3.1"} 1.0 +# HELP coredns_panic_count_total A metrics that counts the number of panics. +# TYPE coredns_panic_count_total counter +coredns_panic_count_total 0.0 +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0.0"} 0.000149309 +go_gc_duration_seconds{quantile="0.25"} 0.000163557 +go_gc_duration_seconds{quantile="0.5"} 0.000185375 +go_gc_duration_seconds{quantile="0.75"} 0.000199059 +go_gc_duration_seconds{quantile="1.0"} 0.000231874 +go_gc_duration_seconds_sum 0.002423979 +go_gc_duration_seconds_count 13.0 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 15.0 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.11.4"} 1.0 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 2.818416e+06 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 4.683968e+06 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.445694e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 12634.0 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 2.817457627458492e-06 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 2.371584e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 2.818416e+06 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 6.0940288e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 4.169728e+06 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 12777.0 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 6.0891136e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 6.5110016e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.5493699519654317e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0.0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 25411.0 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 55296.0 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 65536.0 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 62016.0 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 65536.0 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 5.454832e+06 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 1.489338e+06 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 1.998848e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 1.998848e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 7.2546552e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 44.0 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 0.53 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 9.0 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 2.822144e+07 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.54936851083e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 1.46489344e+08 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1.0 diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go new file mode 100644 index 00000000000..ed5b21a0efd --- /dev/null +++ b/metricbeat/module/coredns/stats/stats.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package stats + +import ( + "github.com/elastic/beats/metricbeat/helper/prometheus" + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + mapping := &prometheus.MetricsMapping{ + Metrics: map[string]prometheus.MetricMap{ + "coredns_panic_count_total": prometheus.Metric("panic.count.total"), + }, + } + + mb.Registry.MustAddMetricSet("coredns", "stats", + prometheus.MetricSetBuilder(mapping), + mb.WithHostParser(prometheus.HostParser)) +} diff --git a/metricbeat/modules.d/coredns.yml.disabled b/metricbeat/modules.d/coredns.yml.disabled new file mode 100644 index 00000000000..644a62bc4b7 --- /dev/null +++ b/metricbeat/modules.d/coredns.yml.disabled @@ -0,0 +1,7 @@ +# Module: coredns +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-coredns.html + +- module: coredns + metricsets: ["stats"] + period: 10s + hosts: ["localhost:9153"] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 37dff29caae..0f84cfda4fd 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -182,6 +182,12 @@ metricbeat.modules: period: 10s hosts: ["localhost:8500"] +#------------------------------- Coredns Module ------------------------------ +- module: coredns + metricsets: ["stats"] + enabled: true + period: 10s + hosts: ["localhost:9153"] #------------------------------ Couchbase Module ------------------------------ - module: couchbase From 40da57fc51177d92e0cb4341e6be68779e9d9a42 Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Sun, 10 Feb 2019 16:55:52 +0200 Subject: [PATCH 02/24] Add more fields to stats Metricset * Update _meta/test/metrics * Move mapping declaration outside of init() Signed-off-by: Ioannis Androulidakis --- .../module/coredns/stats/_meta/test/metrics | 140 +++++++++++++----- metricbeat/module/coredns/stats/stats.go | 28 +++- .../{coredns.yml.disabled => coredns.yml} | 0 .../{system.yml => system.yml.disabled} | 0 4 files changed, 126 insertions(+), 42 deletions(-) rename metricbeat/modules.d/{coredns.yml.disabled => coredns.yml} (100%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics b/metricbeat/module/coredns/stats/_meta/test/metrics index e55fb82cde4..3628f4377da 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics +++ b/metricbeat/module/coredns/stats/_meta/test/metrics @@ -1,102 +1,170 @@ # HELP coredns_build_info A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built. # TYPE coredns_build_info gauge coredns_build_info{goversion="go1.11.4",revision="6b56a9c",version="1.3.1"} 1.0 +# HELP coredns_dns_request_count_total Counter of DNS requests made per zone, protocol and family. +# TYPE coredns_dns_request_count_total counter +coredns_dns_request_count_total{family="1",proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 +# HELP coredns_dns_request_duration_seconds Histogram of the time (in seconds) each request took. +# TYPE coredns_dns_request_duration_seconds histogram +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.00025"} 3894.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.0005"} 3928.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.001"} 3939.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.002"} 3939.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.004"} 3939.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.008"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.016"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.032"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.064"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.128"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.256"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.512"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="1.024"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="2.048"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="4.096"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="8.192"} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 +coredns_dns_request_duration_seconds_sum{server="dns://:53",zone="mydomain.org."} 0.5743562010000005 +coredns_dns_request_duration_seconds_count{server="dns://:53",zone="mydomain.org."} 3940.0 +# HELP coredns_dns_request_size_bytes Size of the EDNS0 UDP buffer in bytes (64K for TCP). +# TYPE coredns_dns_request_size_bytes histogram +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="0.0"} 0.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="100.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="200.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="300.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="400.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="511.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="1023.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="2047.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="4095.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="8291.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="16000.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="32000.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="48000.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="64000.0"} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 +coredns_dns_request_size_bytes_sum{proto="udp",server="dns://:53",zone="mydomain.org."} 236400.0 +coredns_dns_request_size_bytes_count{proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 +# HELP coredns_dns_request_type_count_total Counter of DNS requests per type, per zone. +# TYPE coredns_dns_request_type_count_total counter +coredns_dns_request_type_count_total{server="dns://:53",type="A",zone="mydomain.org."} 3940.0 +# HELP coredns_dns_response_rcode_count_total Counter of response status codes. +# TYPE coredns_dns_response_rcode_count_total counter +coredns_dns_response_rcode_count_total{rcode="NOERROR",server="dns://:53",zone="mydomain.org."} 3940.0 +# HELP coredns_dns_response_size_bytes Size of the returned response in bytes. +# TYPE coredns_dns_response_size_bytes histogram +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="0.0"} 0.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="100.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="200.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="300.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="400.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="511.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="1023.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="2047.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="4095.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="8291.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="16000.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="32000.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="48000.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="64000.0"} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 +coredns_dns_response_size_bytes_sum{proto="udp",server="dns://:53",zone="mydomain.org."} 283680.0 +coredns_dns_response_size_bytes_count{proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 # HELP coredns_panic_count_total A metrics that counts the number of panics. # TYPE coredns_panic_count_total counter coredns_panic_count_total 0.0 # HELP go_gc_duration_seconds A summary of the GC invocation durations. # TYPE go_gc_duration_seconds summary -go_gc_duration_seconds{quantile="0.0"} 0.000149309 -go_gc_duration_seconds{quantile="0.25"} 0.000163557 -go_gc_duration_seconds{quantile="0.5"} 0.000185375 -go_gc_duration_seconds{quantile="0.75"} 0.000199059 -go_gc_duration_seconds{quantile="1.0"} 0.000231874 -go_gc_duration_seconds_sum 0.002423979 -go_gc_duration_seconds_count 13.0 +go_gc_duration_seconds{quantile="0.0"} 4.2171e-05 +go_gc_duration_seconds{quantile="0.25"} 4.8407e-05 +go_gc_duration_seconds{quantile="0.5"} 6.3342e-05 +go_gc_duration_seconds{quantile="0.75"} 8.6721e-05 +go_gc_duration_seconds{quantile="1.0"} 0.000379492 +go_gc_duration_seconds_sum 0.001787514 +go_gc_duration_seconds_count 21.0 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge -go_goroutines 15.0 +go_goroutines 20.0 # HELP go_info Information about the Go environment. # TYPE go_info gauge go_info{version="go1.11.4"} 1.0 # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. # TYPE go_memstats_alloc_bytes gauge -go_memstats_alloc_bytes 2.818416e+06 +go_memstats_alloc_bytes 2.691304e+06 # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. # TYPE go_memstats_alloc_bytes_total counter -go_memstats_alloc_bytes_total 4.683968e+06 +go_memstats_alloc_bytes_total 3.5058456e+07 # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. # TYPE go_memstats_buck_hash_sys_bytes gauge -go_memstats_buck_hash_sys_bytes 1.445694e+06 +go_memstats_buck_hash_sys_bytes 1.455718e+06 # HELP go_memstats_frees_total Total number of frees. # TYPE go_memstats_frees_total counter -go_memstats_frees_total 12634.0 +go_memstats_frees_total 457516.0 # HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. # TYPE go_memstats_gc_cpu_fraction gauge -go_memstats_gc_cpu_fraction 2.817457627458492e-06 +go_memstats_gc_cpu_fraction 5.612791592054919e-06 # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. # TYPE go_memstats_gc_sys_bytes gauge -go_memstats_gc_sys_bytes 2.371584e+06 +go_memstats_gc_sys_bytes 2.379776e+06 # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. # TYPE go_memstats_heap_alloc_bytes gauge -go_memstats_heap_alloc_bytes 2.818416e+06 +go_memstats_heap_alloc_bytes 2.691304e+06 # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. # TYPE go_memstats_heap_idle_bytes gauge -go_memstats_heap_idle_bytes 6.0940288e+07 +go_memstats_heap_idle_bytes 6.1980672e+07 # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. # TYPE go_memstats_heap_inuse_bytes gauge -go_memstats_heap_inuse_bytes 4.169728e+06 +go_memstats_heap_inuse_bytes 4.210688e+06 # HELP go_memstats_heap_objects Number of allocated objects. # TYPE go_memstats_heap_objects gauge -go_memstats_heap_objects 12777.0 +go_memstats_heap_objects 14109.0 # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. # TYPE go_memstats_heap_released_bytes gauge -go_memstats_heap_released_bytes 6.0891136e+07 +go_memstats_heap_released_bytes 6.1341696e+07 # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. # TYPE go_memstats_heap_sys_bytes gauge -go_memstats_heap_sys_bytes 6.5110016e+07 +go_memstats_heap_sys_bytes 6.619136e+07 # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. # TYPE go_memstats_last_gc_time_seconds gauge -go_memstats_last_gc_time_seconds 1.5493699519654317e+09 +go_memstats_last_gc_time_seconds 1.549753511946267e+09 # HELP go_memstats_lookups_total Total number of pointer lookups. # TYPE go_memstats_lookups_total counter go_memstats_lookups_total 0.0 # HELP go_memstats_mallocs_total Total number of mallocs. # TYPE go_memstats_mallocs_total counter -go_memstats_mallocs_total 25411.0 +go_memstats_mallocs_total 471625.0 # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. # TYPE go_memstats_mcache_inuse_bytes gauge -go_memstats_mcache_inuse_bytes 55296.0 +go_memstats_mcache_inuse_bytes 6912.0 # HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. # TYPE go_memstats_mcache_sys_bytes gauge -go_memstats_mcache_sys_bytes 65536.0 +go_memstats_mcache_sys_bytes 16384.0 # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. # TYPE go_memstats_mspan_inuse_bytes gauge -go_memstats_mspan_inuse_bytes 62016.0 +go_memstats_mspan_inuse_bytes 59888.0 # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. # TYPE go_memstats_mspan_sys_bytes gauge -go_memstats_mspan_sys_bytes 65536.0 +go_memstats_mspan_sys_bytes 98304.0 # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. # TYPE go_memstats_next_gc_bytes gauge -go_memstats_next_gc_bytes 5.454832e+06 +go_memstats_next_gc_bytes 4.767168e+06 # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. # TYPE go_memstats_other_sys_bytes gauge -go_memstats_other_sys_bytes 1.489338e+06 +go_memstats_other_sys_bytes 1.225362e+06 # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. # TYPE go_memstats_stack_inuse_bytes gauge -go_memstats_stack_inuse_bytes 1.998848e+06 +go_memstats_stack_inuse_bytes 917504.0 # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. # TYPE go_memstats_stack_sys_bytes gauge -go_memstats_stack_sys_bytes 1.998848e+06 +go_memstats_stack_sys_bytes 917504.0 # HELP go_memstats_sys_bytes Number of bytes obtained from system. # TYPE go_memstats_sys_bytes gauge -go_memstats_sys_bytes 7.2546552e+07 +go_memstats_sys_bytes 7.2284408e+07 # HELP go_threads Number of OS threads created. # TYPE go_threads gauge -go_threads 44.0 +go_threads 17.0 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter -process_cpu_seconds_total 0.53 +process_cpu_seconds_total 2.13 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 @@ -105,13 +173,13 @@ process_max_fds 1.048576e+06 process_open_fds 9.0 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge -process_resident_memory_bytes 2.822144e+07 +process_resident_memory_bytes 2.7910144e+07 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge -process_start_time_seconds 1.54936851083e+09 +process_start_time_seconds 1.54975119258e+09 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge -process_virtual_memory_bytes 1.46489344e+08 +process_virtual_memory_bytes 1.462272e+08 # HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. # TYPE process_virtual_memory_max_bytes gauge process_virtual_memory_max_bytes -1.0 diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index ed5b21a0efd..3bf7df04e09 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -22,13 +22,29 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func init() { - mapping := &prometheus.MetricsMapping{ - Metrics: map[string]prometheus.MetricMap{ - "coredns_panic_count_total": prometheus.Metric("panic.count.total"), - }, - } +var mapping = &prometheus.MetricsMapping{ + Metrics: map[string]prometheus.MetricMap{ + "coredns_panic_count_total": prometheus.Metric("panic.count.total"), + "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), + "coredns_dns_request_duration_seconds": prometheus.Metric("dns.request.duration.seconds"), + "coredns_dns_request_size_bytes": prometheus.Metric("dns.request.size.bytes"), + "coredns_dns_request_do_count_total": prometheus.Metric("dns.request.do.count.total"), + "coredns_dns_request_type_count_total": prometheus.Metric("dns.request.type.count.total"), + "coredns_dns_response_size_bytes": prometheus.Metric("dns.response.size.bytes"), + "coredns_dns_response_rcode_count_total": prometheus.Metric("dns.response.rcode.count.total"), + }, + + Labels: map[string]prometheus.LabelMap{ + "server": prometheus.KeyLabel("server"), + "zone": prometheus.KeyLabel("zone"), + "type": prometheus.KeyLabel("dns.request.type"), + "rcode": prometheus.KeyLabel("dns.response.rcode"), + "proto": prometheus.KeyLabel("proto"), + "family": prometheus.KeyLabel("dns.request.family"), + }, +} +func init() { mb.Registry.MustAddMetricSet("coredns", "stats", prometheus.MetricSetBuilder(mapping), mb.WithHostParser(prometheus.HostParser)) diff --git a/metricbeat/modules.d/coredns.yml.disabled b/metricbeat/modules.d/coredns.yml similarity index 100% rename from metricbeat/modules.d/coredns.yml.disabled rename to metricbeat/modules.d/coredns.yml diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled From 1a50fde4f6cbb67edc88339367fb0e5b7b569c95 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 11 Feb 2019 09:19:31 +0200 Subject: [PATCH 03/24] Add fields' docs Signed-off-by: Chris Mark --- metricbeat/docs/fields.asciidoc | 146 +++++++++++++++++- metricbeat/docs/modules/coredns.asciidoc | 2 +- .../docs/modules/coredns/stats.asciidoc | 2 +- metricbeat/docs/modules_list.asciidoc | 4 +- metricbeat/module/coredns/_meta/fields.yml | 6 +- metricbeat/module/coredns/fields.go | 2 +- .../module/coredns/stats/_meta/fields.yml | 62 +++++++- .../{coredns.yml => coredns.yml.disabled} | 0 .../{system.yml.disabled => system.yml} | 0 9 files changed, 208 insertions(+), 16 deletions(-) rename metricbeat/modules.d/{coredns.yml => coredns.yml.disabled} (100%) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (100%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index be506be1fd0..4ae1478e42e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2571,30 +2571,166 @@ Nanoseconds consumed by stop-the-world garbage collection pauses since Consul st [[exported-fields-coredns]] == coredns fields -experimental[] -coredns module +coredns Module [float] == coredns fields +`coredns` contains statistics that were read from coreDNS [float] == metrics fields -metrics +Contains statistics related to the coreDNS service -*`coredns.metrics.example`*:: +*`coredns.metrics.panic.count.total`*:: ++ +-- +type: long + +Total number of panics + + +-- + +*`coredns.metrics.dns.request.count.total`*:: ++ +-- +type: long + +Total query count + + +-- + +*`coredns.metrics.dns.request.duration.seconds`*:: ++ +-- +type: long + +format: duration + +Duration to process each query + + +-- + +*`coredns.metrics.dns.request.size.bytes`*:: ++ +-- +type: long + +format: bytes + +Size of requests in bytes + + +-- + +*`coredns.metrics.dns.request.do.count.total`*:: ++ +-- +type: long + +Number of queries that have the DO bit set + + +-- + +*`coredns.metrics.dns.request.type.count.total`*:: ++ +-- +type: long + +Counter of queries per zone and type + + +-- + +*`coredns.metrics.dns.request.type`*:: + -- type: keyword -Example field +Holds the query type of the request + + +-- + +*`coredns.metrics.dns.response.rcode`*:: ++ +-- +type: keyword + +Holds the rcode of the response + + +-- + +*`coredns.metrics.dns.request.family`*:: ++ +-- +type: keyword + +The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) + + +-- + +*`coredns.metrics.dns.response.size.bytes`*:: ++ +-- +type: long + +format: bytes + +Response size in bytes + + +-- + +*`coredns.metrics.dns.response.rcode.count.total`*:: ++ +-- +type: long + +Counter of responses per zone and rcode + + +-- + +*`coredns.metrics.server`*:: ++ +-- +type: keyword + +The server responsible for the request + + +-- + +*`coredns.metrics.zone`*:: ++ +-- +type: keyword + +The zonename used for the request/response + + +-- + +*`coredns.metrics.proto`*:: ++ +-- +type: keyword + +The transport of the response ("udp" or "tcp") -- diff --git a/metricbeat/docs/modules/coredns.asciidoc b/metricbeat/docs/modules/coredns.asciidoc index 129a8fca1fa..b93e6e7f646 100644 --- a/metricbeat/docs/modules/coredns.asciidoc +++ b/metricbeat/docs/modules/coredns.asciidoc @@ -5,7 +5,7 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-module-coredns]] == coredns module -experimental[] +beta[] This is the coredns module. diff --git a/metricbeat/docs/modules/coredns/stats.asciidoc b/metricbeat/docs/modules/coredns/stats.asciidoc index d341ddf7971..95ec886d6d7 100644 --- a/metricbeat/docs/modules/coredns/stats.asciidoc +++ b/metricbeat/docs/modules/coredns/stats.asciidoc @@ -5,7 +5,7 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-metricset-coredns-stats]] === coredns stats metricset -experimental[] +beta[] include::../../../module/coredns/stats/_meta/docs.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 69e3fa2bc54..b1b8d255dd2 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -5,7 +5,7 @@ This file is generated! See scripts/docs_collector.py [options="header"] |=================================== |Modules |Dashboards |Metricsets -|<> |image:./images/icon-no.png[No prebuilt dashboards] | +|<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> @@ -166,7 +166,7 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> +.2+| .2+| |<> |<> -- diff --git a/metricbeat/module/coredns/_meta/fields.yml b/metricbeat/module/coredns/_meta/fields.yml index 8442fcdc129..5c410353d96 100644 --- a/metricbeat/module/coredns/_meta/fields.yml +++ b/metricbeat/module/coredns/_meta/fields.yml @@ -1,11 +1,11 @@ - key: coredns title: "coredns" description: > - experimental[] - - coredns module + coredns Module + release: beta fields: - name: coredns type: group description: > + `coredns` contains statistics that were read from coreDNS fields: diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index 06d888ac864..69c7e078a89 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJx8j02ugyAYRees4sa5G2DwZm8VTQdGbhsifwFMdfeNQhpqbe/wfOGc0GPiKjH6SOWSALLOhhJdJZ0AFNMYdcjaO4k/AQBcAqO2dHkwl6vYWX0B69VsKICbplFJ7scebrBsQ9vyGihxj34OlZy03kWtzDJHPaYXPxN+lZZ9Ko6xNshlsGH/W7sSnbg+fFSH24/0tv8iLFHxDAAA///GwWq1" + return "eJy8lUFP20AQhe/5FU85EamkalVxsNRe4FAOpajwA9jsTsgKZ8fMjEHh11fr2BBSKyZS0j1Odt77PLNPOcUDrQp4FgpJR4BFK6nAuK2MR0Ag9RIri5wK/BgB6O7jF4e6pBEgVJJTKjAjcyNgHqkMWjSXT5HckjZN8rFVRQXuheuqrfT45HPX9t3BczIXk0LNWVSLXmELZ3gmIQi5gLnwsjG6uLppFTZRNnGWZBK9vtb7kHZg5XPeAyRUOqMAY9iCOhYoyVP0tNG+PbLubPNuMlcuRT/1XCebGpsr393qvqDkdL/1w46PaM5tFkOqlzMS8HxtpL0MIelU6LEmteORPNYkKzTygxChFpflpkqeU9CPksxZls4KdP17gl60bXnTlbAnVZDzizX7ILXGF5rOVkZ78/Y1DcHexBfKe23dFTH16PROl4+x5avXl5anFakN8sI9URObi9+YRYPS8PozwzEQcZ4130NWJHjhRHApNMofousleqDVM0vYD+onl0GbAa0DkqUyX660jjuItOKkNBXP4ShMjfAbztpucEJzt4zl6nA8twuCC0FyHtfaHZKJS1qxGE6+4Dsur3FyeY0nEs0x/jb5hK9N+exd/WwyGZ7p/00z/rS+yL4fyfLm6o8dls5tKy7/PrsOMf83khz2Baw1O5Y4KylPfDAomfawIFkxa6NWCtsMn3empBI2PizNWwS2YoqTcR2qMVgwNl+NJ6O/AQAA//+PFbyt" } diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 0fe0994a782..32b944a5d61 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -1,9 +1,65 @@ - name: metrics type: group description: > - metrics + Contains statistics related to the coreDNS service + release: beta fields: - - name: example + - name: panic.count.total + type: long + description: > + Total number of panics + - name: dns.request.count.total + type: long + description: > + Total query count + - name: dns.request.duration.seconds + type: long + format: duration + description: > + Duration to process each query + - name: dns.request.size.bytes + type: long + format: bytes + description: > + Size of requests in bytes + - name: dns.request.do.count.total + type: long + description: > + Number of queries that have the DO bit set + - name: dns.request.type.count.total + type: long + description: > + Counter of queries per zone and type + - name: dns.request.type + type: keyword + description: > + Holds the query type of the request + - name: dns.response.rcode + type: keyword + description: > + Holds the rcode of the response + - name: dns.request.family + type: keyword + description: > + The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) + - name: dns.response.size.bytes + type: long + format: bytes + description: > + Response size in bytes + - name: dns.response.rcode.count.total + type: long + description: > + Counter of responses per zone and rcode + - name: server + type: keyword + description: > + The server responsible for the request + - name: zone + type: keyword + description: > + The zonename used for the request/response + - name: proto type: keyword description: > - Example field + The transport of the response ("udp" or "tcp") diff --git a/metricbeat/modules.d/coredns.yml b/metricbeat/modules.d/coredns.yml.disabled similarity index 100% rename from metricbeat/modules.d/coredns.yml rename to metricbeat/modules.d/coredns.yml.disabled diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 100% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml From 9059bfcdc55aed182a918581ac8a7980bcf9e00c Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Sun, 17 Feb 2019 20:41:02 +0200 Subject: [PATCH 04/24] Add Dockerfile and configuration files for coredns Signed-off-by: Ioannis Androulidakis --- metricbeat/module/coredns/_meta/Dockerfile | 15 +++++++ .../module/coredns/_meta/config/Corefile | 39 +++++++++++++++++++ .../coredns/_meta/config/mydomain.hosts | 4 ++ 3 files changed, 58 insertions(+) create mode 100644 metricbeat/module/coredns/_meta/Dockerfile create mode 100644 metricbeat/module/coredns/_meta/config/Corefile create mode 100644 metricbeat/module/coredns/_meta/config/mydomain.hosts diff --git a/metricbeat/module/coredns/_meta/Dockerfile b/metricbeat/module/coredns/_meta/Dockerfile new file mode 100644 index 00000000000..0052f99ba2f --- /dev/null +++ b/metricbeat/module/coredns/_meta/Dockerfile @@ -0,0 +1,15 @@ +# Start from coredns base Docker image +FROM coredns/coredns:latest + +# For DNS +EXPOSE 53 53/udp + +# For Prometheus metrics +EXPOSE 9153 9153/tcp + +# Mount coredns configuration in container +VOLUME ["/etc/coredns"] + +# Start coredns with custom configuration file +ENTRYPOINT ["/coredns"] +CMD ["-conf", "/etc/coredns/Corefile"] diff --git a/metricbeat/module/coredns/_meta/config/Corefile b/metricbeat/module/coredns/_meta/config/Corefile new file mode 100644 index 00000000000..c5f606f124e --- /dev/null +++ b/metricbeat/module/coredns/_meta/config/Corefile @@ -0,0 +1,39 @@ +######################## Description and Version ########################## +# This Corefile configures multiple DNS zones and uses coreDNS plugins +# +# To run a container with the image generated from this Dockerfile, execute: +# +# docker run -m 256m \ +# --name coredns \ +# --expose=9153/tcp -p 127.0.0.1:9153:9153/tcp \ +# -v "$PWD"/config:/etc/coredns \ +# coredns:test -conf /etc/coredns/Corefile +# +# Version: 1.0 +# +########################################################################### + +# Zone1 +my.domain.org { + log + errors + auto + reload 10s + + # Use the prometheus plugin to export metrics at '/metrics' endpoint + prometheus :9153 + + # If you wish to use a standard Hosts File format, uncomment the following + # line and customize the specified file + hosts /etc/coredns/mydomain.hosts my.domain.org +} + +# Zone2 +. { + log + errors + prometheus :9153 + + # Use the forward plugin to forward traffic to external DNS servers + forward . 8.8.8.8 8.8.4.4 +} diff --git a/metricbeat/module/coredns/_meta/config/mydomain.hosts b/metricbeat/module/coredns/_meta/config/mydomain.hosts new file mode 100644 index 00000000000..defef025aa9 --- /dev/null +++ b/metricbeat/module/coredns/_meta/config/mydomain.hosts @@ -0,0 +1,4 @@ +# Hosts file for Domain: my.domain.org +# Place entries below in standard hosts file format: ipaddress hostname fqdn + +192.168.1.1 gateway.my.domain.org From 92f5c3e56870a93bd2598835f4c72a0eaad1477e Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Sat, 16 Feb 2019 16:18:48 +0200 Subject: [PATCH 05/24] Review fixes * Rename metrics to stats in coredns/stats/_meta/fields.yaml * Use .sec naming convention instead of .seconds Signed-off-by: Ioannis Androulidakis --- metricbeat/docs/fields.asciidoc | 2 +- metricbeat/module/coredns/stats/_meta/docs.asciidoc | 2 +- metricbeat/module/coredns/stats/_meta/fields.yml | 4 ++-- metricbeat/module/coredns/stats/stats.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 4ae1478e42e..684ecb49e88 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2609,7 +2609,7 @@ Total query count -- -*`coredns.metrics.dns.request.duration.seconds`*:: +*`coredns.metrics.dns.request.duration.sec`*:: + -- type: long diff --git a/metricbeat/module/coredns/stats/_meta/docs.asciidoc b/metricbeat/module/coredns/stats/_meta/docs.asciidoc index 5bf0e0a26e6..07c9f2f9fed 100644 --- a/metricbeat/module/coredns/stats/_meta/docs.asciidoc +++ b/metricbeat/module/coredns/stats/_meta/docs.asciidoc @@ -1 +1 @@ -This is the metrics metricset of the module coredns. +This is the stats metricset of the module coredns. diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 32b944a5d61..44807499fa4 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: metrics +- name: stats type: group description: > Contains statistics related to the coreDNS service @@ -12,7 +12,7 @@ type: long description: > Total query count - - name: dns.request.duration.seconds + - name: dns.request.duration.sec type: long format: duration description: > diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index 3bf7df04e09..45bba33aa6b 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -26,7 +26,7 @@ var mapping = &prometheus.MetricsMapping{ Metrics: map[string]prometheus.MetricMap{ "coredns_panic_count_total": prometheus.Metric("panic.count.total"), "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), - "coredns_dns_request_duration_seconds": prometheus.Metric("dns.request.duration.seconds"), + "coredns_dns_request_duration_seconds": prometheus.Metric("dns.request.duration.sec"), "coredns_dns_request_size_bytes": prometheus.Metric("dns.request.size.bytes"), "coredns_dns_request_do_count_total": prometheus.Metric("dns.request.do.count.total"), "coredns_dns_request_type_count_total": prometheus.Metric("dns.request.type.count.total"), From 551e51c4b850f2e10be4a8e2cfb4dd6428b2befd Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Sat, 16 Feb 2019 17:21:10 +0200 Subject: [PATCH 06/24] Add function test for coredns Signed-off-by: Ioannis Androulidakis --- metricbeat/module/coredns/stats/stats_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 metricbeat/module/coredns/stats/stats_test.go diff --git a/metricbeat/module/coredns/stats/stats_test.go b/metricbeat/module/coredns/stats/stats_test.go new file mode 100644 index 00000000000..4bb76f1add9 --- /dev/null +++ b/metricbeat/module/coredns/stats/stats_test.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package stats + +import ( + "testing" + + "github.com/elastic/beats/metricbeat/helper/prometheus/ptest" +) + +const testFile = "_meta/test/metrics" + +func TestEventMapping(t *testing.T) { + ptest.TestMetricSet(t, "coredns", "stats", + ptest.TestCases{ + { + MetricsFile: "./_meta/test/metrics", + ExpectedFile: "./_meta/test/metrics.expected", + }, + }, + ) +} From 2cc21fd9c4c83df70b1dc0718955a442e6b81dcb Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 18 Feb 2019 00:57:19 +0200 Subject: [PATCH 07/24] Fix mapping issue Signed-off-by: Chris --- metricbeat/docs/fields.asciidoc | 58 ++++++++++--------- metricbeat/docs/modules_list.asciidoc | 6 +- metricbeat/metricbeat.reference.yml | 7 ++- metricbeat/module/coredns/fields.go | 2 +- .../module/coredns/stats/_meta/fields.yml | 8 +-- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 684ecb49e88..ddfbaf4aa38 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2583,13 +2583,13 @@ coredns Module [float] -== metrics fields +== stats fields Contains statistics related to the coreDNS service -*`coredns.metrics.panic.count.total`*:: +*`coredns.stats.panic.count.total`*:: + -- type: long @@ -2599,7 +2599,7 @@ Total number of panics -- -*`coredns.metrics.dns.request.count.total`*:: +*`coredns.stats.dns.request.count.total`*:: + -- type: long @@ -2609,7 +2609,7 @@ Total query count -- -*`coredns.metrics.dns.request.duration.sec`*:: +*`coredns.stats.dns.request.duration.sec`*:: + -- type: long @@ -2621,7 +2621,7 @@ Duration to process each query -- -*`coredns.metrics.dns.request.size.bytes`*:: +*`coredns.stats.dns.request.size.bytes`*:: + -- type: long @@ -2633,7 +2633,7 @@ Size of requests in bytes -- -*`coredns.metrics.dns.request.do.count.total`*:: +*`coredns.stats.dns.request.do.count.total`*:: + -- type: long @@ -2643,7 +2643,7 @@ Number of queries that have the DO bit set -- -*`coredns.metrics.dns.request.type.count.total`*:: +*`coredns.stats.dns.request.type.count.total`*:: + -- type: long @@ -2653,7 +2653,7 @@ Counter of queries per zone and type -- -*`coredns.metrics.dns.request.type`*:: +*`coredns.stats.dns.request.type`*:: + -- type: keyword @@ -2663,49 +2663,49 @@ Holds the query type of the request -- -*`coredns.metrics.dns.response.rcode`*:: +*`coredns.stats.dns.response.rcode.count.total`*:: + -- -type: keyword +type: long -Holds the rcode of the response +Counter of responses per zone and rcode -- -*`coredns.metrics.dns.request.family`*:: +*`coredns.stats.dns.response.rcode`*:: + -- type: keyword -The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) +Holds the rcode of the response -- -*`coredns.metrics.dns.response.size.bytes`*:: +*`coredns.stats.dns.request.family`*:: + -- -type: long - -format: bytes +type: keyword -Response size in bytes +The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) -- -*`coredns.metrics.dns.response.rcode.count.total`*:: +*`coredns.stats.dns.response.size.bytes`*:: + -- type: long -Counter of responses per zone and rcode +format: bytes + +Response size in bytes -- -*`coredns.metrics.server`*:: +*`coredns.stats.server`*:: + -- type: keyword @@ -2715,7 +2715,7 @@ The server responsible for the request -- -*`coredns.metrics.zone`*:: +*`coredns.stats.zone`*:: + -- type: keyword @@ -2725,13 +2725,14 @@ The zonename used for the request/response -- -*`coredns.metrics.proto`*:: +*`coredns.stats.proto`*:: + -- type: keyword The transport of the response ("udp" or "tcp") + -- [[exported-fields-couchbase]] @@ -21799,7 +21800,7 @@ type: long format: bytes -Used memory by the Lua engine. +Used memory by the Lua engine. -- @@ -21811,7 +21812,7 @@ type: long format: bytes -The size in bytes of the dataset +The size in bytes of the dataset -- @@ -22056,7 +22057,7 @@ type: long format: bytes -The size in bytes of copy-on-write allocations during the last RBD save operation +The size in bytes of copy-on-write allocations during the last RBD save operation -- @@ -22185,7 +22186,7 @@ type: long format: bytes -AOF current file size +AOF current file size -- @@ -22289,7 +22290,7 @@ Total size in bytes of the replication backlog buffer -- type: long -The master offset of the replication backlog buffer +The master offset of the replication backlog buffer -- @@ -26747,3 +26748,4 @@ type: long Epoch value of the Zookeeper transaction ID. An epoch signifies the period in which a server is a leader -- + diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index b1b8d255dd2..66d8188c17a 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -23,8 +23,10 @@ This file is generated! See scripts/docs_collector.py |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] -|<> |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] +|<> |image:./images/icon-no.png[No prebuilt dashboards] | .3+| .3+| |<> |<> |<> diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 2cd9a227efd..f15e55a1174 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -162,7 +162,8 @@ metricbeat.modules: period: 10s hosts: ["localhost:8500"] -#------------------------------- Coredns Module ------------------------------ + +#------------------------------- coredns Module ------------------------------ - module: coredns metricsets: ["stats"] period: 10s @@ -841,7 +842,7 @@ metricbeat.modules: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, +# The supported processors are drop_fields, drop_event, include_fields, # decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that @@ -932,7 +933,7 @@ metricbeat.modules: # match_pids: ["system.process.ppid"] # target: system.process.parent # -# The following example decodes fields containing JSON strings +# The following example decodes fields containing JSON strings # and replaces the strings with valid JSON objects. # #processors: diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index 69c7e078a89..2e5f479743b 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJy8lUFP20AQhe/5FU85EamkalVxsNRe4FAOpajwA9jsTsgKZ8fMjEHh11fr2BBSKyZS0j1Odt77PLNPOcUDrQp4FgpJR4BFK6nAuK2MR0Ag9RIri5wK/BgB6O7jF4e6pBEgVJJTKjAjcyNgHqkMWjSXT5HckjZN8rFVRQXuheuqrfT45HPX9t3BczIXk0LNWVSLXmELZ3gmIQi5gLnwsjG6uLppFTZRNnGWZBK9vtb7kHZg5XPeAyRUOqMAY9iCOhYoyVP0tNG+PbLubPNuMlcuRT/1XCebGpsr393qvqDkdL/1w46PaM5tFkOqlzMS8HxtpL0MIelU6LEmteORPNYkKzTygxChFpflpkqeU9CPksxZls4KdP17gl60bXnTlbAnVZDzizX7ILXGF5rOVkZ78/Y1DcHexBfKe23dFTH16PROl4+x5avXl5anFakN8sI9URObi9+YRYPS8PozwzEQcZ4130NWJHjhRHApNMofousleqDVM0vYD+onl0GbAa0DkqUyX660jjuItOKkNBXP4ShMjfAbztpucEJzt4zl6nA8twuCC0FyHtfaHZKJS1qxGE6+4Dsur3FyeY0nEs0x/jb5hK9N+exd/WwyGZ7p/00z/rS+yL4fyfLm6o8dls5tKy7/PrsOMf83khz2Baw1O5Y4KylPfDAomfawIFkxa6NWCtsMn3empBI2PizNWwS2YoqTcR2qMVgwNl+NJ6O/AQAA//+PFbyt" + return "eJy8lU9P4zwQxu/9FI96otJLXu1qxSHS7gUOy2FZtPABcO0ptUjtMDMBhU+/cv5AiKIGpHZ9nMbP8/MzHvcUD1TnsJHJBVkA6rWgHMuuslwAjsSyL9XHkOPHAkD/PX5FVxW0AJgKMkI51qRmAWw8FU7y5uNTBLOjoUlaWpeU455jVXaVCZ+07rp9d7AxqPFBIGrUi3or0K1RPBMTmIzDhuOuMbq4uukUhihDnCQir9UpoD1QaZ1P4DAVRslBI3RLPQmE+MlbGmwfB9avMe2QuDTB28zGKmimUU3x7qv+BEUM96Mf9hyiWbdJDKHarYkRN62RTDK4IBnTY0WixyN5rIhrNPKzEK5ik+QyIftRik3kndEc/d5PQl5021KXS46WREDGblvuWWLxL5StayX5LO/UpjnYG/9Cqaedu8CHCZ3JZOMxOnz1estSWp66Ed6aJ2pG5uI31l4hNN/6xHAMRJwnzfeQJTFeYiCY4BrlD9FNEj1Q/RzZfQ7qZyycNAG1w5GkEl+qdI57iKSMQShjG90xEhvk1XuNEmucPwh4jNAa4be8WrvZFm7Mzhf14XhutwTjHKcHo9XukZRNkDKy4uQLvuPyGieX13gilvTOfFv9h69N+exd/Wy1ms/03z43+NP5Ivnuf2zSvyLxYeNtNfsW+3VB6TizY5Iu6mFBkmLSRiXkxgz/772CJUeNh6V5u1+jGcDJsnLlEpGxVFsuV4u/AQAA///NeLnx" } diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 44807499fa4..733441e9d86 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -34,6 +34,10 @@ type: keyword description: > Holds the query type of the request + - name: dns.response.rcode.count.total + type: long + description: > + Counter of responses per zone and rcode - name: dns.response.rcode type: keyword description: > @@ -47,10 +51,6 @@ format: bytes description: > Response size in bytes - - name: dns.response.rcode.count.total - type: long - description: > - Counter of responses per zone and rcode - name: server type: keyword description: > From a32796c490de6c4f7423c1509cfc09fa537bd4d4 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 18 Feb 2019 09:38:59 +0200 Subject: [PATCH 08/24] Update x-pack yml Signed-off-by: Chris --- x-pack/metricbeat/metricbeat.reference.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 0f84cfda4fd..3d2db806630 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -182,10 +182,10 @@ metricbeat.modules: period: 10s hosts: ["localhost:8500"] -#------------------------------- Coredns Module ------------------------------ + +#------------------------------- Coredns Module ------------------------------- - module: coredns metricsets: ["stats"] - enabled: true period: 10s hosts: ["localhost:9153"] @@ -871,7 +871,7 @@ metricbeat.modules: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, +# The supported processors are drop_fields, drop_event, include_fields, # decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that @@ -962,7 +962,7 @@ metricbeat.modules: # match_pids: ["system.process.ppid"] # target: system.process.parent # -# The following example decodes fields containing JSON strings +# The following example decodes fields containing JSON strings # and replaces the strings with valid JSON objects. # #processors: From ae4cfa9f85180b5b346b86adee40bdd2acbb48ae Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 18 Feb 2019 09:48:04 +0200 Subject: [PATCH 09/24] Add Changelog entries Signed-off-by: Chris --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b3d4e85068d..22b290743ec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -350,6 +350,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] - Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] - Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055] +- Add `coredns` metricbeat {pull}10585{10585] *Packetbeat* From 62a6e986cd15e0e30e0e6a6736abdfd6a76a3f6d Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 18 Feb 2019 11:25:13 +0200 Subject: [PATCH 10/24] Add draft for datageneration Signed-off-by: Chris Mark --- .../coredns/stats/stats_integration_test.go | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 metricbeat/module/coredns/stats/stats_integration_test.go diff --git a/metricbeat/module/coredns/stats/stats_integration_test.go b/metricbeat/module/coredns/stats/stats_integration_test.go new file mode 100644 index 00000000000..6976e7d990a --- /dev/null +++ b/metricbeat/module/coredns/stats/stats_integration_test.go @@ -0,0 +1,86 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package stats + +import ( + "fmt" + "os" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + compose.EnsureUp(t, "coredns") + // TODO: Use this function to find if an event is the wanted one + eventIs := func(event string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + hasEvent, _ := e.HasKey("event") + return hasEvent + } + } + dataFiles := []struct { + eventType string + path string + }{ + {"rcode_event", "./_meta/data_rcode_event.json"}, + {"panic_event", "./_meta/data_panic_event.json"}, + } + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + + for _, df := range dataFiles { + t.Run(fmt.Sprintf("event type:%s", df.eventType), func(t *testing.T) { + err = mbtest.WriteEventsReporterV2Cond(f, t, df.path, eventIs(df.eventType)) + if err != nil { + t.Fatal("write", err) + } + }) + } + +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "coredns", + "metricsets": []string{"stats"}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} + +func GetEnvHost() string { + host := os.Getenv("COREDNS_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func GetEnvPort() string { + port := os.Getenv("COREDNS_PORT") + + if len(port) == 0 { + port = "9153" + } + return port +} From 6ff4d12a0d1ec75e28c23c4bf321cab2ba9a3e18 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 18 Feb 2019 11:45:36 +0200 Subject: [PATCH 11/24] Add coredns servcie in docker-compose of metricbeat Signed-off-by: Chris Mark --- metricbeat/docker-compose.yml | 4 ++++ metricbeat/module/coredns/_meta/env | 2 ++ 2 files changed, 6 insertions(+) create mode 100644 metricbeat/module/coredns/_meta/env diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index c987f374560..9b4a4c77852 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -16,6 +16,7 @@ services: - ./module/apache/_meta/env - ./module/ceph/_meta/env - ./module/consul/_meta/env + - ./module/coredns/_meta/env - ./module/couchbase/_meta/env - ./module/couchdb/_meta/env - ./module/dropwizard/_meta/env @@ -63,6 +64,9 @@ services: consul: build: ./module/consul/_meta + coredns: + build: ./module/coredns/_meta + couchbase: build: ./module/couchbase/_meta diff --git a/metricbeat/module/coredns/_meta/env b/metricbeat/module/coredns/_meta/env new file mode 100644 index 00000000000..b8ad57b40ff --- /dev/null +++ b/metricbeat/module/coredns/_meta/env @@ -0,0 +1,2 @@ +COREDNS_HOST=coredns +COREDNS_PORT=9153 From 7b970f162cb1073ad8dc5377f8bd80add0219e3a Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 19 Feb 2019 00:28:59 +0200 Subject: [PATCH 12/24] Fix Dockerfile & add data generation Signed-off-by: Chris Co-Authored-By: Ioannis Androulidakis --- metricbeat/module/coredns/_meta/Dockerfile | 14 +++- .../coredns/stats/_meta/data_panic_event.json | 28 +++++++ .../stats/_meta/data_request_count_event.json | 34 ++++++++ .../data_request_duration_sec_event.json | 54 +++++++++++++ .../_meta/data_request_size_bytes_event.json | 78 +++++++++++++++++++ .../stats/_meta/data_request_type_event.json | 30 +++++++ .../_meta/data_response_rcode_event.json | 30 +++++++ .../_meta/data_response_size_bytes_event.json | 78 +++++++++++++++++++ .../coredns/stats/stats_integration_test.go | 16 ++-- metricbeat/module/coredns/stats/stats_test.go | 18 ++--- 10 files changed, 362 insertions(+), 18 deletions(-) create mode 100644 metricbeat/module/coredns/stats/_meta/data_panic_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_request_count_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_request_type_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json diff --git a/metricbeat/module/coredns/_meta/Dockerfile b/metricbeat/module/coredns/_meta/Dockerfile index 0052f99ba2f..3b507109e6d 100644 --- a/metricbeat/module/coredns/_meta/Dockerfile +++ b/metricbeat/module/coredns/_meta/Dockerfile @@ -1,14 +1,24 @@ # Start from coredns base Docker image FROM coredns/coredns:latest +# create an enhanced container with nc command available since coredns is based +# on scratch image making healthcheck impossible +FROM alpine:latest +COPY --from=0 /coredns /coredns +# Expose client, management, and cluster ports # For DNS EXPOSE 53 53/udp # For Prometheus metrics EXPOSE 9153 9153/tcp -# Mount coredns configuration in container -VOLUME ["/etc/coredns"] +# Copy coredns configuration in container +ADD config /etc/coredns + +RUN apk add --update --no-cache bind-tools + +#HEALTHCHECK --retries=10 CMD dig @0.0.0.0 www.elastic.com >/dev/null +HEALTHCHECK --interval=1s --retries=10 CMD dig @0.0.0.0 www.elastic.com >/dev/null # Start coredns with custom configuration file ENTRYPOINT ["/coredns"] diff --git a/metricbeat/module/coredns/stats/_meta/data_panic_event.json b/metricbeat/module/coredns/stats/_meta/data_panic_event.json new file mode 100644 index 00000000000..742b910c93a --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_panic_event.json @@ -0,0 +1,28 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "panic": { + "count": { + "total": 0 + } + } + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json new file mode 100644 index 00000000000..c2586b26840 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "count": { + "total": 2210 + }, + "family": "1" + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json new file mode 100644 index 00000000000..af5dae9b4c9 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json @@ -0,0 +1,54 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "duration": { + "sec": { + "bucket": { + "+Inf": 2210, + "0.00025": 0, + "0.0005": 0, + "0.001": 0, + "0.002": 0, + "0.004": 0, + "0.008": 0, + "0.016": 0, + "0.032": 0, + "0.064": 0, + "0.128": 2200, + "0.256": 2210, + "0.512": 2210, + "1.024": 2210, + "2.048": 2210, + "4.096": 2210, + "8.192": 2210 + }, + "count": 2210, + "sum": 173.6070170089998 + } + } + } + }, + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json new file mode 100644 index 00000000000..6b8dd871b19 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2210, + "0": 0, + "100": 2210, + "1023": 2210, + "16000": 2210, + "200": 2210, + "2047": 2210, + "300": 2210, + "32000": 2210, + "400": 2210, + "4095": 2210, + "48000": 2210, + "511": 2210, + "64000": 2210, + "8291": 2210 + }, + "count": 2210, + "sum": 123760 + } + } + }, + "response": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2210, + "0": 0, + "100": 0, + "1023": 2210, + "16000": 2210, + "200": 2210, + "2047": 2210, + "300": 2210, + "32000": 2210, + "400": 2210, + "4095": 2210, + "48000": 2210, + "511": 2210, + "64000": 2210, + "8291": 2210 + }, + "count": 2210, + "sum": 245310 + } + } + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json new file mode 100644 index 00000000000..fedc70fd19d --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "type": "A" + } + }, + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json new file mode 100644 index 00000000000..a02ff606a21 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "response": { + "rcode": "NOERROR" + } + }, + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json new file mode 100644 index 00000000000..6b8dd871b19 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2210, + "0": 0, + "100": 2210, + "1023": 2210, + "16000": 2210, + "200": 2210, + "2047": 2210, + "300": 2210, + "32000": 2210, + "400": 2210, + "4095": 2210, + "48000": 2210, + "511": 2210, + "64000": 2210, + "8291": 2210 + }, + "count": 2210, + "sum": 123760 + } + } + }, + "response": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2210, + "0": 0, + "100": 0, + "1023": 2210, + "16000": 2210, + "200": 2210, + "2047": 2210, + "300": 2210, + "32000": 2210, + "400": 2210, + "4095": 2210, + "48000": 2210, + "511": 2210, + "64000": 2210, + "8291": 2210 + }, + "count": 2210, + "sum": 245310 + } + } + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/stats_integration_test.go b/metricbeat/module/coredns/stats/stats_integration_test.go index 6976e7d990a..b74c6110517 100644 --- a/metricbeat/module/coredns/stats/stats_integration_test.go +++ b/metricbeat/module/coredns/stats/stats_integration_test.go @@ -32,9 +32,9 @@ import ( func TestData(t *testing.T) { compose.EnsureUp(t, "coredns") // TODO: Use this function to find if an event is the wanted one - eventIs := func(event string) func(e common.MapStr) bool { + eventIs := func(eventType string) func(e common.MapStr) bool { return func(e common.MapStr) bool { - hasEvent, _ := e.HasKey("event") + hasEvent, _ := e.HasKey(eventType) return hasEvent } } @@ -42,15 +42,19 @@ func TestData(t *testing.T) { eventType string path string }{ - {"rcode_event", "./_meta/data_rcode_event.json"}, - {"panic_event", "./_meta/data_panic_event.json"}, + {"coredns.stats.panic.count.total", "./_meta/data_panic_event.json"}, + {"coredns.stats.dns.request.count.total", "./_meta/data_request_count_event.json"}, + {"coredns.stats.dns.request.size.bytes", "./_meta/data_request_size_bytes_event.json"}, + {"coredns.stats.dns.request.duration.sec", "./_meta/data_request_duration_sec_event.json"}, + {"coredns.stats.dns.response.rcode", "./_meta/data_response_rcode_event.json"}, + {"coredns.stats.dns.response.size.bytes", "./_meta/data_response_size_bytes_event.json"}, + {"coredns.stats.dns.request.type", "./_meta/data_request_type_event.json"}, } - f := mbtest.NewReportingMetricSetV2(t, getConfig()) for _, df := range dataFiles { t.Run(fmt.Sprintf("event type:%s", df.eventType), func(t *testing.T) { - err = mbtest.WriteEventsReporterV2Cond(f, t, df.path, eventIs(df.eventType)) + err := mbtest.WriteEventsReporterV2Cond(f, t, df.path, eventIs(df.eventType)) if err != nil { t.Fatal("write", err) } diff --git a/metricbeat/module/coredns/stats/stats_test.go b/metricbeat/module/coredns/stats/stats_test.go index 4bb76f1add9..5910f04981f 100644 --- a/metricbeat/module/coredns/stats/stats_test.go +++ b/metricbeat/module/coredns/stats/stats_test.go @@ -21,19 +21,17 @@ package stats import ( "testing" - - "github.com/elastic/beats/metricbeat/helper/prometheus/ptest" ) const testFile = "_meta/test/metrics" func TestEventMapping(t *testing.T) { - ptest.TestMetricSet(t, "coredns", "stats", - ptest.TestCases{ - { - MetricsFile: "./_meta/test/metrics", - ExpectedFile: "./_meta/test/metrics.expected", - }, - }, - ) + //ptest.TestMetricSet(t, "coredns", "stats", + // ptest.TestCases{ + // { + // MetricsFile: "./_meta/test/metrics", + // ExpectedFile: "./_meta/test/metrics.expected", + // }, + // }, + //) } From 07f2feb687689599d453c2a62fbba820c71f0c01 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 19 Feb 2019 09:45:55 +0200 Subject: [PATCH 13/24] Add system test Signed-off-by: Chris Mark --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 34 +++++++++++---- metricbeat/module/coredns/fields.go | 2 +- .../module/coredns/stats/_meta/fields.yml | 22 ++++++---- metricbeat/tests/system/test_coredns.py | 41 +++++++++++++++++++ 5 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 metricbeat/tests/system/test_coredns.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 005c8261dc5..472c3fcc9b8 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -465,6 +465,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Add more TCP statuses to `socket_summary` metricset. {pull}9430[9430] - Remove experimental tag from ceph metricsets. {pull}9708[9708] - Add MS SQL module to X-Pack {pull}9414[9414 +- Add `coredns` metricbeat {pull}10585{10585] ==== Deprecated diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ddfbaf4aa38..45e60a46ffe 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2609,26 +2609,44 @@ Total query count -- -*`coredns.stats.dns.request.duration.sec`*:: +*`coredns.stats.dns.request.duration.sec.bucket`*:: ++ +-- +type: object + +Request duration histogram buckets + + +-- + +*`coredns.stats.dns.request.duration.sec.sum`*:: + -- type: long format: duration -Duration to process each query +Requests duration, sum of durations -- -*`coredns.stats.dns.request.size.bytes`*:: +*`coredns.stats.dns.request.duration.sec.count`*:: + -- type: long -format: bytes +Requests duration, number of requests + -Size of requests in bytes +-- + +*`coredns.stats.dns.request.size.bytes`*:: ++ +-- +type: object + +Request Size histogram buckets -- @@ -2696,11 +2714,9 @@ The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version *`coredns.stats.dns.response.size.bytes`*:: + -- -type: long - -format: bytes +type: object -Response size in bytes +Response Size histogram buckets -- diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index 2e5f479743b..ae1712745c1 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJy8lU9P4zwQxu/9FI96otJLXu1qxSHS7gUOy2FZtPABcO0ptUjtMDMBhU+/cv5AiKIGpHZ9nMbP8/MzHvcUD1TnsJHJBVkA6rWgHMuuslwAjsSyL9XHkOPHAkD/PX5FVxW0AJgKMkI51qRmAWw8FU7y5uNTBLOjoUlaWpeU455jVXaVCZ+07rp9d7AxqPFBIGrUi3or0K1RPBMTmIzDhuOuMbq4uukUhihDnCQir9UpoD1QaZ1P4DAVRslBI3RLPQmE+MlbGmwfB9avMe2QuDTB28zGKmimUU3x7qv+BEUM96Mf9hyiWbdJDKHarYkRN62RTDK4IBnTY0WixyN5rIhrNPKzEK5ik+QyIftRik3kndEc/d5PQl5021KXS46WREDGblvuWWLxL5StayX5LO/UpjnYG/9Cqaedu8CHCZ3JZOMxOnz1estSWp66Ed6aJ2pG5uI31l4hNN/6xHAMRJwnzfeQJTFeYiCY4BrlD9FNEj1Q/RzZfQ7qZyycNAG1w5GkEl+qdI57iKSMQShjG90xEhvk1XuNEmucPwh4jNAa4be8WrvZFm7Mzhf14XhutwTjHKcHo9XukZRNkDKy4uQLvuPyGieX13gilvTOfFv9h69N+exd/Wy1ms/03z43+NP5Ivnuf2zSvyLxYeNtNfsW+3VB6TizY5Iu6mFBkmLSRiXkxgz/772CJUeNh6V5u1+jGcDJsnLlEpGxVFsuV4u/AQAA///NeLnx" + return "eJy0lk9P40gQxe/5FE85EQm82tWKg6XdCxyGwzAI+AC03RXSg91tqsqg8OlH7T8h8VgxMEkfy+73fn7VXckZnmidIg9M1ssMUKcFpZh3lfkMsCQ5u0pd8Cn+nwHo38f3YOuCZgBTQUYoRUZqZsDSUWElbV4+gzclbZvEpeuKUjxyqKuuMuIT10O37wF58GqcF4gadaIuF+jKKF6JCUzGYsmhbIwur+86hW2UbZwoIpvqGNAeqLguRnCYCqNkoQG6op4EQvzictraPgysX0PabeLKeJcneai9JhrUFDtv9V9QBP84eLDnI5p1H8Xg6zIjRli2RjLKYL0kTM81iR6P5LkmXqORn4SwNZsolwjlSVbnT6SjMCH7Sfnw0RTObWuC3gQrJxoe2ZRoraZD2uGTuvxoUsvApdF0Y/01ctnsP4XUZWxuX/gk+u/d+IM2j9C9n77OfppP3Bsl2VpJDt/xO/dGX+l2OMatuN5kE2+Go27srcwLNWPm8gcypxCavi6R4RiIuIiau5AVMd6CJxhvG+UP0Y0SPdH6NbD9HNS3UFhpAmoHSpSKfLHSOe4hkip4oYTzYI+R2FZevdcgscb5g4DHCK0Rfs+rtZts4dKUrlgfjud+RTDWMomg1e6RlI2XKrDi5G/8h6sbnFzd4IVY4qT+d3GKf5ry+U79fLGYzvQYkwW3nfpnZkv840B82DRbzb6jLiso/thM3op4Lg8LEhWjNmohO2T4a++JqzhoOCzN+3EaHHmczGtbzREYc82r+WL2KwAA//+nCxDn" } diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 733441e9d86..1e54b019cbf 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -12,16 +12,23 @@ type: long description: > Total query count - - name: dns.request.duration.sec + - name: dns.request.duration.sec.bucket + type: object + description: > + Request duration histogram buckets + - name: dns.request.duration.sec.sum type: long format: duration description: > - Duration to process each query - - name: dns.request.size.bytes + Requests duration, sum of durations + - name: dns.request.duration.sec.count type: long - format: bytes description: > - Size of requests in bytes + Requests duration, number of requests + - name: dns.request.size.bytes + type: object + description: > + Request Size histogram buckets - name: dns.request.do.count.total type: long description: > @@ -47,10 +54,9 @@ description: > The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) - name: dns.response.size.bytes - type: long - format: bytes + type: object description: > - Response size in bytes + Response Size histogram buckets - name: server type: keyword description: > diff --git a/metricbeat/tests/system/test_coredns.py b/metricbeat/tests/system/test_coredns.py new file mode 100644 index 00000000000..516da967011 --- /dev/null +++ b/metricbeat/tests/system/test_coredns.py @@ -0,0 +1,41 @@ +import os +import metricbeat +import unittest + +COREDNS_FIELDS = metricbeat.COMMON_FIELDS + ["coredns"] + + +class Test(metricbeat.BaseTest): + + COMPOSE_SERVICES = ['coredns'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_stats(self): + """ + coredns stats test + """ + self.render_config_template(modules=[{ + "name": "coredns", + "metricsets": ["stats"], + "hosts": self.get_hosts(), + "period": "5s", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertGreater(len(output), 0) + + for evt in output: + print("event is:") + print(evt) + self.assert_fields_are_documented(evt) + self.assertItemsEqual(self.de_dot(COREDNS_FIELDS), evt.keys(), evt) + + def get_hosts(self): + return ["{}:{}".format( + os.getenv('COREDNS_HOST', 'localhost'), + os.getenv('COREDNS_PORT', '9153') + )] From 7fba2670379134e83e08e7ca761358219f5c5439 Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Fri, 22 Feb 2019 10:54:08 +0200 Subject: [PATCH 14/24] Add metrics.expected and update unit test Signed-off-by: Ioannis Androulidakis --- .../coredns/stats/_meta/test/metrics.expected | 202 ++++++++++++++++++ metricbeat/module/coredns/stats/stats_test.go | 20 +- 2 files changed, 212 insertions(+), 10 deletions(-) create mode 100644 metricbeat/module/coredns/stats/_meta/test/metrics.expected diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics.expected b/metricbeat/module/coredns/stats/_meta/test/metrics.expected new file mode 100644 index 00000000000..0ceba0f2e9f --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/test/metrics.expected @@ -0,0 +1,202 @@ +[ + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "type": "A" + } + }, + "server": "dns://:53", + "zone": "mydomain.org." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "count": { + "total": 3940 + }, + "family": "1" + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "mydomain.org." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "duration": { + "sec": { + "bucket": { + "+Inf": 3940, + "0.00025": 3894, + "0.0005": 3928, + "0.001": 3939, + "0.002": 3939, + "0.004": 3939, + "0.008": 3940, + "0.016": 3940, + "0.032": 3940, + "0.064": 3940, + "0.128": 3940, + "0.256": 3940, + "0.512": 3940, + "1.024": 3940, + "2.048": 3940, + "4.096": 3940, + "8.192": 3940 + }, + "count": 3940, + "sum": 0.5743562010000005 + } + } + } + }, + "server": "dns://:53", + "zone": "mydomain.org." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "panic": { + "count": { + "total": 0 + } + } + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "response": { + "rcode": "NOERROR" + } + }, + "server": "dns://:53", + "zone": "mydomain.org." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "size": { + "bytes": { + "bucket": { + "+Inf": 3940, + "0": 0, + "100": 3940, + "1023": 3940, + "16000": 3940, + "200": 3940, + "2047": 3940, + "300": 3940, + "32000": 3940, + "400": 3940, + "4095": 3940, + "48000": 3940, + "511": 3940, + "64000": 3940, + "8291": 3940 + }, + "count": 3940, + "sum": 236400 + } + } + }, + "response": { + "size": { + "bytes": { + "bucket": { + "+Inf": 3940, + "0": 0, + "100": 3940, + "1023": 3940, + "16000": 3940, + "200": 3940, + "2047": 3940, + "300": 3940, + "32000": 3940, + "400": 3940, + "4095": 3940, + "48000": 3940, + "511": 3940, + "64000": 3940, + "8291": 3940 + }, + "count": 3940, + "sum": 283680 + } + } + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "mydomain.org." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + } +] \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/stats_test.go b/metricbeat/module/coredns/stats/stats_test.go index 5910f04981f..053ed67b17a 100644 --- a/metricbeat/module/coredns/stats/stats_test.go +++ b/metricbeat/module/coredns/stats/stats_test.go @@ -15,23 +15,23 @@ // specific language governing permissions and limitations // under the License. -// +build !integration - package stats import ( "testing" + + "github.com/elastic/beats/metricbeat/helper/prometheus/ptest" ) const testFile = "_meta/test/metrics" func TestEventMapping(t *testing.T) { - //ptest.TestMetricSet(t, "coredns", "stats", - // ptest.TestCases{ - // { - // MetricsFile: "./_meta/test/metrics", - // ExpectedFile: "./_meta/test/metrics.expected", - // }, - // }, - //) + ptest.TestMetricSet(t, "coredns", "stats", + ptest.TestCases{ + { + MetricsFile: "./_meta/test/metrics", + ExpectedFile: "./_meta/test/metrics.expected", + }, + }, + ) } From 3985f7b82f3c3ce2ba18f838e54967c16bd97bba Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Mon, 25 Feb 2019 12:02:27 +0200 Subject: [PATCH 15/24] Review changes Signed-off-by: Ioannis Androulidakis --- CHANGELOG.asciidoc | 1 - CHANGELOG.next.asciidoc | 1 + metricbeat/module/coredns/_meta/Dockerfile | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 472c3fcc9b8..005c8261dc5 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -465,7 +465,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Add more TCP statuses to `socket_summary` metricset. {pull}9430[9430] - Remove experimental tag from ceph metricsets. {pull}9708[9708] - Add MS SQL module to X-Pack {pull}9414[9414 -- Add `coredns` metricbeat {pull}10585{10585] ==== Deprecated diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 22b290743ec..9c35fc27c77 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -351,6 +351,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] - Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055] - Add `coredns` metricbeat {pull}10585{10585] +- Add `coredns` metricbeat module. {pull}10585{10585] *Packetbeat* diff --git a/metricbeat/module/coredns/_meta/Dockerfile b/metricbeat/module/coredns/_meta/Dockerfile index 3b507109e6d..1a7db8c45a6 100644 --- a/metricbeat/module/coredns/_meta/Dockerfile +++ b/metricbeat/module/coredns/_meta/Dockerfile @@ -17,8 +17,8 @@ ADD config /etc/coredns RUN apk add --update --no-cache bind-tools -#HEALTHCHECK --retries=10 CMD dig @0.0.0.0 www.elastic.com >/dev/null -HEALTHCHECK --interval=1s --retries=10 CMD dig @0.0.0.0 www.elastic.com >/dev/null +# Check if the Coredns container is healthy +HEALTHCHECK --interval=5s --retries=10 CMD dig @0.0.0.0 my.domain.org >/dev/null # Start coredns with custom configuration file ENTRYPOINT ["/coredns"] From a5063c106edf29ac7b0944d4dfd8680bf4943c80 Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Mon, 25 Feb 2019 12:03:25 +0200 Subject: [PATCH 16/24] Finalize format of metrics and events * Enable DO bit metrics with dnssec in DNS request * Merge request and response size bytes metric * Generate updated JSON files and fix integration test * Generate updated metrics and metrics.expected files Signed-off-by: Ioannis Androulidakis --- metricbeat/module/coredns/_meta/Dockerfile | 2 +- .../module/coredns/stats/_meta/data.json | 23 -- .../stats/_meta/data_request_count_event.json | 6 +- .../data_request_duration_sec_event.json | 27 +- .../_meta/data_request_size_bytes_event.json | 78 ------ .../stats/_meta/data_request_type_event.json | 7 +- .../_meta/data_response_rcode_event.json | 7 +- .../_meta/data_response_size_bytes_event.json | 78 ------ .../stats/_meta/data_size_bytes_event.json | 78 ++++++ .../module/coredns/stats/_meta/test/metrics | 233 +++++++++++------- .../coredns/stats/_meta/test/metrics.expected | 191 +++++++------- metricbeat/module/coredns/stats/stats.go | 6 +- .../coredns/stats/stats_integration_test.go | 3 +- 13 files changed, 361 insertions(+), 378 deletions(-) delete mode 100644 metricbeat/module/coredns/stats/_meta/data.json delete mode 100644 metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json delete mode 100644 metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json create mode 100644 metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json diff --git a/metricbeat/module/coredns/_meta/Dockerfile b/metricbeat/module/coredns/_meta/Dockerfile index 1a7db8c45a6..8e174cf779e 100644 --- a/metricbeat/module/coredns/_meta/Dockerfile +++ b/metricbeat/module/coredns/_meta/Dockerfile @@ -18,7 +18,7 @@ ADD config /etc/coredns RUN apk add --update --no-cache bind-tools # Check if the Coredns container is healthy -HEALTHCHECK --interval=5s --retries=10 CMD dig @0.0.0.0 my.domain.org >/dev/null +HEALTHCHECK --interval=5s --retries=10 CMD dig @0.0.0.0 my.domain.org +dnssec >/dev/null # Start coredns with custom configuration file ENTRYPOINT ["/coredns"] diff --git a/metricbeat/module/coredns/stats/_meta/data.json b/metricbeat/module/coredns/stats/_meta/data.json deleted file mode 100644 index 4ae76d5782d..00000000000 --- a/metricbeat/module/coredns/stats/_meta/data.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "@timestamp":"2016-05-23T08:05:34.853Z", - "beat":{ - "hostname":"beathost", - "name":"beathost" - }, - "metricset":{ - "host":"localhost", - "module":"coredns", - "name":"stats", - "rtt":44269 - }, - "coredns": { - "stats": { - "panic": { - "count": { - "total": 0 - } - } - } - }, - "type":"metricsets" -} diff --git a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json index c2586b26840..5ac4d15105a 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json @@ -9,11 +9,11 @@ "dns": { "request": { "count": { - "total": 2210 - }, - "family": "1" + "total": 2207 + } } }, + "family": "1", "proto": "udp", "server": "dns://:53", "zone": "." diff --git a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json index af5dae9b4c9..fb446f77f1b 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json @@ -8,10 +8,15 @@ "stats": { "dns": { "request": { + "do": { + "count": { + "total": 1880 + } + }, "duration": { "sec": { "bucket": { - "+Inf": 2210, + "+Inf": 2207, "0.00025": 0, "0.0005": 0, "0.001": 0, @@ -20,17 +25,17 @@ "0.008": 0, "0.016": 0, "0.032": 0, - "0.064": 0, - "0.128": 2200, - "0.256": 2210, - "0.512": 2210, - "1.024": 2210, - "2.048": 2210, - "4.096": 2210, - "8.192": 2210 + "0.064": 1931, + "0.128": 2193, + "0.256": 2205, + "0.512": 2207, + "1.024": 2207, + "2.048": 2207, + "4.096": 2207, + "8.192": 2207 }, - "count": 2210, - "sum": 173.6070170089998 + "count": 2207, + "sum": 119.93155948800005 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json deleted file mode 100644 index 6b8dd871b19..00000000000 --- a/metricbeat/module/coredns/stats/_meta/data_request_size_bytes_event.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "coredns": { - "stats": { - "dns": { - "request": { - "size": { - "bytes": { - "bucket": { - "+Inf": 2210, - "0": 0, - "100": 2210, - "1023": 2210, - "16000": 2210, - "200": 2210, - "2047": 2210, - "300": 2210, - "32000": 2210, - "400": 2210, - "4095": 2210, - "48000": 2210, - "511": 2210, - "64000": 2210, - "8291": 2210 - }, - "count": 2210, - "sum": 123760 - } - } - }, - "response": { - "size": { - "bytes": { - "bucket": { - "+Inf": 2210, - "0": 0, - "100": 0, - "1023": 2210, - "16000": 2210, - "200": 2210, - "2047": 2210, - "300": 2210, - "32000": 2210, - "400": 2210, - "4095": 2210, - "48000": 2210, - "511": 2210, - "64000": 2210, - "8291": 2210 - }, - "count": 2210, - "sum": 245310 - } - } - } - }, - "proto": "udp", - "server": "dns://:53", - "zone": "." - } - }, - "event": { - "dataset": "coredns.stats", - "duration": 115000, - "module": "coredns" - }, - "metricset": { - "name": "stats" - }, - "service": { - "address": "127.0.0.1:9153", - "type": "coredns" - } -} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json index fedc70fd19d..65291fde75a 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json @@ -8,10 +8,15 @@ "stats": { "dns": { "request": { - "type": "A" + "type": { + "count": { + "total": 2207 + } + } } }, "server": "dns://:53", + "type": "A", "zone": "." } }, diff --git a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json index a02ff606a21..180e8276547 100644 --- a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json @@ -8,9 +8,14 @@ "stats": { "dns": { "response": { - "rcode": "NOERROR" + "rcode": { + "count": { + "total": 2207 + } + } } }, + "rcode": "NOERROR", "server": "dns://:53", "zone": "." } diff --git a/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json deleted file mode 100644 index 6b8dd871b19..00000000000 --- a/metricbeat/module/coredns/stats/_meta/data_response_size_bytes_event.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "coredns": { - "stats": { - "dns": { - "request": { - "size": { - "bytes": { - "bucket": { - "+Inf": 2210, - "0": 0, - "100": 2210, - "1023": 2210, - "16000": 2210, - "200": 2210, - "2047": 2210, - "300": 2210, - "32000": 2210, - "400": 2210, - "4095": 2210, - "48000": 2210, - "511": 2210, - "64000": 2210, - "8291": 2210 - }, - "count": 2210, - "sum": 123760 - } - } - }, - "response": { - "size": { - "bytes": { - "bucket": { - "+Inf": 2210, - "0": 0, - "100": 0, - "1023": 2210, - "16000": 2210, - "200": 2210, - "2047": 2210, - "300": 2210, - "32000": 2210, - "400": 2210, - "4095": 2210, - "48000": 2210, - "511": 2210, - "64000": 2210, - "8291": 2210 - }, - "count": 2210, - "sum": 245310 - } - } - } - }, - "proto": "udp", - "server": "dns://:53", - "zone": "." - } - }, - "event": { - "dataset": "coredns.stats", - "duration": 115000, - "module": "coredns" - }, - "metricset": { - "name": "stats" - }, - "service": { - "address": "127.0.0.1:9153", - "type": "coredns" - } -} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json new file mode 100644 index 00000000000..cb7f49d3dc0 --- /dev/null +++ b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json @@ -0,0 +1,78 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "coredns": { + "stats": { + "dns": { + "request": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2207, + "0": 0, + "100": 2207, + "1023": 2207, + "16000": 2207, + "200": 2207, + "2047": 2207, + "300": 2207, + "32000": 2207, + "400": 2207, + "4095": 2207, + "48000": 2207, + "511": 2207, + "64000": 2207, + "8291": 2207 + }, + "count": 2207, + "sum": 114764 + } + } + }, + "response": { + "size": { + "bytes": { + "bucket": { + "+Inf": 2207, + "0": 0, + "100": 2207, + "1023": 2207, + "16000": 2207, + "200": 2207, + "2047": 2207, + "300": 2207, + "32000": 2207, + "400": 2207, + "4095": 2207, + "48000": 2207, + "511": 2207, + "64000": 2207, + "8291": 2207 + }, + "count": 2207, + "sum": 147869 + } + } + } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "." + } + }, + "event": { + "dataset": "coredns.stats", + "duration": 115000, + "module": "coredns" + }, + "metricset": { + "name": "stats" + }, + "service": { + "address": "127.0.0.1:9153", + "type": "coredns" + } +} \ No newline at end of file diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics b/metricbeat/module/coredns/stats/_meta/test/metrics index 3628f4377da..18b1e18df81 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics +++ b/metricbeat/module/coredns/stats/_meta/test/metrics @@ -3,135 +3,190 @@ coredns_build_info{goversion="go1.11.4",revision="6b56a9c",version="1.3.1"} 1.0 # HELP coredns_dns_request_count_total Counter of DNS requests made per zone, protocol and family. # TYPE coredns_dns_request_count_total counter -coredns_dns_request_count_total{family="1",proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 +coredns_dns_request_count_total{family="1",proto="udp",server="dns://:53",zone="."} 440.0 +# HELP coredns_dns_request_do_count_total Counter of DNS requests with DO bit set per zone. +# TYPE coredns_dns_request_do_count_total counter +coredns_dns_request_do_count_total{server="dns://:53",zone="."} 113.0 # HELP coredns_dns_request_duration_seconds Histogram of the time (in seconds) each request took. # TYPE coredns_dns_request_duration_seconds histogram -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.00025"} 3894.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.0005"} 3928.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.001"} 3939.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.002"} 3939.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.004"} 3939.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.008"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.016"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.032"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.064"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.128"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.256"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="0.512"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="1.024"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="2.048"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="4.096"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="8.192"} 3940.0 -coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 -coredns_dns_request_duration_seconds_sum{server="dns://:53",zone="mydomain.org."} 0.5743562010000005 -coredns_dns_request_duration_seconds_count{server="dns://:53",zone="mydomain.org."} 3940.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.00025"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.0005"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.001"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.002"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.004"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.008"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.016"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.032"} 0.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.064"} 424.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.128"} 437.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.256"} 438.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="0.512"} 440.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="1.024"} 440.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="2.048"} 440.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="4.096"} 440.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="8.192"} 440.0 +coredns_dns_request_duration_seconds_bucket{server="dns://:53",zone=".",le="+Inf"} 440.0 +coredns_dns_request_duration_seconds_sum{server="dns://:53",zone="."} 21.712619368 +coredns_dns_request_duration_seconds_count{server="dns://:53",zone="."} 440.0 # HELP coredns_dns_request_size_bytes Size of the EDNS0 UDP buffer in bytes (64K for TCP). # TYPE coredns_dns_request_size_bytes histogram -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="0.0"} 0.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="100.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="200.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="300.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="400.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="511.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="1023.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="2047.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="4095.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="8291.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="16000.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="32000.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="48000.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="64000.0"} 3940.0 -coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 -coredns_dns_request_size_bytes_sum{proto="udp",server="dns://:53",zone="mydomain.org."} 236400.0 -coredns_dns_request_size_bytes_count{proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="0.0"} 0.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="100.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="200.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="300.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="400.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="511.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="1023.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="2047.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="4095.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="8291.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="16000.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="32000.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="48000.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="64000.0"} 440.0 +coredns_dns_request_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="+Inf"} 440.0 +coredns_dns_request_size_bytes_sum{proto="udp",server="dns://:53",zone="."} 22880.0 +coredns_dns_request_size_bytes_count{proto="udp",server="dns://:53",zone="."} 440.0 # HELP coredns_dns_request_type_count_total Counter of DNS requests per type, per zone. # TYPE coredns_dns_request_type_count_total counter -coredns_dns_request_type_count_total{server="dns://:53",type="A",zone="mydomain.org."} 3940.0 +coredns_dns_request_type_count_total{server="dns://:53",type="A",zone="."} 440.0 # HELP coredns_dns_response_rcode_count_total Counter of response status codes. # TYPE coredns_dns_response_rcode_count_total counter -coredns_dns_response_rcode_count_total{rcode="NOERROR",server="dns://:53",zone="mydomain.org."} 3940.0 +coredns_dns_response_rcode_count_total{rcode="NOERROR",server="dns://:53",zone="."} 440.0 # HELP coredns_dns_response_size_bytes Size of the returned response in bytes. # TYPE coredns_dns_response_size_bytes histogram -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="0.0"} 0.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="100.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="200.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="300.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="400.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="511.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="1023.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="2047.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="4095.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="8291.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="16000.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="32000.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="48000.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="64000.0"} 3940.0 -coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone="mydomain.org.",le="+Inf"} 3940.0 -coredns_dns_response_size_bytes_sum{proto="udp",server="dns://:53",zone="mydomain.org."} 283680.0 -coredns_dns_response_size_bytes_count{proto="udp",server="dns://:53",zone="mydomain.org."} 3940.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="0.0"} 0.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="100.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="200.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="300.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="400.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="511.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="1023.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="2047.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="4095.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="8291.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="16000.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="32000.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="48000.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="64000.0"} 440.0 +coredns_dns_response_size_bytes_bucket{proto="udp",server="dns://:53",zone=".",le="+Inf"} 440.0 +coredns_dns_response_size_bytes_sum{proto="udp",server="dns://:53",zone="."} 29480.0 +coredns_dns_response_size_bytes_count{proto="udp",server="dns://:53",zone="."} 440.0 +# HELP coredns_forward_request_count_total Counter of requests made per upstream. +# TYPE coredns_forward_request_count_total counter +coredns_forward_request_count_total{to="8.8.4.4:53"} 200.0 +coredns_forward_request_count_total{to="8.8.8.8:53"} 240.0 +# HELP coredns_forward_request_duration_seconds Histogram of the time each request took. +# TYPE coredns_forward_request_duration_seconds histogram +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.00025"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.0005"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.001"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.002"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.004"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.008"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.016"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.032"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.064"} 191.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.128"} 198.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.256"} 199.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="0.512"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="1.024"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="2.048"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="4.096"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="8.192"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.4.4:53",le="+Inf"} 200.0 +coredns_forward_request_duration_seconds_sum{to="8.8.4.4:53"} 9.960059515000003 +coredns_forward_request_duration_seconds_count{to="8.8.4.4:53"} 200.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.00025"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.0005"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.001"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.002"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.004"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.008"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.016"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.032"} 0.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.064"} 233.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.128"} 239.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.256"} 239.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="0.512"} 240.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="1.024"} 240.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="2.048"} 240.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="4.096"} 240.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="8.192"} 240.0 +coredns_forward_request_duration_seconds_bucket{to="8.8.8.8:53",le="+Inf"} 240.0 +coredns_forward_request_duration_seconds_sum{to="8.8.8.8:53"} 11.636399849999993 +coredns_forward_request_duration_seconds_count{to="8.8.8.8:53"} 240.0 +# HELP coredns_forward_response_rcode_count_total Counter of requests made per upstream. +# TYPE coredns_forward_response_rcode_count_total counter +coredns_forward_response_rcode_count_total{rcode="NOERROR",to="8.8.4.4:53"} 200.0 +coredns_forward_response_rcode_count_total{rcode="NOERROR",to="8.8.8.8:53"} 240.0 +# HELP coredns_forward_sockets_open Gauge of open sockets per upstream. +# TYPE coredns_forward_sockets_open gauge +coredns_forward_sockets_open{to="8.8.4.4:53"} 2.0 +coredns_forward_sockets_open{to="8.8.8.8:53"} 1.0 # HELP coredns_panic_count_total A metrics that counts the number of panics. # TYPE coredns_panic_count_total counter coredns_panic_count_total 0.0 # HELP go_gc_duration_seconds A summary of the GC invocation durations. # TYPE go_gc_duration_seconds summary -go_gc_duration_seconds{quantile="0.0"} 4.2171e-05 -go_gc_duration_seconds{quantile="0.25"} 4.8407e-05 -go_gc_duration_seconds{quantile="0.5"} 6.3342e-05 -go_gc_duration_seconds{quantile="0.75"} 8.6721e-05 -go_gc_duration_seconds{quantile="1.0"} 0.000379492 -go_gc_duration_seconds_sum 0.001787514 -go_gc_duration_seconds_count 21.0 +go_gc_duration_seconds{quantile="0.0"} 4.1132e-05 +go_gc_duration_seconds{quantile="0.25"} 8.4991e-05 +go_gc_duration_seconds{quantile="0.5"} 0.00012054 +go_gc_duration_seconds{quantile="0.75"} 0.000154605 +go_gc_duration_seconds{quantile="1.0"} 0.003588725 +go_gc_duration_seconds_sum 0.053824534 +go_gc_duration_seconds_count 398.0 # HELP go_goroutines Number of goroutines that currently exist. # TYPE go_goroutines gauge -go_goroutines 20.0 +go_goroutines 22.0 # HELP go_info Information about the Go environment. # TYPE go_info gauge go_info{version="go1.11.4"} 1.0 # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. # TYPE go_memstats_alloc_bytes gauge -go_memstats_alloc_bytes 2.691304e+06 +go_memstats_alloc_bytes 2.9178e+06 # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. # TYPE go_memstats_alloc_bytes_total counter -go_memstats_alloc_bytes_total 3.5058456e+07 +go_memstats_alloc_bytes_total 6.3592992e+07 # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. # TYPE go_memstats_buck_hash_sys_bytes gauge -go_memstats_buck_hash_sys_bytes 1.455718e+06 +go_memstats_buck_hash_sys_bytes 1.453846e+06 # HELP go_memstats_frees_total Total number of frees. # TYPE go_memstats_frees_total counter -go_memstats_frees_total 457516.0 +go_memstats_frees_total 295524.0 # HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. # TYPE go_memstats_gc_cpu_fraction gauge -go_memstats_gc_cpu_fraction 5.612791592054919e-06 +go_memstats_gc_cpu_fraction 7.482469694306849e-06 # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. # TYPE go_memstats_gc_sys_bytes gauge -go_memstats_gc_sys_bytes 2.379776e+06 +go_memstats_gc_sys_bytes 2.371584e+06 # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. # TYPE go_memstats_heap_alloc_bytes gauge -go_memstats_heap_alloc_bytes 2.691304e+06 +go_memstats_heap_alloc_bytes 2.9178e+06 # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. # TYPE go_memstats_heap_idle_bytes gauge -go_memstats_heap_idle_bytes 6.1980672e+07 +go_memstats_heap_idle_bytes 6.1497344e+07 # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. # TYPE go_memstats_heap_inuse_bytes gauge -go_memstats_heap_inuse_bytes 4.210688e+06 +go_memstats_heap_inuse_bytes 4.661248e+06 # HELP go_memstats_heap_objects Number of allocated objects. # TYPE go_memstats_heap_objects gauge -go_memstats_heap_objects 14109.0 +go_memstats_heap_objects 16744.0 # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. # TYPE go_memstats_heap_released_bytes gauge -go_memstats_heap_released_bytes 6.1341696e+07 +go_memstats_heap_released_bytes 6.1308928e+07 # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. # TYPE go_memstats_heap_sys_bytes gauge -go_memstats_heap_sys_bytes 6.619136e+07 +go_memstats_heap_sys_bytes 6.6158592e+07 # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. # TYPE go_memstats_last_gc_time_seconds gauge -go_memstats_last_gc_time_seconds 1.549753511946267e+09 +go_memstats_last_gc_time_seconds 1.5511302733947928e+09 # HELP go_memstats_lookups_total Total number of pointer lookups. # TYPE go_memstats_lookups_total counter go_memstats_lookups_total 0.0 # HELP go_memstats_mallocs_total Total number of mallocs. # TYPE go_memstats_mallocs_total counter -go_memstats_mallocs_total 471625.0 +go_memstats_mallocs_total 312268.0 # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. # TYPE go_memstats_mcache_inuse_bytes gauge go_memstats_mcache_inuse_bytes 6912.0 @@ -140,43 +195,43 @@ go_memstats_mcache_inuse_bytes 6912.0 go_memstats_mcache_sys_bytes 16384.0 # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. # TYPE go_memstats_mspan_inuse_bytes gauge -go_memstats_mspan_inuse_bytes 59888.0 +go_memstats_mspan_inuse_bytes 68096.0 # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. # TYPE go_memstats_mspan_sys_bytes gauge -go_memstats_mspan_sys_bytes 98304.0 +go_memstats_mspan_sys_bytes 81920.0 # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. # TYPE go_memstats_next_gc_bytes gauge -go_memstats_next_gc_bytes 4.767168e+06 +go_memstats_next_gc_bytes 4.950384e+06 # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. # TYPE go_memstats_other_sys_bytes gauge -go_memstats_other_sys_bytes 1.225362e+06 +go_memstats_other_sys_bytes 1.25181e+06 # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. # TYPE go_memstats_stack_inuse_bytes gauge -go_memstats_stack_inuse_bytes 917504.0 +go_memstats_stack_inuse_bytes 950272.0 # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. # TYPE go_memstats_stack_sys_bytes gauge -go_memstats_stack_sys_bytes 917504.0 +go_memstats_stack_sys_bytes 950272.0 # HELP go_memstats_sys_bytes Number of bytes obtained from system. # TYPE go_memstats_sys_bytes gauge go_memstats_sys_bytes 7.2284408e+07 # HELP go_threads Number of OS threads created. # TYPE go_threads gauge -go_threads 17.0 +go_threads 16.0 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter -process_cpu_seconds_total 2.13 +process_cpu_seconds_total 19.48 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 1.048576e+06 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge -process_open_fds 9.0 +process_open_fds 12.0 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge -process_resident_memory_bytes 2.7910144e+07 +process_resident_memory_bytes 1.507328e+07 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge -process_start_time_seconds 1.54975119258e+09 +process_start_time_seconds 1.55103964665e+09 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 1.462272e+08 diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics.expected b/metricbeat/module/coredns/stats/_meta/test/metrics.expected index 0ceba0f2e9f..4e9cb4a7923 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics.expected +++ b/metricbeat/module/coredns/stats/_meta/test/metrics.expected @@ -4,12 +4,17 @@ "ModuleFields": null, "MetricSetFields": { "dns": { - "request": { - "type": "A" + "response": { + "rcode": { + "count": { + "total": 440 + } + } } }, + "rcode": "NOERROR", "server": "dns://:53", - "zone": "mydomain.org." + "zone": "." }, "Index": "", "ID": "", @@ -24,17 +29,11 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "dns": { - "request": { - "count": { - "total": 3940 - }, - "family": "1" + "panic": { + "count": { + "total": 0 } - }, - "proto": "udp", - "server": "dns://:53", - "zone": "mydomain.org." + } }, "Index": "", "ID": "", @@ -51,35 +50,40 @@ "MetricSetFields": { "dns": { "request": { + "do": { + "count": { + "total": 113 + } + }, "duration": { "sec": { "bucket": { - "+Inf": 3940, - "0.00025": 3894, - "0.0005": 3928, - "0.001": 3939, - "0.002": 3939, - "0.004": 3939, - "0.008": 3940, - "0.016": 3940, - "0.032": 3940, - "0.064": 3940, - "0.128": 3940, - "0.256": 3940, - "0.512": 3940, - "1.024": 3940, - "2.048": 3940, - "4.096": 3940, - "8.192": 3940 + "+Inf": 440, + "0.00025": 0, + "0.0005": 0, + "0.001": 0, + "0.002": 0, + "0.004": 0, + "0.008": 0, + "0.016": 0, + "0.032": 0, + "0.064": 424, + "0.128": 437, + "0.256": 438, + "0.512": 440, + "1.024": 440, + "2.048": 440, + "4.096": 440, + "8.192": 440 }, - "count": 3940, - "sum": 0.5743562010000005 + "count": 440, + "sum": 21.712619368 } } } }, "server": "dns://:53", - "zone": "mydomain.org." + "zone": "." }, "Index": "", "ID": "", @@ -94,11 +98,61 @@ "RootFields": null, "ModuleFields": null, "MetricSetFields": { - "panic": { - "count": { - "total": 0 + "dns": { + "request": { + "size": { + "bytes": { + "bucket": { + "+Inf": 440, + "0": 0, + "100": 440, + "1023": 440, + "16000": 440, + "200": 440, + "2047": 440, + "300": 440, + "32000": 440, + "400": 440, + "4095": 440, + "48000": 440, + "511": 440, + "64000": 440, + "8291": 440 + }, + "count": 440, + "sum": 22880 + } + } + }, + "response": { + "size": { + "bytes": { + "bucket": { + "+Inf": 440, + "0": 0, + "100": 440, + "1023": 440, + "16000": 440, + "200": 440, + "2047": 440, + "300": 440, + "32000": 440, + "400": 440, + "4095": 440, + "48000": 440, + "511": 440, + "64000": 440, + "8291": 440 + }, + "count": 440, + "sum": 29480 + } + } } - } + }, + "proto": "udp", + "server": "dns://:53", + "zone": "." }, "Index": "", "ID": "", @@ -114,12 +168,16 @@ "ModuleFields": null, "MetricSetFields": { "dns": { - "response": { - "rcode": "NOERROR" + "request": { + "count": { + "total": 440 + } } }, + "family": "1", + "proto": "udp", "server": "dns://:53", - "zone": "mydomain.org." + "zone": "." }, "Index": "", "ID": "", @@ -136,59 +194,16 @@ "MetricSetFields": { "dns": { "request": { - "size": { - "bytes": { - "bucket": { - "+Inf": 3940, - "0": 0, - "100": 3940, - "1023": 3940, - "16000": 3940, - "200": 3940, - "2047": 3940, - "300": 3940, - "32000": 3940, - "400": 3940, - "4095": 3940, - "48000": 3940, - "511": 3940, - "64000": 3940, - "8291": 3940 - }, - "count": 3940, - "sum": 236400 - } - } - }, - "response": { - "size": { - "bytes": { - "bucket": { - "+Inf": 3940, - "0": 0, - "100": 3940, - "1023": 3940, - "16000": 3940, - "200": 3940, - "2047": 3940, - "300": 3940, - "32000": 3940, - "400": 3940, - "4095": 3940, - "48000": 3940, - "511": 3940, - "64000": 3940, - "8291": 3940 - }, - "count": 3940, - "sum": 283680 + "type": { + "count": { + "total": 440 } } } }, - "proto": "udp", "server": "dns://:53", - "zone": "mydomain.org." + "type": "A", + "zone": "." }, "Index": "", "ID": "", diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index 45bba33aa6b..0ff9a90ad0c 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -37,10 +37,10 @@ var mapping = &prometheus.MetricsMapping{ Labels: map[string]prometheus.LabelMap{ "server": prometheus.KeyLabel("server"), "zone": prometheus.KeyLabel("zone"), - "type": prometheus.KeyLabel("dns.request.type"), - "rcode": prometheus.KeyLabel("dns.response.rcode"), + "type": prometheus.KeyLabel("type"), + "rcode": prometheus.KeyLabel("rcode"), "proto": prometheus.KeyLabel("proto"), - "family": prometheus.KeyLabel("dns.request.family"), + "family": prometheus.KeyLabel("family"), }, } diff --git a/metricbeat/module/coredns/stats/stats_integration_test.go b/metricbeat/module/coredns/stats/stats_integration_test.go index b74c6110517..61bd3eb6fd3 100644 --- a/metricbeat/module/coredns/stats/stats_integration_test.go +++ b/metricbeat/module/coredns/stats/stats_integration_test.go @@ -44,10 +44,9 @@ func TestData(t *testing.T) { }{ {"coredns.stats.panic.count.total", "./_meta/data_panic_event.json"}, {"coredns.stats.dns.request.count.total", "./_meta/data_request_count_event.json"}, - {"coredns.stats.dns.request.size.bytes", "./_meta/data_request_size_bytes_event.json"}, + {"coredns.stats.dns.request.size.bytes", "./_meta/data_size_bytes_event.json"}, {"coredns.stats.dns.request.duration.sec", "./_meta/data_request_duration_sec_event.json"}, {"coredns.stats.dns.response.rcode", "./_meta/data_response_rcode_event.json"}, - {"coredns.stats.dns.response.size.bytes", "./_meta/data_response_size_bytes_event.json"}, {"coredns.stats.dns.request.type", "./_meta/data_request_type_event.json"}, } f := mbtest.NewReportingMetricSetV2(t, getConfig()) From 63ada5216dfd4ed19c7e90bc0e32126a3a3ae348 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 26 Feb 2019 00:30:56 +0200 Subject: [PATCH 17/24] Update docs Signed-off-by: Chris --- metricbeat/docs/fields.asciidoc | 52 ++++++++++++++++--- .../docs/modules/coredns/stats.asciidoc | 6 --- metricbeat/docs/modules_list.asciidoc | 13 ++--- metricbeat/module/coredns/fields.go | 2 +- .../module/coredns/stats/_meta/fields.yml | 31 ++++++++--- metricbeat/tests/system/test_coredns.py | 2 - 6 files changed, 79 insertions(+), 27 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 45e60a46ffe..8c9fe3616dc 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2609,7 +2609,7 @@ Total query count -- -*`coredns.stats.dns.request.duration.sec.bucket`*:: +*`coredns.stats.dns.request.duration.sec.bucket.*`*:: + -- type: object @@ -2641,7 +2641,7 @@ Requests duration, number of requests -- -*`coredns.stats.dns.request.size.bytes`*:: +*`coredns.stats.dns.request.size.bytes.bucket.*`*:: + -- type: object @@ -2649,6 +2649,26 @@ type: object Request Size histogram buckets +-- + +*`coredns.stats.dns.request.size.bytes.sum`*:: ++ +-- +type: double + +Request Size histogram sum + + +-- + +*`coredns.stats.dns.request.size.bytes.count`*:: ++ +-- +type: double + +Request Size histogram count + + -- *`coredns.stats.dns.request.do.count.total`*:: @@ -2671,7 +2691,7 @@ Counter of queries per zone and type -- -*`coredns.stats.dns.request.type`*:: +*`coredns.stats.type`*:: + -- type: keyword @@ -2691,7 +2711,7 @@ Counter of responses per zone and rcode -- -*`coredns.stats.dns.response.rcode`*:: +*`coredns.stats.rcode`*:: + -- type: keyword @@ -2701,7 +2721,7 @@ Holds the rcode of the response -- -*`coredns.stats.dns.request.family`*:: +*`coredns.stats.family`*:: + -- type: keyword @@ -2711,7 +2731,7 @@ The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version -- -*`coredns.stats.dns.response.size.bytes`*:: +*`coredns.stats.dns.response.size.bytes.bucket.*`*:: + -- type: object @@ -2719,6 +2739,26 @@ type: object Response Size histogram buckets +-- + +*`coredns.stats.dns.response.size.bytes.sum`*:: ++ +-- +type: double + +Response Size histogram sum + + +-- + +*`coredns.stats.dns.response.size.bytes.count`*:: ++ +-- +type: double + +Response Size histogram count + + -- *`coredns.stats.server`*:: diff --git a/metricbeat/docs/modules/coredns/stats.asciidoc b/metricbeat/docs/modules/coredns/stats.asciidoc index 95ec886d6d7..780b80f7ec7 100644 --- a/metricbeat/docs/modules/coredns/stats.asciidoc +++ b/metricbeat/docs/modules/coredns/stats.asciidoc @@ -15,9 +15,3 @@ include::../../../module/coredns/stats/_meta/docs.asciidoc[] For a description of each field in the metricset, see the <> section. -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../module/coredns/stats/_meta/data.json[] ----- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 66d8188c17a..0f50974ead2 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -4,8 +4,8 @@ This file is generated! See scripts/docs_collector.py [options="header"] |=================================== -|Modules |Dashboards |Metricsets -|<> |image:./images/icon-no.png[No prebuilt dashboards] | +|Modules |Dashboards |Metricsets +|<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> @@ -23,10 +23,10 @@ This file is generated! See scripts/docs_collector.py |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] +.1+| .1+| |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] -|<> |image:./images/icon-no.png[No prebuilt dashboards] | +|<> |image:./images/icon-no.png[No prebuilt dashboards] | .3+| .3+| |<> |<> |<> @@ -168,8 +168,9 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.2+| .2+| |<> -|<> +.2+| .2+| |<> +|<> +|================================ -- diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index ae1712745c1..d191c137fbc 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJy0lk9P40gQxe/5FE85EQm82tWKg6XdCxyGwzAI+AC03RXSg91tqsqg8OlH7T8h8VgxMEkfy+73fn7VXckZnmidIg9M1ssMUKcFpZh3lfkMsCQ5u0pd8Cn+nwHo38f3YOuCZgBTQUYoRUZqZsDSUWElbV4+gzclbZvEpeuKUjxyqKuuMuIT10O37wF58GqcF4gadaIuF+jKKF6JCUzGYsmhbIwur+86hW2UbZwoIpvqGNAeqLguRnCYCqNkoQG6op4EQvzictraPgysX0PabeLKeJcneai9JhrUFDtv9V9QBP84eLDnI5p1H8Xg6zIjRli2RjLKYL0kTM81iR6P5LkmXqORn4SwNZsolwjlSVbnT6SjMCH7Sfnw0RTObWuC3gQrJxoe2ZRoraZD2uGTuvxoUsvApdF0Y/01ctnsP4XUZWxuX/gk+u/d+IM2j9C9n77OfppP3Bsl2VpJDt/xO/dGX+l2OMatuN5kE2+Go27srcwLNWPm8gcypxCavi6R4RiIuIiau5AVMd6CJxhvG+UP0Y0SPdH6NbD9HNS3UFhpAmoHSpSKfLHSOe4hkip4oYTzYI+R2FZevdcgscb5g4DHCK0Rfs+rtZts4dKUrlgfjud+RTDWMomg1e6RlI2XKrDi5G/8h6sbnFzd4IVY4qT+d3GKf5ry+U79fLGYzvQYkwW3nfpnZkv840B82DRbzb6jLiso/thM3op4Lg8LEhWjNmohO2T4a++JqzhoOCzN+3EaHHmczGtbzREYc82r+WL2KwAA//+nCxDn" + return "eJzElk9T2zAQxe/5FG9yIh1wp50Oh8y0FziUQykD3ItsbYiKLRntGiZ8+o78JySOExPqDDqupLc/rVbPPsEDLaZInCdteQSIkZSmGNeR8QjQxIk3uRhnp/gxAtCsxy+ni5RGgKeUFNMUMYkaATNDqeZpufgEVmW0miQMWeQ0xb13RV5HOvKEcVfvu0PirChjGSxKDItJGDJXgmfyBE9KY+ZdViY6v7ypFVZRVnGCCC+jXUA7oMI468DxlCohDXGQOTUkYPJPJqGV7e2CNaNNu0qcK2uSKHGFlUicqHRtVXOC1Nn71sSOQ5TjNojBFllMHm5WJeJOBm058vRYEMvhSB4L8guU8r0QuvAqyEVMSRQXyQNJ9KkTx8V/KZHWVBX8U63QrohT2hP5ugJBA4K5YXH3XmWocPoLuXYGLrK3VnPmfKZkukz9PnJe7j8GF1logCawJ/rmjf1HK3TQvXZonb6fj80LRfFCiD+0OW7MC72jMVbot7XFe7C2ULVT9BBtv+0Bmd5oAu4QZnS5bLdgSIbqr81cPVHp7ue/ERsBUz9gYDgEIs6C5jpkTh4vzhKU1aVyJ93GREPxQItn5/V+ID9dqrksSuXdQSowhUhdgx014txZpsgnTh+iSis1anK1qlRm7gTcnBmoTqXwa4kqrE6EmcpMuhiO4XZOUFp7Yq61GwzxynLuvODoC77j4gpHF1d4Is/hu/ZtcoyvZfh0LX46mfTf7Qf7MK5rkP2dePMAw1pxN9duL95kGtqMu6m2u3H4wyU/bJNWms3jMHFK4Y+n11PCqx4WJCgGbRRMus3weefjzb0TNyzN6yttuQeOxoXOx3AeY0ny8WT0LwAA//+tuceM" } diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 1e54b019cbf..8eb2d60ab08 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -12,8 +12,9 @@ type: long description: > Total query count - - name: dns.request.duration.sec.bucket + - name: dns.request.duration.sec.bucket.* type: object + object_type: double description: > Request duration histogram buckets - name: dns.request.duration.sec.sum @@ -25,10 +26,19 @@ type: long description: > Requests duration, number of requests - - name: dns.request.size.bytes + - name: dns.request.size.bytes.bucket.* type: object + object_type: double description: > Request Size histogram buckets + - name: dns.request.size.bytes.sum + type: double + description: > + Request Size histogram sum + - name: dns.request.size.bytes.count + type: double + description: > + Request Size histogram count - name: dns.request.do.count.total type: long description: > @@ -37,7 +47,7 @@ type: long description: > Counter of queries per zone and type - - name: dns.request.type + - name: type type: keyword description: > Holds the query type of the request @@ -45,18 +55,27 @@ type: long description: > Counter of responses per zone and rcode - - name: dns.response.rcode + - name: rcode type: keyword description: > Holds the rcode of the response - - name: dns.request.family + - name: family type: keyword description: > The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) - - name: dns.response.size.bytes + - name: dns.response.size.bytes.bucket.* type: object + object_type: double description: > Response Size histogram buckets + - name: dns.response.size.bytes.sum + type: double + description: > + Response Size histogram sum + - name: dns.response.size.bytes.count + type: double + description: > + Response Size histogram count - name: server type: keyword description: > diff --git a/metricbeat/tests/system/test_coredns.py b/metricbeat/tests/system/test_coredns.py index 516da967011..ac3a03d4328 100644 --- a/metricbeat/tests/system/test_coredns.py +++ b/metricbeat/tests/system/test_coredns.py @@ -29,8 +29,6 @@ def test_stats(self): self.assertGreater(len(output), 0) for evt in output: - print("event is:") - print(evt) self.assert_fields_are_documented(evt) self.assertItemsEqual(self.de_dot(COREDNS_FIELDS), evt.keys(), evt) From c58060fa46a6f8f52177fc10fc0b3e82aafadf3f Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 28 Feb 2019 12:13:30 +0200 Subject: [PATCH 18/24] Change Prometheus helper to scale double precision histogram keys Signed-off-by: Chris Mark --- metricbeat/helper/prometheus/metric.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/metricbeat/helper/prometheus/metric.go b/metricbeat/helper/prometheus/metric.go index c726e4ab99b..3035ce32bb2 100644 --- a/metricbeat/helper/prometheus/metric.go +++ b/metricbeat/helper/prometheus/metric.go @@ -24,6 +24,8 @@ import ( "github.com/elastic/beats/libbeat/common" + "fmt" + dto "github.com/prometheus/client_model/go" ) @@ -174,12 +176,32 @@ func (m *commonMetric) GetValue(metric *dto.Metric) interface{} { buckets := histogram.GetBucket() bucketMap := common.MapStr{} + maxDecimalScale := 0.0 for _, bucket := range buckets { key := strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) bucketMap[key] = bucket.GetCumulativeCount() + // key should not be allowed to contain dots. If so, scale that key accordingly to avoid dots + if strings.Contains(key, ".") { + s := strings.Split(key, ".") + decimalLength := len(s[1]) + maxDecimalScale = math.Max(maxDecimalScale, float64(decimalLength)) + } } - if len(bucketMap) != 0 { + if maxDecimalScale != 0 { + // if needed scale keys that are in double precision format + maxDecimalScale = math.Pow(10, maxDecimalScale) + for key, promValue := range bucketMap { + if strings.Contains(key, ".") { + if s, err := strconv.ParseFloat(key, 64); err == nil { + scaledKey := int(s * maxDecimalScale) + stringKey := fmt.Sprintf("%d", scaledKey) + bucketMap.Put(stringKey, promValue) + bucketMap.Delete(key) + } + } + } + } value["bucket"] = bucketMap } From b1bd96c5ea1d9bbf3b922b747edd75b35040e4cb Mon Sep 17 00:00:00 2001 From: Ioannis Androulidakis Date: Thu, 28 Feb 2019 14:22:47 +0200 Subject: [PATCH 19/24] Update generated JSON and metrics.expected Signed-off-by: Ioannis Androulidakis --- .../stats/_meta/data_request_count_event.json | 2 +- .../data_request_duration_sec_event.json | 40 +++---- .../stats/_meta/data_request_type_event.json | 2 +- .../_meta/data_response_rcode_event.json | 2 +- .../stats/_meta/data_size_bytes_event.json | 64 +++++------ .../coredns/stats/_meta/test/metrics.expected | 100 +++++++++--------- 6 files changed, 105 insertions(+), 105 deletions(-) diff --git a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json index 5ac4d15105a..7bc91526c3c 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json @@ -9,7 +9,7 @@ "dns": { "request": { "count": { - "total": 2207 + "total": 92 } } }, diff --git a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json index fb446f77f1b..c0cd43ea9e9 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json @@ -10,32 +10,32 @@ "request": { "do": { "count": { - "total": 1880 + "total": 92 } }, "duration": { "sec": { "bucket": { - "+Inf": 2207, - "0.00025": 0, - "0.0005": 0, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 1931, - "0.128": 2193, - "0.256": 2205, - "0.512": 2207, - "1.024": 2207, - "2.048": 2207, - "4.096": 2207, - "8.192": 2207 + "+Inf": 92, + "100": 0, + "102400": 92, + "12800": 92, + "1600": 0, + "200": 0, + "204800": 92, + "25": 0, + "25600": 92, + "3200": 53, + "400": 0, + "409600": 92, + "50": 0, + "51200": 92, + "6400": 91, + "800": 0, + "819200": 92 }, - "count": 2207, - "sum": 119.93155948800005 + "count": 92, + "sum": 2.852384653 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json index 65291fde75a..6373c461428 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json @@ -10,7 +10,7 @@ "request": { "type": { "count": { - "total": 2207 + "total": 92 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json index 180e8276547..30b3eed67ac 100644 --- a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json @@ -10,7 +10,7 @@ "response": { "rcode": { "count": { - "total": 2207 + "total": 92 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json index cb7f49d3dc0..b55d94e14b0 100644 --- a/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json @@ -11,24 +11,24 @@ "size": { "bytes": { "bucket": { - "+Inf": 2207, + "+Inf": 92, "0": 0, - "100": 2207, - "1023": 2207, - "16000": 2207, - "200": 2207, - "2047": 2207, - "300": 2207, - "32000": 2207, - "400": 2207, - "4095": 2207, - "48000": 2207, - "511": 2207, - "64000": 2207, - "8291": 2207 + "100": 92, + "1023": 92, + "16000": 92, + "200": 92, + "2047": 92, + "300": 92, + "32000": 92, + "400": 92, + "4095": 92, + "48000": 92, + "511": 92, + "64000": 92, + "8291": 92 }, - "count": 2207, - "sum": 114764 + "count": 92, + "sum": 4784 } } }, @@ -36,24 +36,24 @@ "size": { "bytes": { "bucket": { - "+Inf": 2207, + "+Inf": 92, "0": 0, - "100": 2207, - "1023": 2207, - "16000": 2207, - "200": 2207, - "2047": 2207, - "300": 2207, - "32000": 2207, - "400": 2207, - "4095": 2207, - "48000": 2207, - "511": 2207, - "64000": 2207, - "8291": 2207 + "100": 92, + "1023": 92, + "16000": 92, + "200": 92, + "2047": 92, + "300": 92, + "32000": 92, + "400": 92, + "4095": 92, + "48000": 92, + "511": 92, + "64000": 92, + "8291": 92 }, - "count": 2207, - "sum": 147869 + "count": 92, + "sum": 6164 } } } diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics.expected b/metricbeat/module/coredns/stats/_meta/test/metrics.expected index 4e9cb4a7923..094cc96dd9c 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics.expected +++ b/metricbeat/module/coredns/stats/_meta/test/metrics.expected @@ -44,56 +44,6 @@ "Service": "", "Took": 0 }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "dns": { - "request": { - "do": { - "count": { - "total": 113 - } - }, - "duration": { - "sec": { - "bucket": { - "+Inf": 440, - "0.00025": 0, - "0.0005": 0, - "0.001": 0, - "0.002": 0, - "0.004": 0, - "0.008": 0, - "0.016": 0, - "0.032": 0, - "0.064": 424, - "0.128": 437, - "0.256": 438, - "0.512": 440, - "1.024": 440, - "2.048": 440, - "4.096": 440, - "8.192": 440 - }, - "count": 440, - "sum": 21.712619368 - } - } - } - }, - "server": "dns://:53", - "zone": "." - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0 - }, { "RootFields": null, "ModuleFields": null, @@ -213,5 +163,55 @@ "Host": "", "Service": "", "Took": 0 + }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "do": { + "count": { + "total": 113 + } + }, + "duration": { + "sec": { + "bucket": { + "+Inf": 440, + "100": 0, + "102400": 440, + "12800": 437, + "1600": 0, + "200": 0, + "204800": 440, + "25": 0, + "25600": 438, + "3200": 0, + "400": 0, + "409600": 440, + "50": 0, + "51200": 440, + "6400": 424, + "800": 0, + "819200": 440 + }, + "count": 440, + "sum": 21.712619368 + } + } + } + }, + "server": "dns://:53", + "zone": "." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 } ] \ No newline at end of file From d3e45edba9e4218e5430632e7a8f55f031834e89 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 28 Feb 2019 15:43:21 +0200 Subject: [PATCH 20/24] Fix scaling approach Signed-off-by: Chris Mark --- metricbeat/docs/fields.asciidoc | 4 +- metricbeat/helper/prometheus/metric.go | 30 +++--- metricbeat/module/coredns/fields.go | 2 +- .../stats/_meta/data_request_count_event.json | 4 +- .../data_request_duration_sec_event.json | 43 ++++---- .../stats/_meta/data_request_type_event.json | 2 +- .../_meta/data_response_rcode_event.json | 4 +- .../stats/_meta/data_size_bytes_event.json | 64 +++++------ .../module/coredns/stats/_meta/fields.yml | 4 +- .../coredns/stats/_meta/test/metrics.expected | 100 +++++++++--------- 10 files changed, 129 insertions(+), 128 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 8c9fe3616dc..22bf2c1ddab 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2614,7 +2614,7 @@ Total query count -- type: object -Request duration histogram buckets +Request duration histogram buckets in nanoseconds -- @@ -2626,7 +2626,7 @@ type: long format: duration -Requests duration, sum of durations +Requests duration, sum of durations in nanoseconds -- diff --git a/metricbeat/helper/prometheus/metric.go b/metricbeat/helper/prometheus/metric.go index 3035ce32bb2..3826750a9bf 100644 --- a/metricbeat/helper/prometheus/metric.go +++ b/metricbeat/helper/prometheus/metric.go @@ -176,31 +176,37 @@ func (m *commonMetric) GetValue(metric *dto.Metric) interface{} { buckets := histogram.GetBucket() bucketMap := common.MapStr{} - maxDecimalScale := 0.0 + needScale := false for _, bucket := range buckets { key := strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) bucketMap[key] = bucket.GetCumulativeCount() // key should not be allowed to contain dots. If so, scale that key accordingly to avoid dots if strings.Contains(key, ".") { - s := strings.Split(key, ".") - decimalLength := len(s[1]) - maxDecimalScale = math.Max(maxDecimalScale, float64(decimalLength)) + needScale = true } } if len(bucketMap) != 0 { - if maxDecimalScale != 0 { - // if needed scale keys that are in double precision format - maxDecimalScale = math.Pow(10, maxDecimalScale) + if needScale { + // if needed scale keys that are in double precision format, use "nano" as scale factor + scaleFactor := 1000000000.0 for key, promValue := range bucketMap { if strings.Contains(key, ".") { - if s, err := strconv.ParseFloat(key, 64); err == nil { - scaledKey := int(s * maxDecimalScale) - stringKey := fmt.Sprintf("%d", scaledKey) - bucketMap.Put(stringKey, promValue) - bucketMap.Delete(key) + s, err := strconv.ParseFloat(key, 64) + if err != nil { + return nil } + scaledKey := int(s * scaleFactor) + stringKey := fmt.Sprintf("%d", scaledKey) + bucketMap.Put(stringKey, promValue) + bucketMap.Delete(key) } } + // scale sum metric too, since is the summary of the observed values that were scaled + sum, ok := value["sum"].(float64) + if !ok { + return nil + } + value.Put("sum", int(sum*scaleFactor)) } value["bucket"] = bucketMap } diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index d191c137fbc..73abd5d4fc0 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJzElk9T2zAQxe/5FG9yIh1wp50Oh8y0FziUQykD3ItsbYiKLRntGiZ8+o78JySOExPqDDqupLc/rVbPPsEDLaZInCdteQSIkZSmGNeR8QjQxIk3uRhnp/gxAtCsxy+ni5RGgKeUFNMUMYkaATNDqeZpufgEVmW0miQMWeQ0xb13RV5HOvKEcVfvu0PirChjGSxKDItJGDJXgmfyBE9KY+ZdViY6v7ypFVZRVnGCCC+jXUA7oMI468DxlCohDXGQOTUkYPJPJqGV7e2CNaNNu0qcK2uSKHGFlUicqHRtVXOC1Nn71sSOQ5TjNojBFllMHm5WJeJOBm058vRYEMvhSB4L8guU8r0QuvAqyEVMSRQXyQNJ9KkTx8V/KZHWVBX8U63QrohT2hP5ugJBA4K5YXH3XmWocPoLuXYGLrK3VnPmfKZkukz9PnJe7j8GF1logCawJ/rmjf1HK3TQvXZonb6fj80LRfFCiD+0OW7MC72jMVbot7XFe7C2ULVT9BBtv+0Bmd5oAu4QZnS5bLdgSIbqr81cPVHp7ue/ERsBUz9gYDgEIs6C5jpkTh4vzhKU1aVyJ93GREPxQItn5/V+ID9dqrksSuXdQSowhUhdgx014txZpsgnTh+iSis1anK1qlRm7gTcnBmoTqXwa4kqrE6EmcpMuhiO4XZOUFp7Yq61GwzxynLuvODoC77j4gpHF1d4Is/hu/ZtcoyvZfh0LX46mfTf7Qf7MK5rkP2dePMAw1pxN9duL95kGtqMu6m2u3H4wyU/bJNWms3jMHFK4Y+n11PCqx4WJCgGbRRMus3weefjzb0TNyzN6yttuQeOxoXOx3AeY0ny8WT0LwAA//+tuceM" + return "eJzElk9T2zwQxu/5FM/kRN4Bv9NOh0Nm2gscyqGUAe5FtjZExZaMdg0TPn1H/hMSx4kJOIOOK/nZ365Xj32CB1pMkThP2vIIECMpTTGuI+MRoIkTb3Ixzk7xYwSgOY9fThcpjQBPKSmmKWISNQJmhlLN0/LwCazKaDVJWLLIaYp774q8jnTkCeuufu4OibOijGWwKDEsJmHIXAmeyRM8KY2Zd1mZ6PzyplZYRVnFCSK8jHYB7YAK66wDx1OqhDTEQebUkIDJP5mEVh5vN6xZbdpV4lxZk0SJK6xE4kSla6eaClJn71sbO4oo120Qgy2ymDzcrErEnQzacuTpsSCWw5E8FuQXKOV7IXThVZCLmJIoLpIHkui/ThwX/6VEWltV8E91QrsiTmlP5OsKBA0I5obF3XuVocJhGAurrGNKnNX9fV0riYvsrc2dOZ8pmS5J3lcIL58/BhdZmIcm8LFKNt/nBwalA/Z1fuv0/XxsXiiKF0L8qaNzY15oc2z2od82Je/B2kLVTtFDtP1tD8j0Rotwh7Cqy+W4BbsyVH+L5uqJSu8//43YCJj6AQPDIRBxFjTXIXPyeHGWoKwulTvpNjYaigdaPDuv9wP56VLNZVMqZw9SgSlE6h7s6BHnzjJFPnH6EF1a6VGTq9WlMnMn4ObOQH0qhV9bVGF1IsxUZtLFcAy3c4LS2hNzrd1giFeWc+cFR1/wHRdXOLq4whN5Dl+9b5NjfC3Dp2vx08mk/91+sg/jugbZ34k3CxjWiru5dnvxJtPQZtxNtd2Nw/8v+WGHtNJsLoeJUwo/QL2eEm71sCBBMWijYNJthv93Xt7cO3HD0rze0pZ74Ghc6HwM5zGWJB9PRv8CAAD//x0L0vA=" } diff --git a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json index 7bc91526c3c..278abf6cd49 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_count_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_count_event.json @@ -9,14 +9,14 @@ "dns": { "request": { "count": { - "total": 92 + "total": 430 } } }, "family": "1", "proto": "udp", "server": "dns://:53", - "zone": "." + "zone": "dropped" } }, "event": { diff --git a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json index c0cd43ea9e9..555ac3206cd 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json @@ -8,34 +8,29 @@ "stats": { "dns": { "request": { - "do": { - "count": { - "total": 92 - } - }, "duration": { "sec": { "bucket": { - "+Inf": 92, - "100": 0, - "102400": 92, - "12800": 92, - "1600": 0, - "200": 0, - "204800": 92, - "25": 0, - "25600": 92, - "3200": 53, - "400": 0, - "409600": 92, - "50": 0, - "51200": 92, - "6400": 91, - "800": 0, - "819200": 92 + "+Inf": 430, + "1000000": 0, + "1024000000": 0, + "128000000": 0, + "16000000": 0, + "2000000": 0, + "2048000000": 427, + "250000": 0, + "256000000": 0, + "32000000": 0, + "4000000": 0, + "4096000000": 428, + "500000": 0, + "512000000": 0, + "64000000": 0, + "8000000": 0, + "8192000000": 430 }, - "count": 92, - "sum": 2.852384653 + "count": 430, + "sum": 870306874252 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json index 6373c461428..154d614c14a 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_type_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_type_event.json @@ -10,7 +10,7 @@ "request": { "type": { "count": { - "total": 92 + "total": 430 } } } diff --git a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json index 30b3eed67ac..0aa1da95db4 100644 --- a/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_response_rcode_event.json @@ -10,12 +10,12 @@ "response": { "rcode": { "count": { - "total": 92 + "total": 430 } } } }, - "rcode": "NOERROR", + "rcode": "SERVFAIL", "server": "dns://:53", "zone": "." } diff --git a/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json index b55d94e14b0..ecced482d6c 100644 --- a/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_size_bytes_event.json @@ -11,24 +11,24 @@ "size": { "bytes": { "bucket": { - "+Inf": 92, + "+Inf": 430, "0": 0, - "100": 92, - "1023": 92, - "16000": 92, - "200": 92, - "2047": 92, - "300": 92, - "32000": 92, - "400": 92, - "4095": 92, - "48000": 92, - "511": 92, - "64000": 92, - "8291": 92 + "100": 430, + "1023": 430, + "16000": 430, + "200": 430, + "2047": 430, + "300": 430, + "32000": 430, + "400": 430, + "4095": 430, + "48000": 430, + "511": 430, + "64000": 430, + "8291": 430 }, - "count": 92, - "sum": 4784 + "count": 430, + "sum": 24080 } } }, @@ -36,24 +36,24 @@ "size": { "bytes": { "bucket": { - "+Inf": 92, + "+Inf": 430, "0": 0, - "100": 92, - "1023": 92, - "16000": 92, - "200": 92, - "2047": 92, - "300": 92, - "32000": 92, - "400": 92, - "4095": 92, - "48000": 92, - "511": 92, - "64000": 92, - "8291": 92 + "100": 430, + "1023": 430, + "16000": 430, + "200": 430, + "2047": 430, + "300": 430, + "32000": 430, + "400": 430, + "4095": 430, + "48000": 430, + "511": 430, + "64000": 430, + "8291": 430 }, - "count": 92, - "sum": 6164 + "count": 430, + "sum": 24080 } } } diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 8eb2d60ab08..9e642e2c30c 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -16,12 +16,12 @@ type: object object_type: double description: > - Request duration histogram buckets + Request duration histogram buckets in nanoseconds - name: dns.request.duration.sec.sum type: long format: duration description: > - Requests duration, sum of durations + Requests duration, sum of durations in nanoseconds - name: dns.request.duration.sec.count type: long description: > diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics.expected b/metricbeat/module/coredns/stats/_meta/test/metrics.expected index 094cc96dd9c..edd3fc5c900 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics.expected +++ b/metricbeat/module/coredns/stats/_meta/test/metrics.expected @@ -44,6 +44,56 @@ "Service": "", "Took": 0 }, + { + "RootFields": null, + "ModuleFields": null, + "MetricSetFields": { + "dns": { + "request": { + "do": { + "count": { + "total": 113 + } + }, + "duration": { + "sec": { + "bucket": { + "+Inf": 440, + "1000000": 0, + "1024000000": 440, + "128000000": 437, + "16000000": 0, + "2000000": 0, + "2048000000": 440, + "250000": 0, + "256000000": 438, + "32000000": 0, + "4000000": 0, + "4096000000": 440, + "500000": 0, + "512000000": 440, + "64000000": 424, + "8000000": 0, + "8192000000": 440 + }, + "count": 440, + "sum": 21712619368 + } + } + } + }, + "server": "dns://:53", + "zone": "." + }, + "Index": "", + "ID": "", + "Namespace": "", + "Timestamp": "0001-01-01T00:00:00Z", + "Error": null, + "Host": "", + "Service": "", + "Took": 0 + }, { "RootFields": null, "ModuleFields": null, @@ -163,55 +213,5 @@ "Host": "", "Service": "", "Took": 0 - }, - { - "RootFields": null, - "ModuleFields": null, - "MetricSetFields": { - "dns": { - "request": { - "do": { - "count": { - "total": 113 - } - }, - "duration": { - "sec": { - "bucket": { - "+Inf": 440, - "100": 0, - "102400": 440, - "12800": 437, - "1600": 0, - "200": 0, - "204800": 440, - "25": 0, - "25600": 438, - "3200": 0, - "400": 0, - "409600": 440, - "50": 0, - "51200": 440, - "6400": 424, - "800": 0, - "819200": 440 - }, - "count": 440, - "sum": 21.712619368 - } - } - } - }, - "server": "dns://:53", - "zone": "." - }, - "Index": "", - "ID": "", - "Namespace": "", - "Timestamp": "0001-01-01T00:00:00Z", - "Error": null, - "Host": "", - "Service": "", - "Took": 0 } ] \ No newline at end of file From ebb8066727a7469cd95e0931136c326c6d4aecf2 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 28 Feb 2019 16:14:27 +0200 Subject: [PATCH 21/24] Fix docs and CHANGELOG Signed-off-by: Chris Mark --- CHANGELOG.next.asciidoc | 1 - metricbeat/metricbeat.reference.yml | 4 ++-- x-pack/metricbeat/metricbeat.reference.yml | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9c35fc27c77..dfab5a9a01d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -350,7 +350,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] - Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] - Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055] -- Add `coredns` metricbeat {pull}10585{10585] - Add `coredns` metricbeat module. {pull}10585{10585] *Packetbeat* diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f15e55a1174..e7a50eb84f4 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -842,7 +842,7 @@ metricbeat.modules: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, +# The supported processors are drop_fields, drop_event, include_fields, # decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that @@ -933,7 +933,7 @@ metricbeat.modules: # match_pids: ["system.process.ppid"] # target: system.process.parent # -# The following example decodes fields containing JSON strings +# The following example decodes fields containing JSON strings # and replaces the strings with valid JSON objects. # #processors: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 3d2db806630..b256d41696a 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -871,7 +871,7 @@ metricbeat.modules: # # event -> filter1 -> event1 -> filter2 ->event2 ... # -# The supported processors are drop_fields, drop_event, include_fields, +# The supported processors are drop_fields, drop_event, include_fields, # decode_json_fields, and add_cloud_metadata. # # For example, you can use the following processors to keep the fields that @@ -962,7 +962,7 @@ metricbeat.modules: # match_pids: ["system.process.ppid"] # target: system.process.parent # -# The following example decodes fields containing JSON strings +# The following example decodes fields containing JSON strings # and replaces the strings with valid JSON objects. # #processors: From c3c902511fdb407881a6b6a55e34742df1699b37 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 6 Mar 2019 16:28:11 +0200 Subject: [PATCH 22/24] Use multiplier option of Prometheus Fields Signed-off-by: Chris Mark --- metricbeat/helper/prometheus/metric.go | 29 ------------------------ metricbeat/module/coredns/stats/stats.go | 4 +++- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/metricbeat/helper/prometheus/metric.go b/metricbeat/helper/prometheus/metric.go index 3826750a9bf..9ec0e835799 100644 --- a/metricbeat/helper/prometheus/metric.go +++ b/metricbeat/helper/prometheus/metric.go @@ -24,8 +24,6 @@ import ( "github.com/elastic/beats/libbeat/common" - "fmt" - dto "github.com/prometheus/client_model/go" ) @@ -176,38 +174,11 @@ func (m *commonMetric) GetValue(metric *dto.Metric) interface{} { buckets := histogram.GetBucket() bucketMap := common.MapStr{} - needScale := false for _, bucket := range buckets { key := strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) bucketMap[key] = bucket.GetCumulativeCount() - // key should not be allowed to contain dots. If so, scale that key accordingly to avoid dots - if strings.Contains(key, ".") { - needScale = true - } } if len(bucketMap) != 0 { - if needScale { - // if needed scale keys that are in double precision format, use "nano" as scale factor - scaleFactor := 1000000000.0 - for key, promValue := range bucketMap { - if strings.Contains(key, ".") { - s, err := strconv.ParseFloat(key, 64) - if err != nil { - return nil - } - scaledKey := int(s * scaleFactor) - stringKey := fmt.Sprintf("%d", scaledKey) - bucketMap.Put(stringKey, promValue) - bucketMap.Delete(key) - } - } - // scale sum metric too, since is the summary of the observed values that were scaled - sum, ok := value["sum"].(float64) - if !ok { - return nil - } - value.Put("sum", int(sum*scaleFactor)) - } value["bucket"] = bucketMap } diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index 0ff9a90ad0c..054f0af7a8b 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -26,7 +26,9 @@ var mapping = &prometheus.MetricsMapping{ Metrics: map[string]prometheus.MetricMap{ "coredns_panic_count_total": prometheus.Metric("panic.count.total"), "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), - "coredns_dns_request_duration_seconds": prometheus.Metric("dns.request.duration.sec"), + "coredns_dns_request_duration_seconds": prometheus.Metric( + "dns.request.duration.sec", + prometheus.OpMultiplyBuckets(1000000000)), "coredns_dns_request_size_bytes": prometheus.Metric("dns.request.size.bytes"), "coredns_dns_request_do_count_total": prometheus.Metric("dns.request.do.count.total"), "coredns_dns_request_type_count_total": prometheus.Metric("dns.request.type.count.total"), From 2b5d62df9eaf2a33c7a867b4002ceb520387b4ec Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 13 Mar 2019 23:08:49 +0200 Subject: [PATCH 23/24] Use the fixed Hist multiplier Signed-off-by: Chris --- metricbeat/module/coredns/stats/stats.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index 054f0af7a8b..64efd6054f8 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -24,9 +24,9 @@ import ( var mapping = &prometheus.MetricsMapping{ Metrics: map[string]prometheus.MetricMap{ - "coredns_panic_count_total": prometheus.Metric("panic.count.total"), - "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), - "coredns_dns_request_duration_seconds": prometheus.Metric( + "coredns_panic_count_total": prometheus.Metric("panic.count.total"), + "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), + "coredns_dns_request_duration_seconds": prometheus.Metric( "dns.request.duration.sec", prometheus.OpMultiplyBuckets(1000000000)), "coredns_dns_request_size_bytes": prometheus.Metric("dns.request.size.bytes"), From cc34863883049129296720898377666af16b80bf Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 18 Mar 2019 16:32:56 +0200 Subject: [PATCH 24/24] Fix data structures Signed-off-by: Chris Mark --- metricbeat/docs/fields.asciidoc | 14 ++++++------- metricbeat/helper/prometheus/metric.go | 1 + metricbeat/module/coredns/fields.go | 2 +- ...on => data_request_duration_ns_event.json} | 4 ++-- .../module/coredns/stats/_meta/fields.yml | 20 +++++++++---------- .../coredns/stats/_meta/test/metrics.expected | 4 ++-- metricbeat/module/coredns/stats/stats.go | 2 +- .../coredns/stats/stats_integration_test.go | 2 +- 8 files changed, 25 insertions(+), 24 deletions(-) rename metricbeat/module/coredns/stats/_meta/{data_request_duration_sec_event.json => data_request_duration_ns_event.json} (97%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 22bf2c1ddab..3fa49552947 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2609,7 +2609,7 @@ Total query count -- -*`coredns.stats.dns.request.duration.sec.bucket.*`*:: +*`coredns.stats.dns.request.duration.ns.bucket.*`*:: + -- type: object @@ -2619,7 +2619,7 @@ Request duration histogram buckets in nanoseconds -- -*`coredns.stats.dns.request.duration.sec.sum`*:: +*`coredns.stats.dns.request.duration.ns.sum`*:: + -- type: long @@ -2631,7 +2631,7 @@ Requests duration, sum of durations in nanoseconds -- -*`coredns.stats.dns.request.duration.sec.count`*:: +*`coredns.stats.dns.request.duration.ns.count`*:: + -- type: long @@ -2654,7 +2654,7 @@ Request Size histogram buckets *`coredns.stats.dns.request.size.bytes.sum`*:: + -- -type: double +type: long Request Size histogram sum @@ -2664,7 +2664,7 @@ Request Size histogram sum *`coredns.stats.dns.request.size.bytes.count`*:: + -- -type: double +type: long Request Size histogram count @@ -2744,7 +2744,7 @@ Response Size histogram buckets *`coredns.stats.dns.response.size.bytes.sum`*:: + -- -type: double +type: long Response Size histogram sum @@ -2754,7 +2754,7 @@ Response Size histogram sum *`coredns.stats.dns.response.size.bytes.count`*:: + -- -type: double +type: long Response Size histogram count diff --git a/metricbeat/helper/prometheus/metric.go b/metricbeat/helper/prometheus/metric.go index 9ec0e835799..c726e4ab99b 100644 --- a/metricbeat/helper/prometheus/metric.go +++ b/metricbeat/helper/prometheus/metric.go @@ -178,6 +178,7 @@ func (m *commonMetric) GetValue(metric *dto.Metric) interface{} { key := strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) bucketMap[key] = bucket.GetCumulativeCount() } + if len(bucketMap) != 0 { value["bucket"] = bucketMap } diff --git a/metricbeat/module/coredns/fields.go b/metricbeat/module/coredns/fields.go index 73abd5d4fc0..33487332f15 100644 --- a/metricbeat/module/coredns/fields.go +++ b/metricbeat/module/coredns/fields.go @@ -32,5 +32,5 @@ func init() { // AssetCoredns returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/coredns. func AssetCoredns() string { - return "eJzElk9T2zwQxu/5FM/kRN4Bv9NOh0Nm2gscyqGUAe5FtjZExZaMdg0TPn1H/hMSx4kJOIOOK/nZ365Xj32CB1pMkThP2vIIECMpTTGuI+MRoIkTb3Ixzk7xYwSgOY9fThcpjQBPKSmmKWISNQJmhlLN0/LwCazKaDVJWLLIaYp774q8jnTkCeuufu4OibOijGWwKDEsJmHIXAmeyRM8KY2Zd1mZ6PzyplZYRVnFCSK8jHYB7YAK66wDx1OqhDTEQebUkIDJP5mEVh5vN6xZbdpV4lxZk0SJK6xE4kSla6eaClJn71sbO4oo120Qgy2ymDzcrErEnQzacuTpsSCWw5E8FuQXKOV7IXThVZCLmJIoLpIHkui/ThwX/6VEWltV8E91QrsiTmlP5OsKBA0I5obF3XuVocJhGAurrGNKnNX9fV0riYvsrc2dOZ8pmS5J3lcIL58/BhdZmIcm8LFKNt/nBwalA/Z1fuv0/XxsXiiKF0L8qaNzY15oc2z2od82Je/B2kLVTtFDtP1tD8j0Rotwh7Cqy+W4BbsyVH+L5uqJSu8//43YCJj6AQPDIRBxFjTXIXPyeHGWoKwulTvpNjYaigdaPDuv9wP56VLNZVMqZw9SgSlE6h7s6BHnzjJFPnH6EF1a6VGTq9WlMnMn4ObOQH0qhV9bVGF1IsxUZtLFcAy3c4LS2hNzrd1giFeWc+cFR1/wHRdXOLq4whN5Dl+9b5NjfC3Dp2vx08mk/91+sg/jugbZ34k3CxjWiru5dnvxJtPQZtxNtd2Nw/8v+WGHtNJsLoeJUwo/QL2eEm71sCBBMWijYNJthv93Xt7cO3HD0rze0pZ74Ghc6HwM5zGWJB9PRv8CAAD//x0L0vA=" + return "eJzElk9T2zwQxu/5FM/kRN4Bv9NOh0Nm2gscyqGUAe5FtjZExZaMdg0TPn1H/hOMY2ICzqDjSnr2p9XqsY9wR6s5EudJW54AYiSlOaZ1ZDoBNHHiTS7G2Tl+TAA06/HL6SKlCeApJcU0R0yiJsDCUKp5Xi4+glUZtZOEIauc5rj1rsjrSE+eMG7qfTdInBVlLINFiWExCUOWSvBInuBJaSy8y8pEp+dXtUIbpY0TRHgd7QPaAhXGSQ+Op1QJaYiDLKkhAZN/MAm1tncL1owubZs4V9YkUeIKK5E4UemLVc0JUmdvOxNbDlGO6yAGW2QxebhFlYh7GbTlyNN9QSz7I7kvyK9Qyg9C6MKrIBdZjuIiuSOJ/uulcfFfSqQzVQX/vJv3sqJAQ4GlYXG3XmWoYBjGwirrmBJn9XBR2+fhIntrYRfOZ0rma5D3nYPX+w/BRRZ6oQl86CCbV/mBHulhfW7dOvswHpsniuKV0Ge2zZV5os2W2YV9hxYZgHqFqZtggGfMm36F6I2+4PbhT+frRgseZaj+AC3VA5WGf/obsREwDQMGhn0g4iRovoTMyePJWYKyulTupduYaCjuaPXovN4N5KdLNZdFqew8SAWmEKlrsKVGnDvLFPnE6X1UqVWjJlenSmXmXsDNmZHqVAo/l6jC6kVYqMykq/EYrpcEpbUn5lq7wRCvLOfOCw6+4DvOLnBwdoEH8hy+dt9mh/haho9fxI9ns+G7/VQHxmWNsbsHb+KPacL9VNtdeJNoXBvuZ3rdh8PvLvlx27PSbJ6FiVMK/zyDbhLe87ggQTFoo2DSXYb/tz7b3Dtx49I8v8+Ob+BgWuh8CucxlSSfzib/AgAA///RG8xV" } diff --git a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json b/metricbeat/module/coredns/stats/_meta/data_request_duration_ns_event.json similarity index 97% rename from metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json rename to metricbeat/module/coredns/stats/_meta/data_request_duration_ns_event.json index 555ac3206cd..32708e27cfb 100644 --- a/metricbeat/module/coredns/stats/_meta/data_request_duration_sec_event.json +++ b/metricbeat/module/coredns/stats/_meta/data_request_duration_ns_event.json @@ -9,7 +9,7 @@ "dns": { "request": { "duration": { - "sec": { + "ns": { "bucket": { "+Inf": 430, "1000000": 0, @@ -51,4 +51,4 @@ "address": "127.0.0.1:9153", "type": "coredns" } -} \ No newline at end of file +} diff --git a/metricbeat/module/coredns/stats/_meta/fields.yml b/metricbeat/module/coredns/stats/_meta/fields.yml index 9e642e2c30c..fd9bb260b1f 100644 --- a/metricbeat/module/coredns/stats/_meta/fields.yml +++ b/metricbeat/module/coredns/stats/_meta/fields.yml @@ -12,31 +12,31 @@ type: long description: > Total query count - - name: dns.request.duration.sec.bucket.* + - name: dns.request.duration.ns.bucket.* type: object - object_type: double + object_type: long description: > Request duration histogram buckets in nanoseconds - - name: dns.request.duration.sec.sum + - name: dns.request.duration.ns.sum type: long format: duration description: > Requests duration, sum of durations in nanoseconds - - name: dns.request.duration.sec.count + - name: dns.request.duration.ns.count type: long description: > Requests duration, number of requests - name: dns.request.size.bytes.bucket.* type: object - object_type: double + object_type: long description: > Request Size histogram buckets - name: dns.request.size.bytes.sum - type: double + type: long description: > Request Size histogram sum - name: dns.request.size.bytes.count - type: double + type: long description: > Request Size histogram count - name: dns.request.do.count.total @@ -65,15 +65,15 @@ The address family of the transport (1 = IP (IP version 4), 2 = IP6 (IP version 6)) - name: dns.response.size.bytes.bucket.* type: object - object_type: double + object_type: long description: > Response Size histogram buckets - name: dns.response.size.bytes.sum - type: double + type: long description: > Response Size histogram sum - name: dns.response.size.bytes.count - type: double + type: long description: > Response Size histogram count - name: server diff --git a/metricbeat/module/coredns/stats/_meta/test/metrics.expected b/metricbeat/module/coredns/stats/_meta/test/metrics.expected index edd3fc5c900..ef2093b6410 100644 --- a/metricbeat/module/coredns/stats/_meta/test/metrics.expected +++ b/metricbeat/module/coredns/stats/_meta/test/metrics.expected @@ -56,7 +56,7 @@ } }, "duration": { - "sec": { + "ns": { "bucket": { "+Inf": 440, "1000000": 0, @@ -214,4 +214,4 @@ "Service": "", "Took": 0 } -] \ No newline at end of file +] diff --git a/metricbeat/module/coredns/stats/stats.go b/metricbeat/module/coredns/stats/stats.go index 64efd6054f8..d688e10021d 100644 --- a/metricbeat/module/coredns/stats/stats.go +++ b/metricbeat/module/coredns/stats/stats.go @@ -27,7 +27,7 @@ var mapping = &prometheus.MetricsMapping{ "coredns_panic_count_total": prometheus.Metric("panic.count.total"), "coredns_dns_request_count_total": prometheus.Metric("dns.request.count.total"), "coredns_dns_request_duration_seconds": prometheus.Metric( - "dns.request.duration.sec", + "dns.request.duration.ns", prometheus.OpMultiplyBuckets(1000000000)), "coredns_dns_request_size_bytes": prometheus.Metric("dns.request.size.bytes"), "coredns_dns_request_do_count_total": prometheus.Metric("dns.request.do.count.total"), diff --git a/metricbeat/module/coredns/stats/stats_integration_test.go b/metricbeat/module/coredns/stats/stats_integration_test.go index 61bd3eb6fd3..c7766ff91e0 100644 --- a/metricbeat/module/coredns/stats/stats_integration_test.go +++ b/metricbeat/module/coredns/stats/stats_integration_test.go @@ -45,7 +45,7 @@ func TestData(t *testing.T) { {"coredns.stats.panic.count.total", "./_meta/data_panic_event.json"}, {"coredns.stats.dns.request.count.total", "./_meta/data_request_count_event.json"}, {"coredns.stats.dns.request.size.bytes", "./_meta/data_size_bytes_event.json"}, - {"coredns.stats.dns.request.duration.sec", "./_meta/data_request_duration_sec_event.json"}, + {"coredns.stats.dns.request.duration.ns", "./_meta/data_request_duration_ns_event.json"}, {"coredns.stats.dns.response.rcode", "./_meta/data_response_rcode_event.json"}, {"coredns.stats.dns.request.type", "./_meta/data_request_type_event.json"}, }