From ac6aa323437a8a428aa96f2b6f0d9c287af448b5 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 17 Jan 2019 17:16:17 +0200 Subject: [PATCH] Add connection metricset of nats module (#10095) This PR adds `connections` metricset to `nats` module as part of https://github.com/elastic/beats/issues/10071. `connections` metricset retrieves metrics from `connz` [monitoring](https://nats.io/documentation/tutorials/nats-monitoring/) URI of `nats`. _______________________________________________________________________________ Co-Authored-By: Stamatis Katsaounis , @skatsaounis Co-Authored-By: Michael Katsoulis , @MichaelKatsoulis --- CHANGELOG.asciidoc | 3 +- metricbeat/docs/fields.asciidoc | 35 +++++-- metricbeat/docs/modules/nats.asciidoc | 7 +- .../docs/modules/nats/connections.asciidoc | 23 +++++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 3 +- .../module/nats/_meta/config.reference.yml | 3 +- metricbeat/module/nats/_meta/config.yml | 5 +- metricbeat/module/nats/_meta/fields.yml | 8 ++ .../module/nats/connections/_meta/data.json | 21 ++++ .../nats/connections/_meta/docs.asciidoc | 1 + .../module/nats/connections/_meta/fields.yml | 9 ++ .../_meta/test/connectionsmetrics.json | 5 + .../module/nats/connections/connections.go | 98 +++++++++++++++++++ .../connections_integration_test.go | 76 ++++++++++++++ .../nats/connections/connections_test.go | 66 +++++++++++++ metricbeat/module/nats/connections/data.go | 66 +++++++++++++ metricbeat/module/nats/fields.go | 2 +- metricbeat/module/nats/stats/_meta/data.json | 4 +- metricbeat/module/nats/stats/_meta/fields.yml | 8 -- .../{ => stats}/_meta/test/statsmetrics.json | 0 metricbeat/module/nats/stats/data.go | 18 +++- metricbeat/module/nats/stats/stats_test.go | 5 +- metricbeat/modules.d/nats.yml.disabled | 5 +- metricbeat/tests/system/test_nats.py | 25 +++++ x-pack/metricbeat/metricbeat.reference.yml | 3 +- 27 files changed, 465 insertions(+), 38 deletions(-) create mode 100644 metricbeat/docs/modules/nats/connections.asciidoc create mode 100644 metricbeat/module/nats/connections/_meta/data.json create mode 100644 metricbeat/module/nats/connections/_meta/docs.asciidoc create mode 100644 metricbeat/module/nats/connections/_meta/fields.yml create mode 100644 metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json create mode 100644 metricbeat/module/nats/connections/connections.go create mode 100644 metricbeat/module/nats/connections/connections_integration_test.go create mode 100644 metricbeat/module/nats/connections/connections_test.go create mode 100644 metricbeat/module/nats/connections/data.go rename metricbeat/module/nats/{ => stats}/_meta/test/statsmetrics.json (100%) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 22cfd3cbbe5..481264c1e78 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -66,7 +66,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* *Metricbeat* -- Add nats module with stats metricset. {pull}9825[9825] *Packetbeat* @@ -102,6 +101,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657] {pull}9746[9746] - Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709] +- Add `nats` module with `stats` metricset. {pull}9825[9825] +- Add `connections` metricset to the `nats` module. {pull}10095[10095] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7a9ff3f9c11..bb733611e3d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -17232,33 +17232,50 @@ nats Module -[float] -== stats fields +*`nats.server_id`*:: ++ +-- +type: string -Contains nats var related metrics +The server ID +-- -*`nats.stats.server_id`*:: +*`nats.now`*:: + -- -type: keyword +type: date -The server ID +Server time of metric creation -- -*`nats.stats.now`*:: +[float] +== connections fields + +Contains nats connection related metrics + + + +*`nats.connections.total`*:: + -- -type: date +type: integer -Server time of metric creation +The number of currently active clients -- +[float] +== stats fields + +Contains nats var related metrics + + + *`nats.stats.uptime`*:: + -- diff --git a/metricbeat/docs/modules/nats.asciidoc b/metricbeat/docs/modules/nats.asciidoc index 3be7b919295..323b1046be6 100644 --- a/metricbeat/docs/modules/nats.asciidoc +++ b/metricbeat/docs/modules/nats.asciidoc @@ -29,10 +29,11 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" ---- [float] @@ -40,7 +41,11 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> +include::nats/connections.asciidoc[] + include::nats/stats.asciidoc[] diff --git a/metricbeat/docs/modules/nats/connections.asciidoc b/metricbeat/docs/modules/nats/connections.asciidoc new file mode 100644 index 00000000000..e15c0ece302 --- /dev/null +++ b/metricbeat/docs/modules/nats/connections.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-nats-connections]] +=== Nats connections metricset + +experimental[] + +include::../../../module/nats/connections/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/connections/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 1e870a50dd7..abb46f8b498 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -102,7 +102,8 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> experimental[] |<> |<> experimental[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> experimental[] +.2+| .2+| |<> experimental[] +|<> experimental[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 020fa2ddcc5..34f3aeaf6aa 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -124,6 +124,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/mysql/galera_status" _ "github.com/elastic/beats/metricbeat/module/mysql/status" _ "github.com/elastic/beats/metricbeat/module/nats" + _ "github.com/elastic/beats/metricbeat/module/nats/connections" _ "github.com/elastic/beats/metricbeat/module/nats/stats" _ "github.com/elastic/beats/metricbeat/module/nginx" _ "github.com/elastic/beats/metricbeat/module/nginx/stubstatus" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 50732c9ceed..6ab6d456fdb 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -518,10 +518,11 @@ metricbeat.modules: #-------------------------------- Nats Module -------------------------------- - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" #-------------------------------- Nginx Module ------------------------------- - module: nginx diff --git a/metricbeat/module/nats/_meta/config.reference.yml b/metricbeat/module/nats/_meta/config.reference.yml index 227663d8eec..7ed0536aba3 100644 --- a/metricbeat/module/nats/_meta/config.reference.yml +++ b/metricbeat/module/nats/_meta/config.reference.yml @@ -1,5 +1,6 @@ - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" diff --git a/metricbeat/module/nats/_meta/config.yml b/metricbeat/module/nats/_meta/config.yml index b4ca5aea241..7ed0536aba3 100644 --- a/metricbeat/module/nats/_meta/config.yml +++ b/metricbeat/module/nats/_meta/config.yml @@ -1,5 +1,6 @@ - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] - stats.metrics_path: "/varz" + #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" diff --git a/metricbeat/module/nats/_meta/fields.yml b/metricbeat/module/nats/_meta/fields.yml index 7a6842e8dda..84bd89cb097 100644 --- a/metricbeat/module/nats/_meta/fields.yml +++ b/metricbeat/module/nats/_meta/fields.yml @@ -8,3 +8,11 @@ description: > `nats` contains statistics that were read from Nats fields: + - name: server_id + type: string + description: > + The server ID + - name: now + type: date + description: > + Server time of metric creation diff --git a/metricbeat/module/nats/connections/_meta/data.json b/metricbeat/module/nats/connections/_meta/data.json new file mode 100644 index 00000000000..0c2d47082d9 --- /dev/null +++ b/metricbeat/module/nats/connections/_meta/data.json @@ -0,0 +1,21 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"nats", + "name":"connections", + "rtt":44269 + }, + "nats":{ + "server_id": "bUAdpRFtMWddIBWw80Yd9D", + "now": "2018-12-28T12:33:53.026865597Z", + "connections":{ + "total": 10 + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/nats/connections/_meta/docs.asciidoc b/metricbeat/module/nats/connections/_meta/docs.asciidoc new file mode 100644 index 00000000000..1d48f9d5719 --- /dev/null +++ b/metricbeat/module/nats/connections/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the connections metricset of the module nats. diff --git a/metricbeat/module/nats/connections/_meta/fields.yml b/metricbeat/module/nats/connections/_meta/fields.yml new file mode 100644 index 00000000000..43bbcfcb31f --- /dev/null +++ b/metricbeat/module/nats/connections/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: connections + type: group + description: > + Contains nats connection related metrics + fields: + - name: total + type: integer + description: > + The number of currently active clients diff --git a/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json b/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json new file mode 100644 index 00000000000..4dec7c8eaac --- /dev/null +++ b/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json @@ -0,0 +1,5 @@ +{ + "server_id": "bUAdpRFtMWddIBWw80Yd9D", + "now": "2018-12-28T12:33:53.026865597Z", + "total": 10 +} diff --git a/metricbeat/module/nats/connections/connections.go b/metricbeat/module/nats/connections/connections.go new file mode 100644 index 00000000000..398bcac0cc1 --- /dev/null +++ b/metricbeat/module/nats/connections/connections.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connections + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/helper/elastic" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/connz" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + PathConfigKey: "connections.metrics_path", + }.Build() +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("nats", "connections", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + Log *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The nats connections metricset is experimental.") + + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + return &MetricSet{ + base, + http, + logp.NewLogger("nats"), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(r mb.ReporterV2) { + content, err := m.http.FetchContent() + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + err = eventMapping(r, content) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } +} diff --git a/metricbeat/module/nats/connections/connections_integration_test.go b/metricbeat/module/nats/connections/connections_integration_test.go new file mode 100644 index 00000000000..c5bc1bbd095 --- /dev/null +++ b/metricbeat/module/nats/connections/connections_integration_test.go @@ -0,0 +1,76 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package connections + +import ( + "os" + "testing" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + compose.EnsureUp(t, "nats") + + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json") + if err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "nats") + + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "nats", + "metricsets": []string{"connections"}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} + +func GetEnvHost() string { + host := os.Getenv("NATS_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func GetEnvPort() string { + port := os.Getenv("NATS_PORT") + + if len(port) == 0 { + port = "8222" + } + return port +} diff --git a/metricbeat/module/nats/connections/connections_test.go b/metricbeat/module/nats/connections/connections_test.go new file mode 100644 index 00000000000..6c32088938d --- /dev/null +++ b/metricbeat/module/nats/connections/connections_test.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connections + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestEventMapping(t *testing.T) { + content, err := ioutil.ReadFile("./_meta/test/connectionsmetrics.json") + assert.NoError(t, err) + reporter := &mbtest.CapturingReporterV2{} + err = eventMapping(reporter, content) + assert.NoError(t, err) + event := reporter.GetEvents()[0] + d, _ := event.MetricSetFields.GetValue("total") + assert.Equal(t, d, int64(10)) +} + +func TestFetchEventContent(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test") + + response, _ := ioutil.ReadFile(absPath + "/connectionsmetrics.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "nats", + "metricsets": []string{"connections"}, + "hosts": []string{server.URL}, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2(t, config) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go new file mode 100644 index 00000000000..c913899ae8e --- /dev/null +++ b/metricbeat/module/nats/connections/data.go @@ -0,0 +1,66 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connections + +import ( + "encoding/json" + + "github.com/elastic/beats/metricbeat/mb" + + "github.com/pkg/errors" + + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" +) + +var ( + moduleSchema = s.Schema{ + "server_id": c.Str("server_id"), + "now": c.Str("now"), + } + connectionsSchema = s.Schema{ + "total": c.Int("total"), + } +) + +func eventMapping(r mb.ReporterV2, content []byte) error { + var event mb.Event + var inInterface map[string]interface{} + + err := json.Unmarshal(content, &inInterface) + if err != nil { + err = errors.Wrap(err, "failure parsing Nats connections API response") + r.Error(err) + return err + } + event.MetricSetFields, err = connectionsSchema.Apply(inInterface) + if err != nil { + err = errors.Wrap(err, "failure applying connections schema") + r.Error(err) + return err + } + + event.ModuleFields, err = moduleSchema.Apply(inInterface) + if err != nil { + err = errors.Wrap(err, "failure applying module schema") + r.Error(err) + return err + } + r.Event(event) + return nil +} diff --git a/metricbeat/module/nats/fields.go b/metricbeat/module/nats/fields.go index 21a861a023f..699f38a1e2d 100644 --- a/metricbeat/module/nats/fields.go +++ b/metricbeat/module/nats/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzUlzFz2zoMx3d/Clym94bkA3h4d+/aJUMzNOns0BQss5EIBQTjUz99D3KkyAplW66Tczl4kEj8fwDEP+lreMJ6Dt5ImAGIkwLncHVnJFzNADIMll0ljvwc/psBQDMTvlEWC5wBrBwWWZg3b67BmxK7WDqkrnAOOVOsXp8kIup41EWPYMmLcT5AECMuiLMBZG0ENsgIjCaDFVMJd28SfYI+hUYI3dMUyh4cHV9alibjF8PAWBjBDEoUdrYffAixA4L8grxw2c7bFugJ6w3x8N0eLB0Pa3wNC7dfk6KeNkm5zAhO07rf6ogrEWj1mjtYRqMrkuKx0tlJ/YJ8Pj3XCtlRBvKWtgsQK/gnoP03iVBiebOsBcMZKWxkRi8am7iGGEzelOTu/4d7qJgshpCEscQjIM4L5sjTWXwsl8iqXlDurCm2Ik2J+jzA0QcY6ZOtYpJqVZCR0+tjq7hTnLC3OEJiioUl79Fq9A8rVCNU1NtPFzOwhUMvaSjGksY+njOgMOYuCDJmSaWWwvkkwNDDjpQ3JUUvKu+8pdL5XN3ADOamnAx2NlbQ1g5Lc6g8RzDu4RwVbalSW/1jkd4rtiwUh3vnTG2jKDn9BW3rOC+nbR3SeNtCQRu1oRBL5A/b+aoCnUprmUUN5BvfTqKtRaoF4/OiudPcMJEsIrskIy1/op1o3gPGtRM9MkB1oCTvhFhrN5QcwVMr/3VWvjReo3MCH1MU/AzArdAJhCEuw2cANjqn8OnvZ/CpzrF85/Rcjdfe8rsb3vbmOdF3u6KNulwK8QjMFpXxOWKQ/j+mxNwxzD7q+34eB3ok7AAYMheE3TLqGm12r80/vt/ej8TYl0g/me3eG512+IiZmBn8gQuk+BtzuwT8gy6bon8xfBHwyjGVvTHFS4A/6M4pej2zLwE+dXf4HQAA///zSakp" + return "eJzUl7Fy2zwMx3c/BS7T9w3JA3joXa9dMjRDk84OTcEyW4lQQNA59+l7lC2FlihbdpWcq8GDSOP/0x8ESN7CL9zOwSpxMwAxUuAcbh6UuJsZQIZOs6nEkJ3DpxkA1DPhG2W+wBnAymCRuXk9cgtWldjGCo9sK5xDzuSr/ZtExPA8hz89gyYrylgHTpQYJ0Y7kLUSeEVGYFQZrJhKeHiTiAliCoe8QV6YrB1pcJywsXn0eoApPE9r3EeC+689DUuvveiZEhwX+3EXV0yJQCsoUdho0IwqzO6JabIWdRhyPdHY4ROqXxqL60S+BQXGQglme45Yo2txTCUkqjgYaZiMFcyRO2NHyBq/rS+XyMES7ZnRSrEFpcVsEHRh0IrrJ1uUTOjKRvGFdvgqpDPpR0EHi26kGRWyoQzkbR0aB76C/xzq/5MIJZZ3y62gm5Bin4gQm3gL3qm8XrMPn58eoWLS6FwSRhMPgEywPArKjVbFTqS2KOYB9tZBVEgHXJVPUq0KUnK5P7ryB+a4o+bUtbNI1/WkRtVCxXbXWzDrlVEMxVjS0OKZAIUxN06QMUsqNRTGJgG6FT1SXpXkrQR5YzWVxuahTavO3FRdw0FhuZDarjWn7BnBeIRzULShSpX6+yL1FRsW8t3amSht5CWnfyBtLef1pK1FGk6bK+g1tCHnS+R3q/ygAq1KtLeTrft2Em0tUi0YXxb1Dn/HRLLwbJKMtPyJ+szm3WFcGwlbBgQdKMkaoXBWhK7kAF5o5b8n5Uvj1ToX8DF5wY8A3AldQOj80n0EYK1zCV/4/Qi+oDOWb8qeG+I1Z972hLc7eZ7Zd1vTBrtcCnEEZoPK+OLRSXxZTMwdwoxR+/kcBzoStgMMmQlX0KWvr11k4zT/+H7/OBDj2IfEH7OrvcFpp7eYM78M/qILpPjr5nYN+Ce7bIp+o/gq4APHuex1U7wG+JPdOUUf9uxrgE+dHf4EAAD//+cs8ho=" } diff --git a/metricbeat/module/nats/stats/_meta/data.json b/metricbeat/module/nats/stats/_meta/data.json index fa0d8032d12..c2e4c080e5b 100644 --- a/metricbeat/module/nats/stats/_meta/data.json +++ b/metricbeat/module/nats/stats/_meta/data.json @@ -11,6 +11,8 @@ "rtt":44269 }, "nats":{ + "server_id":"bUAdpRFtMWddIBWw80Yd9D", + "now":"2018-12-28T12:33:53.026864929Z", "stats":{ "cores":0, "cpu":0, @@ -31,13 +33,11 @@ "mem":{ "bytes": 806976 }, - "now":"2018-12-28T12:33:53.026864929Z", "out":{ "bytes":0, "messages":0 }, "remotes":0, - "server_id":"bUAdpRFtMWddIBWw80Yd9D", "slow_consumers":0, "total_connections":35, "uptime":9702 diff --git a/metricbeat/module/nats/stats/_meta/fields.yml b/metricbeat/module/nats/stats/_meta/fields.yml index e632fa7ed8d..4f0f055504f 100644 --- a/metricbeat/module/nats/stats/_meta/fields.yml +++ b/metricbeat/module/nats/stats/_meta/fields.yml @@ -3,14 +3,6 @@ description: > Contains nats var related metrics fields: - - name: server_id - type: keyword - description: > - The server ID - - name: now - type: date - description: > - Server time of metric creation - name: uptime type: long description: > diff --git a/metricbeat/module/nats/_meta/test/statsmetrics.json b/metricbeat/module/nats/stats/_meta/test/statsmetrics.json similarity index 100% rename from metricbeat/module/nats/_meta/test/statsmetrics.json rename to metricbeat/module/nats/stats/_meta/test/statsmetrics.json diff --git a/metricbeat/module/nats/stats/data.go b/metricbeat/module/nats/stats/data.go index a2d1f1fc325..14b9f4da479 100644 --- a/metricbeat/module/nats/stats/data.go +++ b/metricbeat/module/nats/stats/data.go @@ -33,6 +33,10 @@ import ( ) var ( + moduleSchema = s.Schema{ + "server_id": c.Str("server_id"), + "now": c.Str("now"), + } httpReqStatsSchema = s.Schema{ "root_uri": c.Int("/"), "connz_uri": c.Int("/connz"), @@ -41,9 +45,7 @@ var ( "varz_uri": c.Int("/varz"), } statsSchema = s.Schema{ - "server_id": c.Str("server_id"), - "now": c.Str("now"), - "uptime": c.Str("uptime"), + "uptime": c.Str("uptime"), "mem": s.Object{ "bytes": c.Int("mem"), }, @@ -139,7 +141,7 @@ func eventMapping(r mb.ReporterV2, content []byte) error { } event, err = statsSchema.Apply(inInterface) if err != nil { - err = errors.Wrap(err, "failure applying index schema") + err = errors.Wrap(err, "failure applying stats schema") r.Error(err) return err } @@ -192,6 +194,12 @@ func eventMapping(r mb.ReporterV2, content []byte) error { }, }, } - r.Event(mb.Event{MetricSetFields: event}) + moduleMetrics, err := moduleSchema.Apply(inInterface) + if err != nil { + err = errors.Wrap(err, "failure applying module schema") + r.Error(err) + return err + } + r.Event(mb.Event{MetricSetFields: event, ModuleFields: moduleMetrics}) return nil } diff --git a/metricbeat/module/nats/stats/stats_test.go b/metricbeat/module/nats/stats/stats_test.go index e96b7df9c2d..864f5e91385 100644 --- a/metricbeat/module/nats/stats/stats_test.go +++ b/metricbeat/module/nats/stats/stats_test.go @@ -30,19 +30,18 @@ import ( ) func TestEventMapping(t *testing.T) { - content, err := ioutil.ReadFile("../_meta/test/statsmetrics.json") + content, err := ioutil.ReadFile("./_meta/test/statsmetrics.json") assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} err = eventMapping(reporter, content) assert.NoError(t, err) - assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total_connections") assert.Equal(t, d, int64(35)) } func TestFetchEventContent(t *testing.T) { - absPath, _ := filepath.Abs("../_meta/test/") + absPath, _ := filepath.Abs("./_meta/test/") response, _ := ioutil.ReadFile(absPath + "/statsmetrics.json") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/metricbeat/modules.d/nats.yml.disabled b/metricbeat/modules.d/nats.yml.disabled index f3a318088d1..990c36f7fb1 100644 --- a/metricbeat/modules.d/nats.yml.disabled +++ b/metricbeat/modules.d/nats.yml.disabled @@ -2,7 +2,8 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-nats.html - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] - stats.metrics_path: "/varz" + #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" diff --git a/metricbeat/tests/system/test_nats.py b/metricbeat/tests/system/test_nats.py index 453214a082b..e0cb08d0100 100644 --- a/metricbeat/tests/system/test_nats.py +++ b/metricbeat/tests/system/test_nats.py @@ -34,6 +34,31 @@ def test_stats(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_connections(self): + """ + nats connections test + """ + self.render_config_template(modules=[{ + "name": "nats", + "metricsets": ["connections"], + "hosts": self.get_hosts(), + "period": "5s", + "connections.metrics_path": "/connz" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + def get_hosts(self): return ["{}:{}".format( os.getenv('NATS_HOST', 'localhost'), diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index e81e7ddd49d..a9a00a777c8 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -536,10 +536,11 @@ metricbeat.modules: #--------------------------------- Nats Module --------------------------------- - module: nats - metricsets: ["stats"] + metricsets: ["connections", "stats"] period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" + #connections.metrics_path: "/connz" #-------------------------------- Nginx Module -------------------------------- - module: nginx