diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ddb6ca6d5b3..11ceeb914be 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -293,7 +293,8 @@ https://github.com/elastic/beats/compare/v6.2.3...v6.3.0[View commits] - Added support for haproxy 1.7 and 1.8. {pull}6793[6793] - Add accumulated I/O stats to diskio in the line of `docker stats`. {pull}6701[6701] - Ignore virtual filesystem types by default in system module. {pull}6819[6819] -- Release config reloading feature as GA. +- Release config reloading feature as GA. {pull}6891[6891] +- Collect accumulated docker network metrics and mark old ones as deprecated. {pull}7253[7253] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 80b77f0d161..4c0763fa9fb 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3188,7 +3188,7 @@ Network interface name. [float] == in fields -Incoming network stats. +Incoming network stats per second. @@ -3237,7 +3237,7 @@ Total number of incoming packets. [float] == out fields -Outgoing network stats. +Outgoing network stats per second. @@ -3281,6 +3281,104 @@ type: long Total number of outgoing packets. +-- + +[float] +== inbound fields + +Incoming network stats since the container started. + + + +*`docker.network.inbound.bytes`*:: ++ +-- +type: long + +format: bytes + +Total number of incoming bytes. + + +-- + +*`docker.network.inbound.dropped`*:: ++ +-- +type: long + +Total number of dropped incoming packets. + + +-- + +*`docker.network.inbound.errors`*:: ++ +-- +type: long + +Total errors on incoming packets. + + +-- + +*`docker.network.inbound.packets`*:: ++ +-- +type: long + +Total number of incoming packets. + + +-- + +[float] +== outbound fields + +Outgoing network stats since the container started. + + + +*`docker.network.outbound.bytes`*:: ++ +-- +type: long + +format: bytes + +Total number of outgoing bytes. + + +-- + +*`docker.network.outbound.dropped`*:: ++ +-- +type: long + +Total number of dropped outgoing packets. + + +-- + +*`docker.network.outbound.errors`*:: ++ +-- +type: long + +Total errors on outgoing packets. + + +-- + +*`docker.network.outbound.packets`*:: ++ +-- +type: long + +Total number of outgoing packets. + + -- [[exported-fields-dropwizard]] diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go index 4af3965d1cc..8c38da39a30 100644 --- a/metricbeat/mb/testing/data_generator.go +++ b/metricbeat/mb/testing/data_generator.go @@ -57,6 +57,28 @@ func WriteEvents(f mb.EventsFetcher, t testing.TB) error { return nil } +// WriteEventsReporterV2 fetches events and writes the first event to a ./_meta/data.json +// file. +func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error { + if !*dataFlag { + t.Skip("skip data generation tests") + } + + events, errs := ReportingFetchV2(f) + if len(errs) > 0 { + return errs[0] + } + + if len(events) == 0 { + return fmt.Errorf("no events were generated") + } + + e := StandardizeEvent(f, events[0]) + + WriteEventToDataJSON(t, e) + return nil +} + // CreateFullEvent builds a full event given the data generated by a MetricSet. // This simulates the output of Metricbeat as if it were // 2016-05-23T08:05:34.853Z and the hostname is host.example.com. diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index f7b4524cdce..ddb3c827a8a 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -6,19 +6,16 @@ }, "docker": { "container": { - "id": "da57ef738524e5a4f6ae17b477f134d30719603db7b96d2c01b6f34010412e66", + "id": "452523bf833fd9fd1a8425135b720de4cb9b5a32096deac5b52a97e97bb6d16d", "labels": { - "build-date": "20170911", - "com_docker_compose_config-hash": "13a74b89a90a6fdcb6bbbc7eb37b7cb0615bdaf8", + "com_docker_compose_config-hash": "68a840a9e1c606ca1026492e50620e139ca342c585f330025a90f39a5fd32538", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.5.0", - "license": "GPLv2", + "com_docker_compose_version": "1.21.0", "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e", - "name": "CentOS Base Image", - "vendor": "CentOS" + "org_label-schema_schema-version": "= 1.0 org.label-schema.name=CentOS Base Image org.label-schema.vendor=CentOS org.label-schema.license=GPLv2 org.label-schema.build-date=20180402" }, "name": "metricbeat_elasticsearch_1" }, @@ -29,12 +26,24 @@ "errors": 0, "packets": 0 }, + "inbound": { + "bytes": 61694097, + "dropped": 0, + "errors": 0, + "packets": 714036 + }, "interface": "eth0", "out": { "bytes": 0, "dropped": 0, "errors": 0, "packets": 0 + }, + "outbound": { + "bytes": 69114459, + "dropped": 0, + "errors": 0, + "packets": 713985 } } }, @@ -44,4 +53,4 @@ "name": "network", "rtt": 115 } -} \ No newline at end of file +} diff --git a/metricbeat/module/docker/network/_meta/fields.yml b/metricbeat/module/docker/network/_meta/fields.yml index cb10ad1b482..f66b8492e13 100644 --- a/metricbeat/module/docker/network/_meta/fields.yml +++ b/metricbeat/module/docker/network/_meta/fields.yml @@ -11,8 +11,9 @@ Network interface name. - name: in type: group + deprecated: true description: > - Incoming network stats. + Incoming network stats per second. fields: - name: bytes type: long @@ -33,8 +34,9 @@ Total number of incoming packets. - name: out type: group + deprecated: true description: > - Outgoing network stats. + Outgoing network stats per second. fields: - name: bytes type: long @@ -53,3 +55,49 @@ type: long description: > Total number of outgoing packets. + - name: inbound + type: group + deprecated: true + description: > + Incoming network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of incoming bytes. + - name: dropped + type: long + description: > + Total number of dropped incoming packets. + - name: errors + type: long + description: > + Total errors on incoming packets. + - name: packets + type: long + description: > + Total number of incoming packets. + - name: outbound + type: group + deprecated: true + description: > + Outgoing network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of outgoing bytes. + - name: dropped + type: long + description: > + Total number of dropped outgoing packets. + - name: errors + type: long + description: > + Total errors on outgoing packets. + - name: packets + type: long + description: > + Total number of outgoing packets. diff --git a/metricbeat/module/docker/network/data.go b/metricbeat/module/docker/network/data.go index 0844e276d6f..bb4bc9ff623 100644 --- a/metricbeat/module/docker/network/data.go +++ b/metricbeat/module/docker/network/data.go @@ -5,32 +5,44 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func eventsMapping(netsStatsList []NetStats) []common.MapStr { - myEvents := []common.MapStr{} +func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) { for _, netsStats := range netsStatsList { - myEvents = append(myEvents, eventMapping(&netsStats)) + eventMapping(r, &netsStats) } - return myEvents } -func eventMapping(stats *NetStats) common.MapStr { - event := common.MapStr{ - mb.ModuleDataKey: common.MapStr{ +func eventMapping(r mb.ReporterV2, stats *NetStats) { + // Deprecated fields + r.Event(mb.Event{ + ModuleFields: common.MapStr{ "container": stats.Container.ToMapStr(), }, - "interface": stats.NameInterface, - "in": common.MapStr{ - "bytes": stats.RxBytes, - "dropped": stats.RxDropped, - "errors": stats.RxErrors, - "packets": stats.RxPackets, + MetricSetFields: common.MapStr{ + "interface": stats.NameInterface, + "in": common.MapStr{ + "bytes": stats.RxBytes, + "dropped": stats.RxDropped, + "errors": stats.RxErrors, + "packets": stats.RxPackets, + }, + "out": common.MapStr{ + "bytes": stats.TxBytes, + "dropped": stats.TxDropped, + "errors": stats.TxErrors, + "packets": stats.TxPackets, + }, + "inbound": common.MapStr{ + "bytes": stats.Total.RxBytes, + "dropped": stats.Total.RxDropped, + "errors": stats.Total.RxErrors, + "packets": stats.Total.RxPackets, + }, + "outbound": common.MapStr{ + "bytes": stats.Total.TxBytes, + "dropped": stats.Total.TxDropped, + "errors": stats.Total.TxErrors, + "packets": stats.Total.TxPackets, + }, }, - "out": common.MapStr{ - "bytes": stats.TxBytes, - "dropped": stats.TxDropped, - "errors": stats.TxErrors, - "packets": stats.TxPackets, - }, - } - return event + }) } diff --git a/metricbeat/module/docker/network/helper.go b/metricbeat/module/docker/network/helper.go index 6df1b8e969e..cc22146bd84 100644 --- a/metricbeat/module/docker/network/helper.go +++ b/metricbeat/module/docker/network/helper.go @@ -47,6 +47,7 @@ type NetStats struct { TxDropped float64 TxErrors float64 TxPackets float64 + Total *types.NetworkStats } func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, dedot bool) []NetStats { @@ -68,6 +69,7 @@ func (n *NetService) getNetworkStats(nameInterface string, rawNetStats *types.Ne Container: docker.NewContainer(myRawstats.Container, dedot), Time: myRawstats.Stats.Read, NameInterface: nameInterface, + Total: rawNetStats, } if exist { diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index 5915b465d63..568bf6fb4d9 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -3,7 +3,6 @@ package network import ( "github.com/docker/docker/client" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) @@ -45,12 +44,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch methods creates a list of network events for each container. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { - return nil, err + r.Error(err) + return } formattedStats := m.netService.getNetworkStatsPerContainer(stats, m.dedot) - return eventsMapping(formattedStats), nil + eventsMapping(r, formattedStats) } diff --git a/metricbeat/module/docker/network/network_integration_test.go b/metricbeat/module/docker/network/network_integration_test.go index 55bdfe4fa11..fa380e20e34 100644 --- a/metricbeat/module/docker/network/network_integration_test.go +++ b/metricbeat/module/docker/network/network_integration_test.go @@ -9,8 +9,8 @@ import ( ) func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) + ms := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(ms, t) if err != nil { t.Fatal("write", err) }