Cherry-pick #7253 to 6.3: Metricbeat: Collect accumulated docker netw…

…ork metrics (#7363)

Cherry-pick of PR #7253 to 6.3 branch. Original message: 

Collect accumulated docker network metrics and mark previous metrics as deprecated.

Fixes #7240

CHANGELOG.asciidoc

@@ -293,7 +293,8 @@ https://github.com/elastic/beats/compare/v6.2.3...v6.3.0[View commits]
 - Added support for haproxy 1.7 and 1.8. {pull}6793[6793]
 - Add accumulated I/O stats to diskio in the line of `docker stats`. {pull}6701[6701]
 - Ignore virtual filesystem types by default in system module. {pull}6819[6819]
-- Release config reloading feature as GA.
+- Release config reloading feature as GA. {pull}6891[6891]
+- Collect accumulated docker network metrics and mark old ones as deprecated. {pull}7253[7253]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -3188,7 +3188,7 @@ Network interface name.
 [float]
 == in fields
 
-Incoming network stats.
+Incoming network stats per second.
 
 
 
@@ -3237,7 +3237,7 @@ Total number of incoming packets.
 [float]
 == out fields
 
-Outgoing network stats.
+Outgoing network stats per second.
 
 
 
@@ -3281,6 +3281,104 @@ type: long
 Total number of outgoing packets.
 
 
+--
+
+[float]
+== inbound fields
+
+Incoming network stats since the container started.
+
+
+
+*`docker.network.inbound.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of incoming bytes.
+
+
+--
+
+*`docker.network.inbound.dropped`*::
++
+--
+type: long
+
+Total number of dropped incoming packets.
+
+
+--
+
+*`docker.network.inbound.errors`*::
++
+--
+type: long
+
+Total errors on incoming packets.
+
+
+--
+
+*`docker.network.inbound.packets`*::
++
+--
+type: long
+
+Total number of incoming packets.
+
+
+--
+
+[float]
+== outbound fields
+
+Outgoing network stats since the container started.
+
+
+
+*`docker.network.outbound.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of outgoing bytes.
+
+
+--
+
+*`docker.network.outbound.dropped`*::
++
+--
+type: long
+
+Total number of dropped outgoing packets.
+
+
+--
+
+*`docker.network.outbound.errors`*::
++
+--
+type: long
+
+Total errors on outgoing packets.
+
+
+--
+
+*`docker.network.outbound.packets`*::
++
+--
+type: long
+
+Total number of outgoing packets.
+
+
 --
 
 [[exported-fields-dropwizard]]

metricbeat/mb/testing/data_generator.go

@@ -57,6 +57,28 @@ func WriteEvents(f mb.EventsFetcher, t testing.TB) error {
 	return nil
 }
 
+// WriteEventsReporterV2 fetches events and writes the first event to a ./_meta/data.json
+// file.
+func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error {
+	if !*dataFlag {
+		t.Skip("skip data generation tests")
+	}
+
+	events, errs := ReportingFetchV2(f)
+	if len(errs) > 0 {
+		return errs[0]
+	}
+
+	if len(events) == 0 {
+		return fmt.Errorf("no events were generated")
+	}
+
+	e := StandardizeEvent(f, events[0])
+
+	WriteEventToDataJSON(t, e)
+	return nil
+}
+
 // CreateFullEvent builds a full event given the data generated by a MetricSet.
 // This simulates the output of Metricbeat as if it were
 // 2016-05-23T08:05:34.853Z and the hostname is host.example.com.

metricbeat/module/docker/network/_meta/data.json

@@ -6,19 +6,16 @@
     },
     "docker": {
         "container": {
-            "id": "da57ef738524e5a4f6ae17b477f134d30719603db7b96d2c01b6f34010412e66",
+            "id": "452523bf833fd9fd1a8425135b720de4cb9b5a32096deac5b52a97e97bb6d16d",
             "labels": {
-                "build-date": "20170911",
-                "com_docker_compose_config-hash": "13a74b89a90a6fdcb6bbbc7eb37b7cb0615bdaf8",
+                "com_docker_compose_config-hash": "68a840a9e1c606ca1026492e50620e139ca342c585f330025a90f39a5fd32538",
                 "com_docker_compose_container-number": "1",
                 "com_docker_compose_oneoff": "False",
                 "com_docker_compose_project": "metricbeat",
                 "com_docker_compose_service": "elasticsearch",
-                "com_docker_compose_version": "1.5.0",
-                "license": "GPLv2",
+                "com_docker_compose_version": "1.21.0",
                 "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e",
-                "name": "CentOS Base Image",
-                "vendor": "CentOS"
+                "org_label-schema_schema-version": "= 1.0     org.label-schema.name=CentOS Base Image     org.label-schema.vendor=CentOS     org.label-schema.license=GPLv2     org.label-schema.build-date=20180402"
             },
             "name": "metricbeat_elasticsearch_1"
         },
@@ -29,12 +26,24 @@
                 "errors": 0,
                 "packets": 0
             },
+            "inbound": {
+                "bytes": 61694097,
+                "dropped": 0,
+                "errors": 0,
+                "packets": 714036
+            },
             "interface": "eth0",
             "out": {
                 "bytes": 0,
                 "dropped": 0,
                 "errors": 0,
                 "packets": 0
+            },
+            "outbound": {
+                "bytes": 69114459,
+                "dropped": 0,
+                "errors": 0,
+                "packets": 713985
             }
         }
     },
@@ -44,4 +53,4 @@
         "name": "network",
         "rtt": 115
     }
-}
+}

metricbeat/module/docker/network/_meta/fields.yml

@@ -11,8 +11,9 @@
         Network interface name.
     - name: in
       type: group
+      deprecated: true
       description: >
-        Incoming network stats.
+        Incoming network stats per second.
       fields:
         - name: bytes
           type: long
@@ -33,8 +34,9 @@
             Total number of incoming packets.
     - name: out
       type: group
+      deprecated: true
       description: >
-        Outgoing network stats.
+        Outgoing network stats per second.
       fields:
         - name: bytes
           type: long
@@ -53,3 +55,49 @@
           type: long
           description: >
             Total number of outgoing packets.
+    - name: inbound
+      type: group
+      deprecated: true
+      description: >
+        Incoming network stats since the container started.
+      fields:
+        - name: bytes
+          type: long
+          format: bytes
+          description: >
+            Total number of incoming bytes.
+        - name: dropped
+          type: long
+          description: >
+            Total number of dropped incoming packets.
+        - name: errors
+          type: long
+          description: >
+            Total errors on incoming packets.
+        - name: packets
+          type: long
+          description: >
+            Total number of incoming packets.
+    - name: outbound
+      type: group
+      deprecated: true
+      description: >
+        Outgoing network stats since the container started.
+      fields:
+        - name: bytes
+          type: long
+          format: bytes
+          description: >
+            Total number of outgoing bytes.
+        - name: dropped
+          type: long
+          description: >
+            Total number of dropped outgoing packets.
+        - name: errors
+          type: long
+          description: >
+            Total errors on outgoing packets.
+        - name: packets
+          type: long
+          description: >
+            Total number of outgoing packets.

metricbeat/module/docker/network/data.go

@@ -5,32 +5,44 @@ import (
 	"github.com/elastic/beats/metricbeat/mb"
 )
 
-func eventsMapping(netsStatsList []NetStats) []common.MapStr {
-	myEvents := []common.MapStr{}
+func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) {
 	for _, netsStats := range netsStatsList {
-		myEvents = append(myEvents, eventMapping(&netsStats))
+		eventMapping(r, &netsStats)
 	}
-	return myEvents
 }
 
-func eventMapping(stats *NetStats) common.MapStr {
-	event := common.MapStr{
-		mb.ModuleDataKey: common.MapStr{
+func eventMapping(r mb.ReporterV2, stats *NetStats) {
+	// Deprecated fields
+	r.Event(mb.Event{
+		ModuleFields: common.MapStr{
 			"container": stats.Container.ToMapStr(),
 		},
-		"interface": stats.NameInterface,
-		"in": common.MapStr{
-			"bytes":   stats.RxBytes,
-			"dropped": stats.RxDropped,
-			"errors":  stats.RxErrors,
-			"packets": stats.RxPackets,
+		MetricSetFields: common.MapStr{
+			"interface": stats.NameInterface,
+			"in": common.MapStr{
+				"bytes":   stats.RxBytes,
+				"dropped": stats.RxDropped,
+				"errors":  stats.RxErrors,
+				"packets": stats.RxPackets,
+			},
+			"out": common.MapStr{
+				"bytes":   stats.TxBytes,
+				"dropped": stats.TxDropped,
+				"errors":  stats.TxErrors,
+				"packets": stats.TxPackets,
+			},
+			"inbound": common.MapStr{
+				"bytes":   stats.Total.RxBytes,
+				"dropped": stats.Total.RxDropped,
+				"errors":  stats.Total.RxErrors,
+				"packets": stats.Total.RxPackets,
+			},
+			"outbound": common.MapStr{
+				"bytes":   stats.Total.TxBytes,
+				"dropped": stats.Total.TxDropped,
+				"errors":  stats.Total.TxErrors,
+				"packets": stats.Total.TxPackets,
+			},
 		},
-		"out": common.MapStr{
-			"bytes":   stats.TxBytes,
-			"dropped": stats.TxDropped,
-			"errors":  stats.TxErrors,
-			"packets": stats.TxPackets,
-		},
-	}
-	return event
+	})
 }

metricbeat/module/docker/network/helper.go

@@ -47,6 +47,7 @@ type NetStats struct {
 	TxDropped     float64
 	TxErrors      float64
 	TxPackets     float64
+	Total         *types.NetworkStats
 }
 
 func (n *NetService) getNetworkStatsPerContainer(rawStats []docker.Stat, dedot bool) []NetStats {
@@ -68,6 +69,7 @@ func (n *NetService) getNetworkStats(nameInterface string, rawNetStats *types.Ne
 		Container:     docker.NewContainer(myRawstats.Container, dedot),
 		Time:          myRawstats.Stats.Read,
 		NameInterface: nameInterface,
+		Total:         rawNetStats,
 	}
 
 	if exist {

metricbeat/module/docker/network/network.go

@@ -3,7 +3,6 @@ package network
 import (
 	"github.com/docker/docker/client"
 
-	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/metricbeat/module/docker"
 )
@@ -45,12 +44,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 }
 
 // Fetch methods creates a list of network events for each container.
-func (m *MetricSet) Fetch() ([]common.MapStr, error) {
+func (m *MetricSet) Fetch(r mb.ReporterV2) {
 	stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout)
 	if err != nil {
-		return nil, err
+		r.Error(err)
+		return
 	}
 
 	formattedStats := m.netService.getNetworkStatsPerContainer(stats, m.dedot)
-	return eventsMapping(formattedStats), nil
+	eventsMapping(r, formattedStats)
 }