auditbeat,filebeat,packetbeat: format mac addresses according to ECS (#…

…32622)

* packetbeat,x-pack/{auditbeat,filebeat}: make MAC addresses render in ECS format
* x-pack/filebeat/processors/decode_cef: canonicalise MAC addresses for ECS fields

    This retains the original format in the cef.extension fields but ensures
    that the MAC address format used in ECS fields matches the spec.

CHANGELOG.next.asciidoc

@@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - auditd module: Fix parsing of audit rules where arguments are quoted (like file paths containing spaces). {pull}32421[32421]
 - auditd module: Fix minimum AuditStatus length so that library can support kernels from 2.6.32. {pull}32421[32421]
 - system/socket: Reduce memory usage of the dataset. {issue}32191[32191] {pull}32192[32192]
+- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622]
 
 *Filebeat*
 
@@ -63,6 +64,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - gcp-pubsub input: Restart Pub/Sub client on all errors. {issue}32550[32550] {pull}32712[32712]
 - Update `cloud.region` parsing in cloudtrail fileset. {pull}32763[32763]
 - Fix file.path field in cloudtrail fileset to use json.digestS3Object. {pull}32759[32759]
+- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622]
 
 *Heartbeat*
 
@@ -85,6 +87,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Packetbeat*
 
 - Fix formatting of debug logs. {pull}32698[32698]
+- Fix rendering of MAC addresses to conform to ECS. {issue}32621[32621] {pull}32622[32622]
 
 *Winlogbeat*
 

packetbeat/_meta/sample_outputs/flow.json

@@ -37,15 +37,15 @@
             "start": "2018-11-30T01:15:50.119Z",
             "pid": 3468
         },
-        "mac": "08:00:27:d5:9d:5a",
+        "mac": "08-00-27-D5-9D-5A",
         "ip": "10.0.2.15",
         "port": 23377,
         "packets": 12
     },
     "destination": {
         "packets": 16,
         "bytes": 5558,
-        "mac": "52:54:00:12:35:02",
+        "mac": "52-54-00-12-35-02",
         "ip": "130.211.38.145",
         "port": 443
     },

packetbeat/docs/packetbeat-options.asciidoc

@@ -340,7 +340,7 @@ Here’s an example of a flow information sent by Packetbeat. See
   "destination": {
     "bytes": 460,
     "ip": "198.51.100.2",
-    "mac": "06:05:04:03:02:01",
+    "mac": "06-05-04-03-02-01",
     "packets": 2,
     "port": 80
   },
@@ -365,7 +365,7 @@ Here’s an example of a flow information sent by Packetbeat. See
   "source": {
     "bytes": 10,
     "ip": "203.0.113.3",
-    "mac": "01:02:03:04:05:06",
+    "mac": "01-02-03-04-05-06",
     "packets": 1,
     "port": 38901
   }

packetbeat/flows/flows_test.go

@@ -141,8 +141,8 @@ func TestFlowsCounting(t *testing.T) {
 	network := event["network"].(mapstr.M)
 
 	// validate generated event
-	assert.Equal(t, net.HardwareAddr(mac1).String(), source["mac"])
-	assert.Equal(t, net.HardwareAddr(mac2).String(), dest["mac"])
+	assert.Equal(t, formatHardwareAddr(net.HardwareAddr(mac1)), source["mac"])
+	assert.Equal(t, formatHardwareAddr(net.HardwareAddr(mac2)), dest["mac"])
 	assert.Equal(t, net.IP(ip1).String(), source["ip"])
 	assert.Equal(t, net.IP(ip2).String(), dest["ip"])
 	assert.Equal(t, uint16(256), source["port"])

packetbeat/flows/worker.go

@@ -323,8 +323,8 @@ func createEvent(watcher procs.ProcessesWatcher, ts time.Time, f *biFlow, isOver
 
 	// add ethernet layer meta data
 	if src, dst, ok := f.id.EthAddr(); ok {
-		source["mac"] = net.HardwareAddr(src).String()
-		dest["mac"] = net.HardwareAddr(dst).String()
+		source["mac"] = formatHardwareAddr(net.HardwareAddr(src))
+		dest["mac"] = formatHardwareAddr(net.HardwareAddr(dst))
 	}
 
 	// add vlan
@@ -520,6 +520,19 @@ func createEvent(watcher procs.ProcessesWatcher, ts time.Time, f *biFlow, isOver
 	}
 }
 
+// formatHardwareAddr formats hardware addresses according to the ECS spec.
+func formatHardwareAddr(addr net.HardwareAddr) string {
+	buf := make([]byte, 0, len(addr)*3-1)
+	for _, b := range addr {
+		if len(buf) != 0 {
+			buf = append(buf, '-')
+		}
+		const hexDigit = "0123456789ABCDEF"
+		buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf])
+	}
+	return string(buf)
+}
+
 func encodeStats(stats *flowStats, ints, uints, floats []string) map[string]interface{} {
 	report := make(map[string]interface{})
 

packetbeat/flows/worker_test.go

@@ -70,14 +70,14 @@ func TestCreateEvent(t *testing.T) {
 	// Validate the contents of the event.
 	validate := lookslike.MustCompile(map[string]interface{}{
 		"source": map[string]interface{}{
-			"mac":     "01:02:03:04:05:06",
+			"mac":     "01-02-03-04-05-06",
 			"ip":      "203.0.113.3",
 			"port":    port1,
 			"bytes":   uint64(10),
 			"packets": uint64(1),
 		},
 		"destination": map[string]interface{}{
-			"mac":     "06:05:04:03:02:01",
+			"mac":     "06-05-04-03-02-01",
 			"ip":      "198.51.100.2",
 			"port":    port2,
 			"bytes":   uint64(460),
@@ -116,7 +116,7 @@ func TestCreateEvent(t *testing.T) {
 
 	// Write the event to disk if -data is used.
 	if *dataFlag {
-		_, _ = event.Fields.Put("@timestamp", common.Time(end))
+		event.Fields.Put("@timestamp", common.Time(end)) //nolint:errcheck // Never fails.
 		output, err := json.MarshalIndent(&event.Fields, "", "  ")
 		if err != nil {
 			t.Fatal(err)

packetbeat/protos/dhcpv4/dhcpv4.go

@@ -15,10 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//nolint:errcheck // All complaints are about mapstr.M puts.
 package dhcpv4
 
 import (
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/insomniacslk/dhcp/dhcpv4"
@@ -126,14 +128,15 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event {
 	fields["type"] = pbf.Event.Dataset
 	fields["status"] = "OK"
 
+	mac16 := v4.ClientHwAddr()
 	dhcpData := mapstr.M{
 		"op_code":        strings.ToLower(v4.OpcodeToString()),
 		"hardware_type":  v4.HwTypeToString(),
 		"hops":           v4.HopCount(), // Set to non-zero by relays.
 		"transaction_id": fmt.Sprintf("0x%08x", v4.TransactionID()),
 		"seconds":        v4.NumSeconds(),
 		"flags":          strings.ToLower(v4.FlagsToString()),
-		"client_mac":     v4.ClientHwAddrToString(),
+		"client_mac":     formatHardwareAddr(net.HardwareAddr(mac16[:v4.HwAddrLen()])),
 	}
 	fields["dhcpv4"] = dhcpData
 
@@ -160,8 +163,21 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event {
 		p.log.Warnw("Failed converting DHCP options to map",
 			"dhcpv4", v4, "error", err)
 	} else if len(opts) > 0 {
-		dhcpData.Put("option", opts)
+		_, _ = dhcpData.Put("option", opts)
 	}
 
 	return &evt
 }
+
+// formatHardwareAddr formats hardware addresses according to the ECS spec.
+func formatHardwareAddr(addr net.HardwareAddr) string {
+	buf := make([]byte, 0, len(addr)*3-1)
+	for _, b := range addr {
+		if len(buf) != 0 {
+			buf = append(buf, '-')
+		}
+		const hexDigit = "0123456789ABCDEF"
+		buf = append(buf, hexDigit[b>>4], hexDigit[b&0xf])
+	}
+	return string(buf)
+}

packetbeat/protos/dhcpv4/dhcpv4_test.go

@@ -82,7 +82,7 @@ var (
 )
 
 func TestParseDHCPRequest(t *testing.T) {
-	logp.TestingSetup()
+	_ = logp.TestingSetup()
 	p, err := newPlugin(true, nil, procs.ProcessesWatcher{}, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -137,7 +137,7 @@ func TestParseDHCPRequest(t *testing.T) {
 				"ip": []string{"0.0.0.0", "255.255.255.255"},
 			},
 			"dhcpv4": mapstr.M{
-				"client_mac":     "00:0b:82:01:fc:42",
+				"client_mac":     "00-0B-82-01-FC-42",
 				"flags":          "unicast",
 				"hardware_type":  "Ethernet",
 				"hops":           0,
@@ -161,7 +161,8 @@ func TestParseDHCPRequest(t *testing.T) {
 
 	actual := p.parseDHCPv4(pkt)
 	if assert.NotNil(t, actual) {
-		publish.MarshalPacketbeatFields(actual, nil, nil)
+		_, err := publish.MarshalPacketbeatFields(actual, nil, nil)
+		assert.NoError(t, err, "marshalling packet beat fields")
 		t.Logf("DHCP event: %+v", actual)
 		assertEqual(t, expected, *actual)
 	}
@@ -223,7 +224,7 @@ func TestParseDHCPACK(t *testing.T) {
 			},
 			"dhcpv4": mapstr.M{
 				"assigned_ip":    "192.168.0.10",
-				"client_mac":     "00:0b:82:01:fc:42",
+				"client_mac":     "00-0B-82-01-FC-42",
 				"flags":          "unicast",
 				"hardware_type":  "Ethernet",
 				"hops":           0,
@@ -244,7 +245,8 @@ func TestParseDHCPACK(t *testing.T) {
 
 	actual := p.parseDHCPv4(pkt)
 	if assert.NotNil(t, actual) {
-		publish.MarshalPacketbeatFields(actual, nil, nil)
+		_, err := publish.MarshalPacketbeatFields(actual, nil, nil)
+		assert.NoError(t, err, "marshalling packet beat fields")
 		t.Logf("DHCP event: %+v", actual)
 		assertEqual(t, expected, *actual)
 	}

packetbeat/tests/system/test_0060_flows.py

@@ -43,8 +43,8 @@ def test_mysql_flow(self):
         assert len(objs) == 1
         check_fields(objs[0], {
             'flow.final': True,
-            'source.mac': '0a:00:27:00:00:00',
-            'destination.mac': '08:00:27:76:d7:41',
+            'source.mac': '0A-00-27-00-00-00',
+            'destination.mac': '08-00-27-76-D7-41',
             'destination.ip': '192.168.33.14',
             'source.ip': '192.168.33.1',
             'network.transport': 'tcp',
@@ -78,8 +78,8 @@ def test_memcache_udp_flow(self):
         assert len(objs) == 1
         check_fields(objs[0], {
             'flow.final': True,
-            'source.mac': 'ac:bc:32:77:41:0b',
-            'destination.mac': '08:00:27:dd:3b:28',
+            'source.mac': 'AC-BC-32-77-41-0B',
+            'destination.mac': '08-00-27-DD-3B-28',
             'source.ip': '192.168.188.37',
             'destination.ip': '192.168.188.38',
             'network.transport': 'udp',
@@ -106,8 +106,8 @@ def test_icmp4_ping(self):
         assert len(objs) == 1
         check_fields(objs[0], {
             'flow.final': True,
-            'source.mac': '00:00:00:00:00:01',
-            'destination.mac': '00:00:00:00:00:02',
+            'source.mac': '00-00-00-00-00-01',
+            'destination.mac': '00-00-00-00-00-02',
             'flow.vlan': 10,
             'source.ip': '10.0.0.1',
             'destination.ip': '10.0.0.2',
@@ -135,11 +135,11 @@ def test_icmp6_ping(self):
         check_fields(objs[0], {
             'flow.final': True,
             'flow.vlan': 10,
-            'source.mac': '00:00:00:00:00:01',
+            'source.mac': '00-00-00-00-00-01',
             'source.ip': '::1',
             'source.bytes': 70,
             'source.packets': 1,
-            'destination.mac': '00:00:00:00:00:02',
+            'destination.mac': '00-00-00-00-00-02',
             'destination.ip': '::2',
             'destination.bytes': 70,
             'destination.packets': 1,
@@ -168,8 +168,8 @@ def test_q_in_q_flow(self):
             'source.ip': '192.168.1.1',
             'source.bytes': 82,
             'source.packets': 1,
-            'source.mac': '08:00:27:3d:25:4e',
-            'destination.mac': '1c:af:f7:70:ed:7c',
+            'source.mac': '08-00-27-3D-25-4E',
+            'destination.mac': '1C-AF-F7-70-ED-7C',
             'destination.ip': '192.168.1.2',
             'network.bytes': 82,
             'network.packets': 1,

packetbeat/tests/system/test_0066_dhcp.py

@@ -18,7 +18,7 @@ def test_dhcp(self):
         assert objs[0]["client.ip"] == "0.0.0.0"
         assert objs[0]["client.port"] == 68
         assert objs[0]["destination.ip"] == "255.255.255.255"
-        assert objs[0]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42"
+        assert objs[0]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42"
         assert objs[0]["dhcpv4.flags"] == "unicast"
         assert objs[0]["dhcpv4.hardware_type"] == "Ethernet"
         assert objs[0]["dhcpv4.hops"] == 0
@@ -50,7 +50,7 @@ def test_dhcp(self):
         assert objs[1]["client.port"] == 68
         assert objs[1]["destination.ip"] == "192.168.0.10"
         assert objs[1]["dhcpv4.assigned_ip"] == "192.168.0.10"
-        assert objs[1]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42"
+        assert objs[1]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42"
         assert objs[1]["dhcpv4.flags"] == "unicast"
         assert objs[1]["dhcpv4.hardware_type"] == "Ethernet"
         assert objs[1]["dhcpv4.hops"] == 0
@@ -79,7 +79,7 @@ def test_dhcp(self):
         assert "event.start" in objs[2]
         assert objs[2]["client.ip"] == "0.0.0.0"
         assert objs[2]["client.port"] == 68
-        assert objs[2]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42"
+        assert objs[2]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42"
         assert objs[2]["dhcpv4.flags"] == "unicast"
         assert objs[2]["dhcpv4.hardware_type"] == "Ethernet"
         assert objs[2]["dhcpv4.hops"] == 0
@@ -111,7 +111,7 @@ def test_dhcp(self):
         assert objs[3]["client.port"] == 68
         assert objs[3]["destination.ip"] == "192.168.0.10"
         assert objs[3]["dhcpv4.assigned_ip"] == "192.168.0.10"
-        assert objs[3]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42"
+        assert objs[3]["dhcpv4.client_mac"] == "00-0B-82-01-FC-42"
         assert objs[3]["dhcpv4.flags"] == "unicast"
         assert objs[3]["dhcpv4.hardware_type"] == "Ethernet"
         assert objs[3]["dhcpv4.hops"] == 0

x-pack/auditbeat/module/system/host/_meta/data.json

@@ -33,10 +33,10 @@
                     "fe80::42:9eff:fed3:d888"
                 ],
                 "mac": [
-                    "02:2d:fd:81:e7:47",
-                    "08:00:27:1f:71:60",
-                    "02:42:83:be:1a:3a",
-                    "02:42:9e:d3:d8:88"
+                    "02-2D-FD-81-E7-47",
+                    "08-00-27-1F-71-60",
+                    "02-42-83-BE-1A-3A",
+                    "02-42-9E-D3-D8-88"
                 ],
                 "os": {
                     "family": "debian",