Update DHCPv4 protocol to use ECS fields

The DHCPv4 protocol dataset works on uni-flows (it's not transacted, see #7956) so the `source` and `destination` will indicate the original packet header data. Meanwhile the `client` / `server` fields are copies of the `source`/`destination`, but they are copied based on which side is the client and server. Here's a summary of what fields changed. Part of #7968 Changed - bytes_in -> source.bytes - transport -> network.transport = udp Added - source - destination - event.dataset = dhcpv4 - event.start - network.bytes - network.community_id - network.protocol = dhcpv4 - network.type Unchanged Packetbeat Fields - status - type = dhcpv4 (we might remove this since we have event.dataset)
elastic · Jan 16, 2019 · 2123e0e · 2123e0e
1 parent 42a3dbb
commit 2123e0e
Show file tree

Hide file tree

Showing 5 changed files with 167 additions and 68 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -59,6 +59,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Removed trailing dot from domain names reported by the DNS protocol. {pull}9941[9941]
 - Changed TLS protocol fields to align with ECS. {pull}9980[9980]
 - Changed ICMP protocol fields to align with ECS. {pull}10062[10062]
+- Changed DHCPv4 protocol fields to align with ECS. {pull}10089[10089]
 
 *Winlogbeat*
 

diff --git a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json
@@ -104,7 +104,7 @@
       },
       "id": "8460fcd0-8baa-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
+      "updated_at": "2019-01-15T18:40:16.104Z",
       "version": 1
     },
     {
@@ -167,7 +167,7 @@
       },
       "id": "4ad9db20-8bab-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
+      "updated_at": "2019-01-15T18:40:16.104Z",
       "version": 1
     },
     {
@@ -249,7 +249,7 @@
       },
       "id": "418dfbe0-8bac-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
+      "updated_at": "2019-01-15T18:40:16.104Z",
       "version": 1
     },
     {
@@ -258,7 +258,8 @@
           "dhcpv4.transaction_id",
           "dhcpv4.op_code",
           "dhcpv4.option.message_type",
-          "client_ip",
+          "source.ip",
+          "destination.ip",
           "dhcpv4.client_mac",
           "dhcpv4.option.hostname",
           "dhcpv4.option.class_identifier"
@@ -276,7 +277,7 @@
                   "alias": null,
                   "disabled": false,
                   "index": "packetbeat-*",
-                  "key": "type",
+                  "key": "event.dataset",
                   "negate": false,
                   "params": {
                     "query": "dhcpv4",
@@ -287,7 +288,7 @@
                 },
                 "query": {
                   "match": {
-                    "type": {
+                    "event.dataset": {
                       "query": "dhcpv4",
                       "type": "phrase"
                     }
@@ -313,8 +314,8 @@
       },
       "id": "b8992150-8ba8-11e8-9676-ef67484126fb",
       "type": "search",
-      "updated_at": "2019-01-04T14:55:16.796Z",
-      "version": 1
+      "updated_at": "2019-01-15T18:46:52.929Z",
+      "version": 4
     },
     {
       "attributes": {
@@ -379,7 +380,7 @@
       },
       "id": "d0120dc0-8bac-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
+      "updated_at": "2019-01-15T18:40:16.104Z",
       "version": 1
     },
     {
@@ -445,7 +446,7 @@
       },
       "id": "11d33ea0-8bad-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
+      "updated_at": "2019-01-15T18:40:16.104Z",
       "version": 1
     },
     {
@@ -471,7 +472,7 @@
               "id": "1",
               "params": {
                 "customLabel": "Requests",
-                "field": "bytes_in"
+                "field": "client.bytes"
               },
               "schema": "metric",
               "type": "sum"
@@ -481,7 +482,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Responses",
-                "field": "bytes_out"
+                "field": "server.bytes"
               },
               "schema": "metric",
               "type": "sum"
@@ -521,8 +522,8 @@
       },
       "id": "f43a8f20-8bb5-11e8-9676-ef67484126fb",
       "type": "visualization",
-      "updated_at": "2019-01-04T14:55:16.796Z",
-      "version": 1
+      "updated_at": "2019-01-15T18:45:46.912Z",
+      "version": 2
     },
     {
       "attributes": {
@@ -650,9 +651,9 @@
       },
       "id": "a7b35890-8baa-11e8-9676-ef67484126fb",
       "type": "dashboard",
-      "updated_at": "2019-01-04T15:03:10.809Z",
+      "updated_at": "2019-01-15T18:46:22.620Z",
       "version": 2
     }
   ],
-  "version": "6.6.0-SNAPSHOT"
-}
+  "version": "7.0.0-SNAPSHOT"
+}
diff --git a/packetbeat/protos/dhcpv4/dhcpv4.go b/packetbeat/protos/dhcpv4/dhcpv4.go
@@ -27,7 +27,9 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/monitoring"
+	"github.com/elastic/beats/packetbeat/pb"
 	"github.com/elastic/beats/packetbeat/protos"
+	"github.com/elastic/ecs/code/go/ecs"
 )
 
 var (
@@ -86,10 +88,38 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event {
 	v4, err := dhcpv4.FromBytes(pkt.Payload)
 	if err != nil {
 		metricParseFailures.Inc()
-		p.log.Warnw("dropping packet: failed parsing DHCP data", "error", err)
+		p.log.Warnw("Dropping packet: failed parsing DHCP data", "error", err)
 		return nil
 	}
 
+	evt, pbf := pb.NewBeatEvent(pkt.Ts)
+
+	// source/destination (note: this protocol does not produce a bi-flow.)
+	src, dst := common.MakeEndpointPair(pkt.Tuple.BaseTuple, nil)
+	pbf.SetSource(&src)
+	pbf.SetDestination(&dst)
+	pbf.Source.Bytes = int64(len(pkt.Payload))
+
+	if v4.Opcode() == dhcpv4.OpcodeBootReply {
+		// Reverse
+		client, server := ecs.Client(*pbf.Destination), ecs.Server(*pbf.Source)
+		pbf.Client = &client
+		pbf.Server = &server
+	} else {
+		client, server := ecs.Client(*pbf.Source), ecs.Server(*pbf.Destination)
+		pbf.Client = &client
+		pbf.Server = &server
+	}
+
+	pbf.Event.Start = pkt.Ts
+	pbf.Event.Dataset = "dhcpv4"
+	pbf.Network.Transport = "udp"
+	pbf.Network.Protocol = pbf.Event.Dataset
+
+	fields := evt.Fields
+	fields["type"] = pbf.Event.Dataset
+	fields["status"] = "OK"
+
 	dhcpData := common.MapStr{
 		"op_code":        strings.ToLower(v4.OpcodeToString()),
 		"hardware_type":  v4.HwTypeToString(),
@@ -99,6 +129,7 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event {
 		"flags":          strings.ToLower(v4.FlagsToString()),
 		"client_mac":     v4.ClientHwAddrToString(),
 	}
+	fields["dhcpv4"] = dhcpData
 
 	if !v4.ClientIPAddr().IsUnspecified() {
 		dhcpData.Put("client_ip", v4.ClientIPAddr().String())
@@ -117,33 +148,11 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event {
 	}
 
 	if opts, err := optionsToMap(v4.StrippedOptions()); err != nil {
-		p.log.Warnw("failed converting DHCP options to map",
+		p.log.Warnw("Failed converting DHCP options to map",
 			"dhcpv4", v4, "error", err)
 	} else if len(opts) > 0 {
 		dhcpData.Put("option", opts)
 	}
 
-	event := &beat.Event{
-		Timestamp: pkt.Ts,
-		Fields: common.MapStr{
-			"transport": "udp",
-			"type":      "dhcpv4",
-			"status":    "OK",
-			"dhcpv4":    dhcpData,
-		},
-	}
-
-	src, dst := common.MakeEndpointPair(pkt.Tuple.BaseTuple, nil)
-	if v4.Opcode() == dhcpv4.OpcodeBootReply {
-		// Reverse
-		event.PutValue("src", &dst)
-		event.PutValue("dst", &src)
-		event.PutValue("bytes_out", len(pkt.Payload))
-	} else {
-		event.PutValue("src", &src)
-		event.PutValue("dst", &dst)
-		event.PutValue("bytes_in", len(pkt.Payload))
-	}
-
-	return event
+	return &evt
 }
diff --git a/packetbeat/protos/dhcpv4/dhcpv4_test.go b/packetbeat/protos/dhcpv4/dhcpv4_test.go
@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/packetbeat/pb"
 	"github.com/elastic/beats/packetbeat/protos"
 )
 
@@ -95,18 +96,37 @@ func TestParseDHCPRequest(t *testing.T) {
 	expected := beat.Event{
 		Timestamp: pkt.Ts,
 		Fields: common.MapStr{
-			"type":      "dhcpv4",
-			"transport": "udp",
-			"status":    "OK",
-			"src": &common.Endpoint{
-				IP:   "0.0.0.0",
-				Port: 68,
+			"type":   "dhcpv4",
+			"status": "OK",
+			"source": common.MapStr{
+				"ip":    "0.0.0.0",
+				"port":  68,
+				"bytes": 272,
 			},
-			"dst": &common.Endpoint{
-				IP:   "255.255.255.255",
-				Port: 67,
+			"destination": common.MapStr{
+				"ip":   "255.255.255.255",
+				"port": 67,
+			},
+			"client": common.MapStr{
+				"ip":    "0.0.0.0",
+				"port":  68,
+				"bytes": 272,
+			},
+			"server": common.MapStr{
+				"ip":   "255.255.255.255",
+				"port": 67,
+			},
+			"event": common.MapStr{
+				"dataset": "dhcpv4",
+				"start":   pkt.Ts,
+			},
+			"network": common.MapStr{
+				"type":         "ipv4",
+				"transport":    "udp",
+				"protocol":     "dhcpv4",
+				"bytes":        272,
+				"community_id": "1:t9O1j0qj71O4wJM7gnaHtgmfev8=",
 			},
-			"bytes_in": 272,
 			"dhcpv4": common.MapStr{
 				"client_mac":     "00:0b:82:01:fc:42",
 				"flags":          "unicast",
@@ -130,7 +150,7 @@ func TestParseDHCPRequest(t *testing.T) {
 		},
 	}
 
-	actual := p.parseDHCPv4(pkt)
+	actual := marshalPacketbeatFields(t, p.parseDHCPv4(pkt))
 	if assert.NotNil(t, actual) {
 		t.Logf("DHCP event: %+v", actual)
 		assertEqual(t, expected, *actual)
@@ -153,18 +173,38 @@ func TestParseDHCPACK(t *testing.T) {
 	expected := beat.Event{
 		Timestamp: pkt.Ts,
 		Fields: common.MapStr{
-			"type":      "dhcpv4",
-			"transport": "udp",
-			"status":    "OK",
-			"src": &common.Endpoint{
-				IP:   "192.168.0.10",
-				Port: 68,
+			"type":   "dhcpv4",
+			"status": "OK",
+			"source": common.MapStr{
+				"ip":    "192.168.0.1",
+				"port":  67,
+				"bytes": 300,
+			},
+			"destination": common.MapStr{
+				"ip":   "192.168.0.10",
+				"port": 68,
 			},
-			"dst": &common.Endpoint{
-				IP:   "192.168.0.1",
-				Port: 67,
+			"client": common.MapStr{
+				"ip":   "192.168.0.10",
+				"port": 68,
 			},
-			"bytes_out": 300,
+			"server": common.MapStr{
+				"ip":    "192.168.0.1",
+				"port":  67,
+				"bytes": 300,
+			},
+			"event": common.MapStr{
+				"dataset": "dhcpv4",
+				"start":   pkt.Ts,
+			},
+			"network": common.MapStr{
+				"type":         "ipv4",
+				"transport":    "udp",
+				"protocol":     "dhcpv4",
+				"bytes":        300,
+				"community_id": "1:VbRSZnvQqvLiQRhYHLrdVI17sLQ=",
+			},
+
 			"dhcpv4": common.MapStr{
 				"assigned_ip":    "192.168.0.10",
 				"client_mac":     "00:0b:82:01:fc:42",
@@ -186,7 +226,7 @@ func TestParseDHCPACK(t *testing.T) {
 		},
 	}
 
-	actual := p.parseDHCPv4(pkt)
+	actual := marshalPacketbeatFields(t, p.parseDHCPv4(pkt))
 	if assert.NotNil(t, actual) {
 		t.Logf("DHCP event: %+v", actual)
 		assertEqual(t, expected, *actual)
@@ -209,3 +249,20 @@ func normalizeEvent(t testing.TB, event beat.Event) interface{} {
 	}
 	return out
 }
+
+func marshalPacketbeatFields(t testing.TB, evt *beat.Event) *beat.Event {
+	pbf, err := pb.GetFields(evt.Fields)
+	if err != nil || pbf == nil {
+		t.Fatal("failed getting _packetbeat", err)
+	}
+	delete(evt.Fields, pb.FieldsKey)
+
+	if err = pbf.ComputeValues(nil); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = pbf.MarshalMapStr(evt.Fields); err != nil {
+		t.Fatal(err)
+	}
+	return evt
+}