From 2123e0e2005f397e5cdd07df40a34f690e37b815 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Jan 2019 13:50:50 -0500 Subject: [PATCH] Update DHCPv4 protocol to use ECS fields The DHCPv4 protocol dataset works on uni-flows (it's not transacted, see #7956) so the `source` and `destination` will indicate the original packet header data. Meanwhile the `client` / `server` fields are copies of the `source`/`destination`, but they are copied based on which side is the client and server. Here's a summary of what fields changed. Part of #7968 Changed - bytes_in -> source.bytes - transport -> network.transport = udp Added - source - destination - event.dataset = dhcpv4 - event.start - network.bytes - network.community_id - network.protocol = dhcpv4 - network.type Unchanged Packetbeat Fields - status - type = dhcpv4 (we might remove this since we have event.dataset) --- CHANGELOG.next.asciidoc | 1 + .../kibana/6/dashboard/Packetbeat-dhcpv4.json | 35 +++--- packetbeat/protos/dhcpv4/dhcpv4.go | 59 +++++----- packetbeat/protos/dhcpv4/dhcpv4_test.go | 101 ++++++++++++++---- packetbeat/tests/system/test_0066_dhcp.py | 39 ++++++- 5 files changed, 167 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 953592f4a49..3e22463c4e8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -59,6 +59,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Removed trailing dot from domain names reported by the DNS protocol. {pull}9941[9941] - Changed TLS protocol fields to align with ECS. {pull}9980[9980] - Changed ICMP protocol fields to align with ECS. {pull}10062[10062] +- Changed DHCPv4 protocol fields to align with ECS. {pull}10089[10089] *Winlogbeat* diff --git a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json index 19c63b5827a..2d53da50329 100644 --- a/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json +++ b/packetbeat/_meta/kibana/6/dashboard/Packetbeat-dhcpv4.json @@ -104,7 +104,7 @@ }, "id": "8460fcd0-8baa-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", + "updated_at": "2019-01-15T18:40:16.104Z", "version": 1 }, { @@ -167,7 +167,7 @@ }, "id": "4ad9db20-8bab-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", + "updated_at": "2019-01-15T18:40:16.104Z", "version": 1 }, { @@ -249,7 +249,7 @@ }, "id": "418dfbe0-8bac-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", + "updated_at": "2019-01-15T18:40:16.104Z", "version": 1 }, { @@ -258,7 +258,8 @@ "dhcpv4.transaction_id", "dhcpv4.op_code", "dhcpv4.option.message_type", - "client_ip", + "source.ip", + "destination.ip", "dhcpv4.client_mac", "dhcpv4.option.hostname", "dhcpv4.option.class_identifier" @@ -276,7 +277,7 @@ "alias": null, "disabled": false, "index": "packetbeat-*", - "key": "type", + "key": "event.dataset", "negate": false, "params": { "query": "dhcpv4", @@ -287,7 +288,7 @@ }, "query": { "match": { - "type": { + "event.dataset": { "query": "dhcpv4", "type": "phrase" } @@ -313,8 +314,8 @@ }, "id": "b8992150-8ba8-11e8-9676-ef67484126fb", "type": "search", - "updated_at": "2019-01-04T14:55:16.796Z", - "version": 1 + "updated_at": "2019-01-15T18:46:52.929Z", + "version": 4 }, { "attributes": { @@ -379,7 +380,7 @@ }, "id": "d0120dc0-8bac-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", + "updated_at": "2019-01-15T18:40:16.104Z", "version": 1 }, { @@ -445,7 +446,7 @@ }, "id": "11d33ea0-8bad-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", + "updated_at": "2019-01-15T18:40:16.104Z", "version": 1 }, { @@ -471,7 +472,7 @@ "id": "1", "params": { "customLabel": "Requests", - "field": "bytes_in" + "field": "client.bytes" }, "schema": "metric", "type": "sum" @@ -481,7 +482,7 @@ "id": "2", "params": { "customLabel": "Responses", - "field": "bytes_out" + "field": "server.bytes" }, "schema": "metric", "type": "sum" @@ -521,8 +522,8 @@ }, "id": "f43a8f20-8bb5-11e8-9676-ef67484126fb", "type": "visualization", - "updated_at": "2019-01-04T14:55:16.796Z", - "version": 1 + "updated_at": "2019-01-15T18:45:46.912Z", + "version": 2 }, { "attributes": { @@ -650,9 +651,9 @@ }, "id": "a7b35890-8baa-11e8-9676-ef67484126fb", "type": "dashboard", - "updated_at": "2019-01-04T15:03:10.809Z", + "updated_at": "2019-01-15T18:46:22.620Z", "version": 2 } ], - "version": "6.6.0-SNAPSHOT" -} + "version": "7.0.0-SNAPSHOT" +} \ No newline at end of file diff --git a/packetbeat/protos/dhcpv4/dhcpv4.go b/packetbeat/protos/dhcpv4/dhcpv4.go index 0905a27ea44..19b81c0717e 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4.go +++ b/packetbeat/protos/dhcpv4/dhcpv4.go @@ -27,7 +27,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/monitoring" + "github.com/elastic/beats/packetbeat/pb" "github.com/elastic/beats/packetbeat/protos" + "github.com/elastic/ecs/code/go/ecs" ) var ( @@ -86,10 +88,38 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { v4, err := dhcpv4.FromBytes(pkt.Payload) if err != nil { metricParseFailures.Inc() - p.log.Warnw("dropping packet: failed parsing DHCP data", "error", err) + p.log.Warnw("Dropping packet: failed parsing DHCP data", "error", err) return nil } + evt, pbf := pb.NewBeatEvent(pkt.Ts) + + // source/destination (note: this protocol does not produce a bi-flow.) + src, dst := common.MakeEndpointPair(pkt.Tuple.BaseTuple, nil) + pbf.SetSource(&src) + pbf.SetDestination(&dst) + pbf.Source.Bytes = int64(len(pkt.Payload)) + + if v4.Opcode() == dhcpv4.OpcodeBootReply { + // Reverse + client, server := ecs.Client(*pbf.Destination), ecs.Server(*pbf.Source) + pbf.Client = &client + pbf.Server = &server + } else { + client, server := ecs.Client(*pbf.Source), ecs.Server(*pbf.Destination) + pbf.Client = &client + pbf.Server = &server + } + + pbf.Event.Start = pkt.Ts + pbf.Event.Dataset = "dhcpv4" + pbf.Network.Transport = "udp" + pbf.Network.Protocol = pbf.Event.Dataset + + fields := evt.Fields + fields["type"] = pbf.Event.Dataset + fields["status"] = "OK" + dhcpData := common.MapStr{ "op_code": strings.ToLower(v4.OpcodeToString()), "hardware_type": v4.HwTypeToString(), @@ -99,6 +129,7 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { "flags": strings.ToLower(v4.FlagsToString()), "client_mac": v4.ClientHwAddrToString(), } + fields["dhcpv4"] = dhcpData if !v4.ClientIPAddr().IsUnspecified() { dhcpData.Put("client_ip", v4.ClientIPAddr().String()) @@ -117,33 +148,11 @@ func (p *dhcpv4Plugin) parseDHCPv4(pkt *protos.Packet) *beat.Event { } if opts, err := optionsToMap(v4.StrippedOptions()); err != nil { - p.log.Warnw("failed converting DHCP options to map", + p.log.Warnw("Failed converting DHCP options to map", "dhcpv4", v4, "error", err) } else if len(opts) > 0 { dhcpData.Put("option", opts) } - event := &beat.Event{ - Timestamp: pkt.Ts, - Fields: common.MapStr{ - "transport": "udp", - "type": "dhcpv4", - "status": "OK", - "dhcpv4": dhcpData, - }, - } - - src, dst := common.MakeEndpointPair(pkt.Tuple.BaseTuple, nil) - if v4.Opcode() == dhcpv4.OpcodeBootReply { - // Reverse - event.PutValue("src", &dst) - event.PutValue("dst", &src) - event.PutValue("bytes_out", len(pkt.Payload)) - } else { - event.PutValue("src", &src) - event.PutValue("dst", &dst) - event.PutValue("bytes_in", len(pkt.Payload)) - } - - return event + return &evt } diff --git a/packetbeat/protos/dhcpv4/dhcpv4_test.go b/packetbeat/protos/dhcpv4/dhcpv4_test.go index 72e91025ff3..9760d215fbd 100644 --- a/packetbeat/protos/dhcpv4/dhcpv4_test.go +++ b/packetbeat/protos/dhcpv4/dhcpv4_test.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/packetbeat/pb" "github.com/elastic/beats/packetbeat/protos" ) @@ -95,18 +96,37 @@ func TestParseDHCPRequest(t *testing.T) { expected := beat.Event{ Timestamp: pkt.Ts, Fields: common.MapStr{ - "type": "dhcpv4", - "transport": "udp", - "status": "OK", - "src": &common.Endpoint{ - IP: "0.0.0.0", - Port: 68, + "type": "dhcpv4", + "status": "OK", + "source": common.MapStr{ + "ip": "0.0.0.0", + "port": 68, + "bytes": 272, }, - "dst": &common.Endpoint{ - IP: "255.255.255.255", - Port: 67, + "destination": common.MapStr{ + "ip": "255.255.255.255", + "port": 67, + }, + "client": common.MapStr{ + "ip": "0.0.0.0", + "port": 68, + "bytes": 272, + }, + "server": common.MapStr{ + "ip": "255.255.255.255", + "port": 67, + }, + "event": common.MapStr{ + "dataset": "dhcpv4", + "start": pkt.Ts, + }, + "network": common.MapStr{ + "type": "ipv4", + "transport": "udp", + "protocol": "dhcpv4", + "bytes": 272, + "community_id": "1:t9O1j0qj71O4wJM7gnaHtgmfev8=", }, - "bytes_in": 272, "dhcpv4": common.MapStr{ "client_mac": "00:0b:82:01:fc:42", "flags": "unicast", @@ -130,7 +150,7 @@ func TestParseDHCPRequest(t *testing.T) { }, } - actual := p.parseDHCPv4(pkt) + actual := marshalPacketbeatFields(t, p.parseDHCPv4(pkt)) if assert.NotNil(t, actual) { t.Logf("DHCP event: %+v", actual) assertEqual(t, expected, *actual) @@ -153,18 +173,38 @@ func TestParseDHCPACK(t *testing.T) { expected := beat.Event{ Timestamp: pkt.Ts, Fields: common.MapStr{ - "type": "dhcpv4", - "transport": "udp", - "status": "OK", - "src": &common.Endpoint{ - IP: "192.168.0.10", - Port: 68, + "type": "dhcpv4", + "status": "OK", + "source": common.MapStr{ + "ip": "192.168.0.1", + "port": 67, + "bytes": 300, + }, + "destination": common.MapStr{ + "ip": "192.168.0.10", + "port": 68, }, - "dst": &common.Endpoint{ - IP: "192.168.0.1", - Port: 67, + "client": common.MapStr{ + "ip": "192.168.0.10", + "port": 68, }, - "bytes_out": 300, + "server": common.MapStr{ + "ip": "192.168.0.1", + "port": 67, + "bytes": 300, + }, + "event": common.MapStr{ + "dataset": "dhcpv4", + "start": pkt.Ts, + }, + "network": common.MapStr{ + "type": "ipv4", + "transport": "udp", + "protocol": "dhcpv4", + "bytes": 300, + "community_id": "1:VbRSZnvQqvLiQRhYHLrdVI17sLQ=", + }, + "dhcpv4": common.MapStr{ "assigned_ip": "192.168.0.10", "client_mac": "00:0b:82:01:fc:42", @@ -186,7 +226,7 @@ func TestParseDHCPACK(t *testing.T) { }, } - actual := p.parseDHCPv4(pkt) + actual := marshalPacketbeatFields(t, p.parseDHCPv4(pkt)) if assert.NotNil(t, actual) { t.Logf("DHCP event: %+v", actual) assertEqual(t, expected, *actual) @@ -209,3 +249,20 @@ func normalizeEvent(t testing.TB, event beat.Event) interface{} { } return out } + +func marshalPacketbeatFields(t testing.TB, evt *beat.Event) *beat.Event { + pbf, err := pb.GetFields(evt.Fields) + if err != nil || pbf == nil { + t.Fatal("failed getting _packetbeat", err) + } + delete(evt.Fields, pb.FieldsKey) + + if err = pbf.ComputeValues(nil); err != nil { + t.Fatal(err) + } + + if err = pbf.MarshalMapStr(evt.Fields); err != nil { + t.Fatal(err) + } + return evt +} diff --git a/packetbeat/tests/system/test_0066_dhcp.py b/packetbeat/tests/system/test_0066_dhcp.py index 09668440896..2025df30278 100644 --- a/packetbeat/tests/system/test_0066_dhcp.py +++ b/packetbeat/tests/system/test_0066_dhcp.py @@ -14,8 +14,10 @@ def test_dhcp(self): objs = self.read_output(types=['dhcpv4']) assert len(objs) == 4 + assert "event.start" in objs[0] assert objs[0]["client.ip"] == "0.0.0.0" assert objs[0]["client.port"] == 68 + assert objs[0]["destination.ip"] == "255.255.255.255" assert objs[0]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" assert objs[0]["dhcpv4.flags"] == "unicast" assert objs[0]["dhcpv4.hardware_type"] == "Ethernet" @@ -31,14 +33,22 @@ def test_dhcp(self): assert objs[0]["dhcpv4.option.requested_ip_address"] == "0.0.0.0" assert objs[0]["dhcpv4.seconds"] == 0 assert objs[0]["dhcpv4.transaction_id"] == "0x00003d1d" + assert objs[0]["event.dataset"] == "dhcpv4" assert objs[0]["server.ip"] == "255.255.255.255" assert objs[0]["server.port"] == 67 + assert objs[0]["source.ip"] == "0.0.0.0" assert objs[0]["status"] == "OK" - assert objs[0]["transport"] == "udp" + assert objs[0]["network.type"] == "ipv4" + assert objs[0]["network.transport"] == "udp" + assert objs[0]["network.protocol"] == "dhcpv4" + assert objs[0]["network.bytes"] == 272 + assert objs[0]["network.community_id"] == "1:t9O1j0qj71O4wJM7gnaHtgmfev8=" assert objs[0]["type"] == "dhcpv4" + assert "event.start" in objs[1] assert objs[1]["client.ip"] == "192.168.0.10" assert objs[1]["client.port"] == 68 + assert objs[1]["destination.ip"] == "192.168.0.10" assert objs[1]["dhcpv4.assigned_ip"] == "192.168.0.10" assert objs[1]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" assert objs[1]["dhcpv4.flags"] == "unicast" @@ -53,12 +63,20 @@ def test_dhcp(self): assert objs[1]["dhcpv4.option.subnet_mask"] == "255.255.255.0" assert objs[1]["dhcpv4.seconds"] == 0 assert objs[1]["dhcpv4.transaction_id"] == "0x00003d1d" + assert objs[1]["event.dataset"] == "dhcpv4" + assert objs[1]["network.bytes"] == 300 + assert objs[1]["network.community_id"] == "1:VbRSZnvQqvLiQRhYHLrdVI17sLQ=" + assert objs[1]["network.protocol"] == "dhcpv4" + assert objs[1]["network.transport"] == "udp" + assert objs[1]["network.type"] == "ipv4" + assert objs[1]["server.bytes"] == 300 assert objs[1]["server.ip"] == "192.168.0.1" assert objs[1]["server.port"] == 67 + assert objs[1]["source.ip"] == "192.168.0.1" assert objs[1]["status"] == "OK" - assert objs[1]["transport"] == "udp" assert objs[1]["type"] == "dhcpv4" + assert "event.start" in objs[2] assert objs[2]["client.ip"] == "0.0.0.0" assert objs[2]["client.port"] == 68 assert objs[2]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" @@ -77,14 +95,21 @@ def test_dhcp(self): assert objs[2]["dhcpv4.option.server_identifier"] == "192.168.0.1" assert objs[2]["dhcpv4.seconds"] == 0 assert objs[2]["dhcpv4.transaction_id"] == "0x00003d1e" + assert objs[2]["event.dataset"] == "dhcpv4" + assert objs[2]["network.bytes"] == 272 + assert objs[2]["network.community_id"] == "1:t9O1j0qj71O4wJM7gnaHtgmfev8=" + assert objs[2]["network.protocol"] == "dhcpv4" + assert objs[2]["network.transport"] == "udp" + assert objs[2]["network.type"] == "ipv4" assert objs[2]["server.ip"] == "255.255.255.255" assert objs[2]["server.port"] == 67 assert objs[2]["status"] == "OK" - assert objs[2]["transport"] == "udp" assert objs[2]["type"] == "dhcpv4" + assert "event.start" in objs[3] assert objs[3]["client.ip"] == "192.168.0.10" assert objs[3]["client.port"] == 68 + assert objs[3]["destination.ip"] == "192.168.0.10" assert objs[3]["dhcpv4.assigned_ip"] == "192.168.0.10" assert objs[3]["dhcpv4.client_mac"] == "00:0b:82:01:fc:42" assert objs[3]["dhcpv4.flags"] == "unicast" @@ -99,8 +124,14 @@ def test_dhcp(self): assert objs[3]["dhcpv4.option.subnet_mask"] == "255.255.255.0" assert objs[3]["dhcpv4.seconds"] == 0 assert objs[3]["dhcpv4.transaction_id"] == "0x00003d1e" + assert objs[3]["event.dataset"] == "dhcpv4" + assert objs[3]["network.bytes"] == 300 + assert objs[3]["network.community_id"] == "1:VbRSZnvQqvLiQRhYHLrdVI17sLQ=" + assert objs[3]["network.protocol"] == "dhcpv4" + assert objs[3]["network.transport"] == "udp" + assert objs[3]["network.type"] == "ipv4" assert objs[3]["server.ip"] == "192.168.0.1" assert objs[3]["server.port"] == 67 + assert objs[3]["source.ip"] == "192.168.0.1" assert objs[3]["status"] == "OK" - assert objs[3]["transport"] == "udp" assert objs[3]["type"] == "dhcpv4"