From ff5eaf64fdc8b53e9182cd84f2569a300e0f2e00 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 21 Feb 2019 19:08:13 -0500 Subject: [PATCH 1/2] Set event.category and event.kind for Suricata Set `event.category: network_traffic`. And conditionally set event.kind based on the `event_type` field. Use `alert` for suricata alerts and `event` for everything else. Also skip geoip if the event has already been enriched (like if the data went through Logstash first). --- CHANGELOG.next.asciidoc | 1 + .../module/suricata/eve/ingest/pipeline.json | 16 +++++ .../eve/test/eve-alerts.log-expected.json | 60 ++++++++++++------- .../eve/test/eve-small.log-expected.json | 10 +++- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 77906646291..700e8f08b6e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -244,6 +244,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736] - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845] - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877] +- Set `event.category: network_traffic` for Suricata. {pull}10882[10882] *Heartbeat* diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 67ec81b85a5..85a3d591978 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -227,6 +227,7 @@ }, { "geoip": { + "if": "ctx.source?.geo == null", "field": "source.ip", "target_field": "source.geo", "ignore_missing": true @@ -234,6 +235,7 @@ }, { "geoip": { + "if": "ctx.destination?.geo == null", "field": "destination.ip", "target_field": "destination.geo", "ignore_missing": true @@ -255,9 +257,23 @@ }, { "set": { + "if": "ctx.suricata?.eve?.event_type == \"alert\"", + "field": "event.kind", + "value": "alert" + } + }, + { + "set": { + "if": "ctx.suricata?.eve?.event_type != \"alert\"", "field": "event.kind", "value": "event" } + }, + { + "set": { + "field": "event.category", + "value": "network_traffic" + } } ], "on_failure": [ diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 346691a4a2c..64f62fcb7ec 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -14,10 +14,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 223000000, "event.end": "2018-10-03T14:42:44.836Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -75,10 +76,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 244000000, "event.end": "2018-10-03T16:16:26.711Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -136,10 +138,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 233000000, "event.end": "2018-10-03T16:44:50.813Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -197,10 +200,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 231000000, "event.end": "2018-10-03T16:45:09.267Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -258,10 +262,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 229000000, "event.end": "2018-10-03T16:45:34.481Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -319,10 +324,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 301000000, "event.end": "2018-10-03T17:02:38.900Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, @@ -380,10 +386,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 85000000, "event.end": "2018-10-04T09:34:59.009Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -441,10 +448,11 @@ "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 242000000, "event.end": "2018-10-04T09:34:59.168Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -502,10 +510,11 @@ "destination.packets": 5, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 362000000, "event.end": "2018-10-04T09:34:59.288Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -563,10 +572,11 @@ "destination.packets": 62, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 365000000, "event.end": "2018-10-04T09:34:59.289Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -624,10 +634,11 @@ "destination.packets": 98, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 432000000, "event.end": "2018-10-04T09:34:59.356Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -685,10 +696,11 @@ "destination.packets": 221, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 532000000, "event.end": "2018-10-04T09:34:59.456Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -746,10 +758,11 @@ "destination.packets": 67, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 821000000, "event.end": "2018-10-04T09:34:59.747Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -807,10 +820,11 @@ "destination.packets": 119, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1027000000, "event.end": "2018-10-04T09:34:59.953Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -868,10 +882,11 @@ "destination.packets": 253, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1324000000, "event.end": "2018-10-04T09:35:00.250Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -929,10 +944,11 @@ "destination.packets": 314, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1475000000, "event.end": "2018-10-04T09:35:00.401Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -990,10 +1006,11 @@ "destination.packets": 588, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1850000000, "event.end": "2018-10-04T09:35:00.776Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -1051,10 +1068,11 @@ "destination.packets": 591, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1971000000, "event.end": "2018-10-04T09:35:00.897Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -1112,10 +1130,11 @@ "destination.packets": 979, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 2436000000, "event.end": "2018-10-04T09:35:01.362Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, @@ -1172,10 +1191,11 @@ "destination.packets": 1079, "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 2649000000, "event.end": "2018-10-04T09:35:01.575Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index f7aa462b334..3cb37412bbe 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -4,6 +4,7 @@ "destination.ip": "192.168.253.112", "destination.port": 22, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:01:09.820Z", "event.kind": "event", @@ -33,10 +34,11 @@ "destination.packets": 3, "destination.port": 443, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 1251000000, "event.end": "2018-07-05T19:07:20.910Z", - "event.kind": "event", + "event.kind": "alert", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 1, @@ -76,6 +78,7 @@ "destination.ip": "192.168.86.28", "destination.port": 63963, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:43:47.690Z", "event.kind": "event", @@ -116,6 +119,7 @@ "destination.ip": "192.168.86.85", "destination.port": 56118, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:44:33.222Z", "event.kind": "event", @@ -164,6 +168,7 @@ "destination.ip": "192.168.86.85", "destination.port": 39464, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:51:20.213Z", "event.kind": "event", @@ -192,6 +197,7 @@ { "@timestamp": "2018-07-05T19:51:23.009Z", "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:51:23.009Z", "event.kind": "event", @@ -330,6 +336,7 @@ "destination.ip": "17.142.164.13", "destination.port": 443, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:51:50.666Z", "event.kind": "event", @@ -363,6 +370,7 @@ "destination.packets": 0, "destination.port": 547, "ecs.version": "1.0.0-beta2", + "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.duration": 30548000000, "event.end": "2018-07-05T19:51:54.001Z", From f3b893d2811618a7ccca93401ef429766625b67c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 22 Feb 2019 20:48:53 -0500 Subject: [PATCH 2/2] Handle event_type:stats as event.kind:metric --- .../module/suricata/eve/ingest/pipeline.json | 20 +++---------------- .../eve/test/eve-small.log-expected.json | 3 +-- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 85a3d591978..e1fcf7c0c6e 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -256,23 +256,9 @@ } }, { - "set": { - "if": "ctx.suricata?.eve?.event_type == \"alert\"", - "field": "event.kind", - "value": "alert" - } - }, - { - "set": { - "if": "ctx.suricata?.eve?.event_type != \"alert\"", - "field": "event.kind", - "value": "event" - } - }, - { - "set": { - "field": "event.category", - "value": "network_traffic" + "script": { + "lang": "painless", + "source": "def t = ctx.suricata?.eve?.event_type; if (t == \"stats\") {\n ctx['event']['kind'] = \"metric\";\n} else if (t == \"alert\") {\n ctx['event']['kind'] = \"alert\";\n ctx['event']['category'] = \"network_traffic\";\n} else {\n ctx['event']['kind'] = \"event\";\n ctx['event']['category'] = \"network_traffic\";\n}" } } ], diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 3cb37412bbe..82c1fd66725 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -197,10 +197,9 @@ { "@timestamp": "2018-07-05T19:51:23.009Z", "ecs.version": "1.0.0-beta2", - "event.category": "network_traffic", "event.dataset": "suricata.eve", "event.end": "2018-07-05T19:51:23.009Z", - "event.kind": "event", + "event.kind": "metric", "event.module": "suricata", "fileset.name": "eve", "input.type": "log",