From cb46da8caa1354c6793baf258bfc040fe770d957 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 10 Jan 2019 20:10:03 +0100 Subject: [PATCH 01/17] Populate more ECS fields in the Suricata module A few more ECS fields are populated by the ingest pipeline that enriches Suricata's eve.json events. Additions: - http.request.referrer (from suricata.eve.http.http_refer) - event.action (from suricata.eve.alert.category) describes the action that caused the event. Examples: "Attempted Denial of Service", "Successful Administrator Privilege Gain" - event.outcome (from suricata.eve.alert.action) Possible values: "allowed", "blocked" - event.severity (from suricata.eve.alert.severity) Possible values: 1, 2 or 3. - network.transport (from suricata.eve.proto) Examples: "tcp", "udp", "ipv6-icmp" --- .../module/suricata/eve/ingest/pipeline.json | 41 ++++++- .../eve/test/eve-alerts.log-expected.json | 100 ++++++++++++++---- .../eve/test/eve-small.log-expected.json | 20 ++-- 3 files changed, 131 insertions(+), 30 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 6b2b8ebd5f6..e5375d12d1d 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -54,6 +54,13 @@ ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.http.http_refer" + ,"target_field": "http.request.referrer" + ,"type": "string" + ,"ignore_missing": true + } + } , {"convert": {"field": "suricata.eve.fileinfo.filename" ,"target_field": "file.path" @@ -93,7 +100,39 @@ , "value": "{{suricata.eve.event_type}}" } } - + , {"convert": + {"field": "suricata.eve.alert.category" + ,"target_field": "event.action" + ,"type": "string" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.alert.action" + ,"target_field": "event.outcome" + ,"type": "string" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.alert.severity" + ,"target_field": "event.severity" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.proto" + ,"target_field": "network.transport" + ,"type": "string" + ,"ignore_missing": true + } + } + , { "lowercase": + { "field": "network.transport" + , "ignore_missing": true + } + } , { "user_agent": { "field": "user_agent.original" , "target_field": "user_agent" diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 5ac3f24248d..7612c744d14 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -11,15 +11,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 0, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32858, "suricata.eve.alert.action": "allowed", @@ -77,15 +80,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 723, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32864, "suricata.eve.alert.action": "allowed", @@ -143,15 +149,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 1445, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32870, "suricata.eve.alert.action": "allowed", @@ -209,15 +218,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 2168, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32872, "suricata.eve.alert.action": "allowed", @@ -275,15 +287,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 2889, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32876, "suricata.eve.alert.action": "allowed", @@ -341,15 +356,18 @@ "destination.ip": "93.184.216.34", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 3611, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 32892, "suricata.eve.alert.action": "allowed", @@ -407,15 +425,18 @@ "destination.ip": "91.189.88.152", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 4334, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 37742, "suricata.eve.alert.action": "allowed", @@ -471,15 +492,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "304", "input.type": "log", "log.offset": 5140, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -535,15 +559,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 5931, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -599,15 +626,18 @@ "destination.ip": "91.189.88.152", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 6734, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 37742, "suricata.eve.alert.action": "allowed", @@ -663,15 +693,18 @@ "destination.ip": "91.189.88.152", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 7630, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 37742, "suricata.eve.alert.action": "allowed", @@ -727,15 +760,18 @@ "destination.ip": "91.189.88.152", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 8533, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 37742, "suricata.eve.alert.action": "allowed", @@ -791,15 +827,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 9443, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -855,15 +894,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 10252, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -919,15 +961,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 11147, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -983,15 +1028,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 12048, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -1047,15 +1095,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 12951, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -1111,15 +1162,18 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "http.response.status_code": "200", "input.type": "log", "log.offset": 13860, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -1175,14 +1229,17 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "input.type": "log", "log.offset": 14767, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", @@ -1237,14 +1294,17 @@ "destination.ip": "91.189.91.23", "destination.port": 80, "ecs.version": "1.0.0-beta2", + "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, "event.type": "alert", "fileset.name": "eve", "http.request.method": "GET", "input.type": "log", "log.offset": 15651, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.1.146", "source.port": 52340, "suricata.eve.alert.action": "allowed", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index aa2b53b6c7e..faa31236bad 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -10,7 +10,7 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 0, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.86.85", "source.port": 55406, "suricata.eve.dest_ip": "192.168.253.112", @@ -35,13 +35,16 @@ "destination.ip": "192.168.156.70", "destination.port": 443, "ecs.version": "1.0.0-beta2", + "event.action": "Potential Corporate Privacy Violation", "event.dataset": "suricata.eve", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 1, "event.type": "alert", "fileset.name": "eve", "input.type": "log", "log.offset": 350, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.86.85", "source.port": 55641, "suricata.eve.alert.action": "allowed", @@ -87,7 +90,7 @@ "http.response.status_code": "200", "input.type": "log", "log.offset": 985, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.86.85", "source.port": 56119, "suricata.eve.dest_ip": "192.168.86.28", @@ -137,7 +140,7 @@ "http.response.status_code": "200", "input.type": "log", "log.offset": 1507, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.86.28", "source.port": 8008, "suricata.eve.app_proto": "http", @@ -192,7 +195,7 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 2347, - "service.type": "suricata", + "network.transport": "udp", "source.ip": "192.168.86.1", "source.port": 53, "suricata.eve.dest_ip": "192.168.86.85", @@ -224,7 +227,6 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 2687, - "service.type": "suricata", "suricata.eve.event_type": "stats", "suricata.eve.stats.app_layer.flow.dcerpc_tcp": 0, "suricata.eve.stats.app_layer.flow.dcerpc_udp": 0, @@ -362,7 +364,7 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 4683, - "service.type": "suricata", + "network.transport": "tcp", "source.ip": "192.168.86.85", "source.port": 56187, "suricata.eve.dest_ip": "17.142.164.13", @@ -397,7 +399,7 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 5308, - "service.type": "suricata", + "network.transport": "udp", "source.ip": "fe80:0000:0000:0000:fada:0cff:fedc:87f1", "source.port": 546, "suricata.eve.app_proto": "failed", @@ -423,4 +425,4 @@ "suricata" ] } -] \ No newline at end of file +] From 7d3a2d666f5963e68e9ef6a39568ffc9e286938a Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 09:44:15 +0100 Subject: [PATCH 02/17] Use message for suricata.eve.alert.category Instead of event.action, which is expected to have a fixed set of enumeration values. --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index e5375d12d1d..8a567f1d204 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -102,7 +102,7 @@ } , {"convert": {"field": "suricata.eve.alert.category" - ,"target_field": "event.action" + ,"target_field": "message" ,"type": "string" ,"ignore_missing": true } From 0085424c86df0267fac32bce380a7fe6b1518142 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 12:33:20 +0100 Subject: [PATCH 03/17] Populate destination.domain When http.hostname is present. --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 8a567f1d204..d9eccee459b 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -54,6 +54,13 @@ ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.http.hostname" + ,"target_field": "destination.domain" + ,"type": "string" + ,"ignore_missing": true + } + } , {"convert": {"field": "suricata.eve.http.http_refer" ,"target_field": "http.request.referrer" From 30a7fe221b3ea33440b4a7289b29801e5680c004 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 12:33:50 +0100 Subject: [PATCH 04/17] Populate event.{start,end,duration} --- .../module/suricata/eve/ingest/pipeline.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index d9eccee459b..5bdad37583f 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -128,6 +128,25 @@ ,"ignore_missing": true } } + , {"date": + {"field": "suricata.eve.flow.start" + ,"target_field": "event.start" + ,"formats": ["ISO8601"] + ,"ignore_failure": true + } + } + , {"date": + {"field": "suricata.eve.timestamp" + ,"target_field": "event.end" + , "formats": ["ISO8601"] + ,"ignore_failure": true + } + } + , { "script": + { "lang": "painless" + , "source": "Instant ins(def d){try{return Instant.parse(d);}catch(Exception e){return null;}}def ev=ctx['event'];if(ev!=null){def start=ins(ev['start']); def end=ins(ev['end']); if(start!=null && end!=null && !start.isAfter(end)) {ev['duration'] = Duration.between(start,end).toNanos();}}" + } + } , {"convert": {"field": "suricata.eve.proto" ,"target_field": "network.transport" From 4a6c6a70b1a53d816aab76dd642af8d38307f6c1 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 13:50:11 +0100 Subject: [PATCH 05/17] populate network.protocol --- .../module/suricata/eve/ingest/pipeline.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 5bdad37583f..ebfb6add90f 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -159,6 +159,18 @@ , "ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.app_proto" + ,"target_field": "network.protocol" + ,"type": "string" + ,"ignore_missing": true + } + } + , { "lowercase": + { "field": "network.protocol" + , "ignore_missing": true + } + } , { "user_agent": { "field": "user_agent.original" , "target_field": "user_agent" From 4cbea47cd666a8b5fa7fd70736d8eceb49941f23 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 15:16:25 +0100 Subject: [PATCH 06/17] url.hostname is url.domain --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index ebfb6add90f..8ead4b44da3 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -49,7 +49,7 @@ } , {"convert": {"field": "suricata.eve.http.hostname" - ,"target_field": "url.hostname" + ,"target_field": "url.domain" ,"type": "string" ,"ignore_missing": true } From 2b5dac58caffa29964dabbef0585d22ed6808968 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 16:44:19 +0100 Subject: [PATCH 07/17] Populate url.path, url.fragment, url.query From http.url --- .../module/suricata/eve/ingest/pipeline.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 8ead4b44da3..2e576aa4642 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -54,6 +54,18 @@ ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.http.url" + ,"target_field": "url.path" + ,"type": "string" + ,"ignore_missing": true + } + } + , { "script": + { "lang": "painless" + , "source": "String[] split(String s, String sep) { def p=s.indexOf(sep); return p>=0? new String[]{s.substring(0,p), s.substring(p+1)} : new String[]{s};} def u=ctx['url']; def p=u!=null? u['path'] : null; if(p!=null) {def q=split(p,'#');if(q.length>1)u['fragment']=q[1];q=split(q[0],'?');if(q.length>1)u['query']=q[1];u['path']=q[0];}" + } + } , {"convert": {"field": "suricata.eve.http.hostname" ,"target_field": "destination.domain" From 905828094997424e84dbcddd49795fc124bf10f1 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 16:55:06 +0100 Subject: [PATCH 08/17] Lowercase http request method --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 2e576aa4642..ce9f6e5f8c2 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -40,6 +40,11 @@ ,"ignore_missing": true } } + , { "lowercase": + { "field": "http.request.method" + , "ignore_missing": true + } + } , {"convert": {"field": "suricata.eve.http.status" ,"target_field": "http.response.status_code" From b0f163ea65ab63a08df8cc34d98b07a304b4d7de Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 17:31:33 +0100 Subject: [PATCH 09/17] Source/Destination and aggregated counters This assumes client=source server=destination. Populates - source.{packets|bytes} - destination.{packets|bytes} - network.{packets|bytes} --- .../module/suricata/eve/ingest/pipeline.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index ce9f6e5f8c2..ca1ce74a756 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -145,6 +145,39 @@ ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.flow.pkts_toclient" + ,"target_field": "destination.packets" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.pkts_toserver" + ,"target_field": "source.packets" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.bytes_toclient" + ,"target_field": "destination.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.bytes_toserver" + ,"target_field": "source.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } + , { "script": + { "lang": "painless" + , "source": "long g(def map, def key) { if(map!=null && map[key]!=null) { return map[key]; } return 0; } def n=ctx['network']; if (n==null){n=new HashMap(); ctx['network']=n;}def s=ctx['source'], d=ctx['destination']; def sp=g(s,'packets'), sb=g(s,'bytes'), dp=g(d,'packets'), db=g(d,'bytes'); if(sb+db>0)n['bytes']=sb+db; if(sp+dp>0)n['packets']=sp+dp;" + } + } , {"date": {"field": "suricata.eve.flow.start" ,"target_field": "event.start" From 5dd36d00108b10684f2ede1e05ad10c5f904edca Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 11 Jan 2019 17:42:06 +0100 Subject: [PATCH 10/17] Updated golden files --- .../eve/test/eve-alerts.log-expected.json | 360 +++++++++++++++--- .../eve/test/eve-small.log-expected.json | 41 +- 2 files changed, 336 insertions(+), 65 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 7612c744d14..0d8a518f50e 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -1,6 +1,8 @@ [ { "@timestamp": "2018-10-03T14:42:44.836Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -9,21 +11,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 223000000, + "event.end": "2018-10-03T14:42:44.836Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T14:42:44.613Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 0, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32858, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -59,7 +70,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -70,6 +82,8 @@ }, { "@timestamp": "2018-10-03T16:16:26.711Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -78,21 +92,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 244000000, + "event.end": "2018-10-03T16:16:26.711Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T16:16:26.467Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 723, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32864, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -128,7 +151,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -139,6 +163,8 @@ }, { "@timestamp": "2018-10-03T16:44:50.813Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -147,21 +173,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 233000000, + "event.end": "2018-10-03T16:44:50.813Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T16:44:50.580Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 1445, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32870, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -197,7 +232,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -208,6 +244,8 @@ }, { "@timestamp": "2018-10-03T16:45:09.267Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -216,21 +254,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 231000000, + "event.end": "2018-10-03T16:45:09.267Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T16:45:09.036Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 2168, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32872, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -266,7 +313,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -277,6 +325,8 @@ }, { "@timestamp": "2018-10-03T16:45:34.481Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -285,21 +335,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 229000000, + "event.end": "2018-10-03T16:45:34.481Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T16:45:34.252Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 2889, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32876, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -335,7 +394,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -346,6 +406,8 @@ }, { "@timestamp": "2018-10-03T17:02:38.900Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -354,21 +416,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Attempted Information Leak", "event.dataset": "suricata.eve", + "event.duration": 301000000, + "event.end": "2018-10-03T17:02:38.900Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 2, + "event.start": "2018-10-03T17:02:38.599Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 3611, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32892, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -404,7 +475,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -415,6 +487,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.009Z", + "destination.bytes": 1654, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -423,21 +497,30 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 85000000, + "event.end": "2018-10-04T09:34:59.009Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 4334, + "message": "Not Suspicious Traffic", + "network.bytes": 2151, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 497, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -472,7 +555,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -482,6 +566,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.168Z", + "destination.bytes": 417, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -490,21 +576,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 242000000, + "event.end": "2018-10-04T09:34:59.168Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "304", "input.type": "log", "log.offset": 5140, + "message": "Not Suspicious Traffic", + "network.bytes": 904, + "network.packets": 7, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 487, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -539,7 +634,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -549,6 +645,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.288Z", + "destination.bytes": 3445, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -557,21 +655,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 5, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 362000000, + "event.end": "2018-10-04T09:34:59.288Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 5931, + "message": "Not Suspicious Traffic", + "network.bytes": 4287, + "network.packets": 11, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 842, "source.ip": "192.168.1.146", + "source.packets": 6, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -606,7 +713,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -616,6 +724,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.289Z", + "destination.bytes": 90543, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -624,21 +734,30 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 62, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 365000000, + "event.end": "2018-10-04T09:34:59.289Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 6734, + "message": "Not Suspicious Traffic", + "network.bytes": 95353, + "network.packets": 126, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 4810, "source.ip": "192.168.1.146", + "source.packets": 64, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -673,7 +792,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/main/source/by-hash/SHA256/f5ec03d97ca76c98162d9233c8b7c578c52897e2136428277baf2e7b633a8e72", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -683,6 +803,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.356Z", + "destination.bytes": 145014, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -691,21 +813,30 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 98, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 432000000, + "event.end": "2018-10-04T09:34:59.356Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 7630, + "message": "Not Suspicious Traffic", + "network.bytes": 151605, + "network.packets": 185, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 6591, "source.ip": "192.168.1.146", + "source.packets": 87, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -740,7 +871,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/main/binary-amd64/by-hash/SHA256/c5b8346a3221bc9a23a79ba4dc4e730a6319a77fc9d63872dfc56539a0810015", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -750,6 +882,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.456Z", + "destination.bytes": 330525, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -758,21 +892,30 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 221, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 532000000, + "event.end": "2018-10-04T09:34:59.456Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 8533, + "message": "Not Suspicious Traffic", + "network.bytes": 341985, + "network.packets": 377, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 11460, "source.ip": "192.168.1.146", + "source.packets": 156, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -807,7 +950,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/universe/binary-amd64/by-hash/SHA256/e5cc957139a25a0fee47cbf2c0fac8ad5cab50346d6a74abe031748924c5b558", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -817,6 +961,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.747Z", + "destination.bytes": 96554, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -825,21 +971,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 67, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 821000000, + "event.end": "2018-10-04T09:34:59.747Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 9443, + "message": "Not Suspicious Traffic", + "network.bytes": 101449, + "network.packets": 131, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 4895, "source.ip": "192.168.1.146", + "source.packets": 64, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -874,7 +1029,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-backports/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -884,6 +1040,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.953Z", + "destination.bytes": 174843, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -892,21 +1050,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 119, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 1027000000, + "event.end": "2018-10-04T09:34:59.953Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 10252, + "message": "Not Suspicious Traffic", + "network.bytes": 181775, + "network.packets": 210, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 6932, "source.ip": "192.168.1.146", + "source.packets": 91, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -941,7 +1108,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/main/source/by-hash/SHA256/65f2e3a4e9d89d9d4b5e3d42e586bc96f48a24466b0ad0b4a707255e44a26b03", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -951,6 +1119,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.250Z", + "destination.bytes": 376452, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -959,21 +1129,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 253, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 1324000000, + "event.end": "2018-10-04T09:35:00.250Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 11147, + "message": "Not Suspicious Traffic", + "network.bytes": 388131, + "network.packets": 412, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 11679, "source.ip": "192.168.1.146", + "source.packets": 159, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1008,7 +1187,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/source/by-hash/SHA256/56cfd9cc2efa61dff7428dddf921c3cd6047ab8e6484a7f1888e4c3f7252f1ef", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1018,6 +1198,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.401Z", + "destination.bytes": 468170, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1026,21 +1208,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 314, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 1475000000, + "event.end": "2018-10-04T09:35:00.401Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 12048, + "message": "Not Suspicious Traffic", + "network.bytes": 482156, + "network.packets": 504, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 13986, "source.ip": "192.168.1.146", + "source.packets": 190, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1075,7 +1266,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/main/binary-amd64/by-hash/SHA256/4360137dc8f98b47648da1fef5472ef234fb02115bc2b29873bcaeee62637e70", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1085,6 +1277,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.776Z", + "destination.bytes": 880323, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1093,21 +1287,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 588, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 1850000000, + "event.end": "2018-10-04T09:35:00.776Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 12951, + "message": "Not Suspicious Traffic", + "network.bytes": 903684, + "network.packets": 916, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 23361, "source.ip": "192.168.1.146", + "source.packets": 328, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1142,7 +1345,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/restricted/binary-amd64/by-hash/SHA256/c93fdc7f10cad1263349fd7b5bdd6a7f7163165b96ad263b3e12022e319d0d12", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1152,6 +1356,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.897Z", + "destination.bytes": 884342, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1160,21 +1366,30 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 591, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 1971000000, + "event.end": "2018-10-04T09:35:00.897Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 13860, + "message": "Not Suspicious Traffic", + "network.bytes": 908100, + "network.packets": 921, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 23758, "source.ip": "192.168.1.146", + "source.packets": 330, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1209,7 +1424,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/binary-amd64/by-hash/SHA256/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1219,6 +1435,8 @@ }, { "@timestamp": "2018-10-04T09:35:01.362Z", + "destination.bytes": 1467603, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1227,20 +1445,29 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 979, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 2436000000, + "event.end": "2018-10-04T09:35:01.362Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "input.type": "log", "log.offset": 14767, + "message": "Not Suspicious Traffic", + "network.bytes": 1504422, + "network.packets": 1503, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 36819, "source.ip": "192.168.1.146", + "source.packets": 524, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1274,7 +1501,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/i18n/by-hash/SHA256/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1284,6 +1512,8 @@ }, { "@timestamp": "2018-10-04T09:35:01.575Z", + "destination.bytes": 1618380, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1292,20 +1522,29 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 1079, "destination.port": 80, "ecs.version": "1.0.0-beta2", - "event.action": "Not Suspicious Traffic", "event.dataset": "suricata.eve", + "event.duration": 2649000000, + "event.end": "2018-10-04T09:35:01.575Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "input.type": "log", "log.offset": 15651, + "message": "Not Suspicious Traffic", + "network.bytes": 1658832, + "network.packets": 1654, + "network.protocol": "http", "network.transport": "tcp", + "source.bytes": 40452, "source.ip": "192.168.1.146", + "source.packets": 575, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1339,7 +1578,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/multiverse/binary-amd64/by-hash/SHA256/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index faa31236bad..bd485f51d0f 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -5,6 +5,7 @@ "destination.port": 22, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:01:09.820Z", "event.module": "suricata", "event.type": "ssh", "fileset.name": "eve", @@ -32,20 +33,30 @@ }, { "@timestamp": "2018-07-05T19:07:20.910Z", + "destination.bytes": 343, "destination.ip": "192.168.156.70", + "destination.packets": 3, "destination.port": 443, "ecs.version": "1.0.0-beta2", - "event.action": "Potential Corporate Privacy Violation", "event.dataset": "suricata.eve", + "event.duration": 1251000000, + "event.end": "2018-07-05T19:07:20.910Z", "event.module": "suricata", "event.outcome": "allowed", "event.severity": 1, + "event.start": "2018-07-05T19:07:19.659Z", "event.type": "alert", "fileset.name": "eve", "input.type": "log", "log.offset": 350, + "message": "Potential Corporate Privacy Violation", + "network.bytes": 1136, + "network.packets": 7, + "network.protocol": "tls", "network.transport": "tcp", + "source.bytes": 793, "source.ip": "192.168.86.85", + "source.packets": 4, "source.port": 55641, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Potential Corporate Privacy Violation", @@ -79,14 +90,16 @@ }, { "@timestamp": "2018-07-05T19:43:47.690Z", + "destination.domain": "192.168.86.28", "destination.ip": "192.168.86.28", "destination.port": 63963, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:43:47.690Z", "event.module": "suricata", "event.type": "http", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 985, @@ -114,7 +127,8 @@ "tags": [ "suricata" ], - "url.hostname": "192.168.86.28", + "url.domain": "192.168.86.28", + "url.path": "/dd.xml", "user_agent.device": "Other", "user_agent.major": "67", "user_agent.minor": "0", @@ -127,19 +141,22 @@ }, { "@timestamp": "2018-07-05T19:44:33.222Z", + "destination.domain": "192.168.86.28", "destination.ip": "192.168.86.85", "destination.port": 56118, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:44:33.222Z", "event.module": "suricata", "event.type": "fileinfo", "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", "http.response.status_code": "200", "input.type": "log", "log.offset": 1507, + "network.protocol": "http", "network.transport": "tcp", "source.ip": "192.168.86.28", "source.port": 8008, @@ -173,7 +190,8 @@ "tags": [ "suricata" ], - "url.hostname": "192.168.86.28", + "url.domain": "192.168.86.28", + "url.path": "/ssdp/device-desc.xml", "user_agent.device": "Other", "user_agent.major": "67", "user_agent.minor": "0", @@ -190,6 +208,7 @@ "destination.port": 39464, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:20.213Z", "event.module": "suricata", "event.type": "dns", "fileset.name": "eve", @@ -222,6 +241,7 @@ "@timestamp": "2018-07-05T19:51:23.009Z", "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:23.009Z", "event.module": "suricata", "event.type": "stats", "fileset.name": "eve", @@ -359,6 +379,7 @@ "destination.port": 443, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:50.666Z", "event.module": "suricata", "event.type": "tls", "fileset.name": "eve", @@ -390,17 +411,27 @@ }, { "@timestamp": "2018-07-05T19:51:54.001Z", + "destination.bytes": 0, "destination.ip": "ff02:0000:0000:0000:0000:0000:0001:0002", + "destination.packets": 0, "destination.port": 547, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 30548000000, + "event.end": "2018-07-05T19:51:54.001Z", "event.module": "suricata", + "event.start": "2018-07-05T19:51:23.453Z", "event.type": "flow", "fileset.name": "eve", "input.type": "log", "log.offset": 5308, + "network.bytes": 110, + "network.packets": 1, + "network.protocol": "failed", "network.transport": "udp", + "source.bytes": 110, "source.ip": "fe80:0000:0000:0000:fada:0cff:fedc:87f1", + "source.packets": 1, "source.port": 546, "suricata.eve.app_proto": "failed", "suricata.eve.dest_ip": "ff02:0000:0000:0000:0000:0000:0001:0002", From dd802e258c2a7b4a04f2f4a96d132c33403e77b6 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 14 Jan 2019 15:44:45 +0100 Subject: [PATCH 11/17] Populate ECS field `http.response.body.bytes` --- .../module/suricata/eve/ingest/pipeline.json | 7 +++++++ .../eve/test/eve-alerts.log-expected.json | 20 +++++++++++++++++++ .../eve/test/eve-small.log-expected.json | 2 ++ 3 files changed, 29 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index ca1ce74a756..d3cdb4cff41 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -85,6 +85,13 @@ ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.http.length" + ,"target_field": "http.response.body.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } , {"convert": {"field": "suricata.eve.fileinfo.filename" ,"target_field": "file.path" diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 0d8a518f50e..888a61600c8 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -24,6 +24,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 0, @@ -105,6 +106,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 723, @@ -186,6 +188,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1126, "http.response.status_code": "200", "input.type": "log", "log.offset": 1445, @@ -267,6 +270,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 2168, @@ -348,6 +352,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 2889, @@ -429,6 +434,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1126, "http.response.status_code": "200", "input.type": "log", "log.offset": 3611, @@ -510,6 +516,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1138, "http.response.status_code": "200", "input.type": "log", "log.offset": 4334, @@ -589,6 +596,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 0, "http.response.status_code": "304", "input.type": "log", "log.offset": 5140, @@ -668,6 +676,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2601, "http.response.status_code": "200", "input.type": "log", "log.offset": 5931, @@ -747,6 +756,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1241, "http.response.status_code": "200", "input.type": "log", "log.offset": 6734, @@ -826,6 +836,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 7630, @@ -905,6 +916,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2688, "http.response.status_code": "200", "input.type": "log", "log.offset": 8533, @@ -984,6 +996,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2601, "http.response.status_code": "200", "input.type": "log", "log.offset": 9443, @@ -1063,6 +1076,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 10252, @@ -1142,6 +1156,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2688, "http.response.status_code": "200", "input.type": "log", "log.offset": 11147, @@ -1221,6 +1236,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 12048, @@ -1300,6 +1316,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2691, "http.response.status_code": "200", "input.type": "log", "log.offset": 12951, @@ -1379,6 +1396,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 13860, @@ -1458,6 +1476,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 0, "input.type": "log", "log.offset": 14767, "message": "Not Suspicious Traffic", @@ -1535,6 +1554,7 @@ "event.type": "alert", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 0, "input.type": "log", "log.offset": 15651, "message": "Not Suspicious Traffic", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index bd485f51d0f..0c36d4e677b 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -100,6 +100,7 @@ "event.type": "http", "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1155, "http.response.status_code": "200", "input.type": "log", "log.offset": 985, @@ -153,6 +154,7 @@ "file.size": 1071, "fileset.name": "eve", "http.request.method": "get", + "http.response.body.bytes": 1071, "http.response.status_code": "200", "input.type": "log", "log.offset": 1507, From 1643e1b6b917ff5dac1448b849a4cc6fddee69d2 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 14 Jan 2019 17:19:48 +0100 Subject: [PATCH 12/17] Use grok pattern to parse url fields Replace ugly painless code. --- .../module/suricata/eve/ingest/pipeline.json | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index d3cdb4cff41..34cdc5b151d 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -59,16 +59,15 @@ ,"ignore_missing": true } } - , {"convert": - {"field": "suricata.eve.http.url" - ,"target_field": "url.path" - ,"type": "string" - ,"ignore_missing": true - } - } - , { "script": - { "lang": "painless" - , "source": "String[] split(String s, String sep) { def p=s.indexOf(sep); return p>=0? new String[]{s.substring(0,p), s.substring(p+1)} : new String[]{s};} def u=ctx['url']; def p=u!=null? u['path'] : null; if(p!=null) {def q=split(p,'#');if(q.length>1)u['fragment']=q[1];q=split(q[0],'?');if(q.length>1)u['query']=q[1];u['path']=q[0];}" + , { "grok": + { "field": "suricata.eve.http.url" + , "patterns": ["%{PATH:url.path}(?:\\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?"] + , "ignore_missing": true + , "pattern_definitions": + { "PATH": "[^?#]*" + , "QUERY": "[^#]*" + , "ANY": ".*" + } } } , {"convert": From 3dfeb445d42d6cdcc7e34b4de777afb2b16e3242 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 14 Jan 2019 18:59:56 +0100 Subject: [PATCH 13/17] Avoid pairs of convert/lowercase Lowercase processor can have a target field so its not neccesary to copy the field in a previous step. --- .../module/suricata/eve/ingest/pipeline.json | 40 +++++-------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 34cdc5b151d..d19c5147d60 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -33,16 +33,10 @@ ,"ignore_missing": true } } - , {"convert": - {"field": "suricata.eve.http.http_method" - ,"target_field": "http.request.method" - ,"type": "string" - ,"ignore_missing": true - } - } - , { "lowercase": - { "field": "http.request.method" - , "ignore_missing": true + , {"lowercase": + {"field": "suricata.eve.http.http_method" + ,"target_field": "http.request.method" + ,"ignore_missing": true } } , {"convert": @@ -122,14 +116,10 @@ , { "lowercase": { "field": "suricata.eve.event_type" + , "target_field": "event.type" , "ignore_missing": true } } - , { "set": - { "field": "event.type" - , "value": "{{suricata.eve.event_type}}" - } - } , {"convert": {"field": "suricata.eve.alert.category" ,"target_field": "message" @@ -203,27 +193,15 @@ , "source": "Instant ins(def d){try{return Instant.parse(d);}catch(Exception e){return null;}}def ev=ctx['event'];if(ev!=null){def start=ins(ev['start']); def end=ins(ev['end']); if(start!=null && end!=null && !start.isAfter(end)) {ev['duration'] = Duration.between(start,end).toNanos();}}" } } - , {"convert": - {"field": "suricata.eve.proto" - ,"target_field": "network.transport" - ,"type": "string" - ,"ignore_missing": true - } - } , { "lowercase": - { "field": "network.transport" + { "field": "suricata.eve.proto" + , "target_field": "network.transport" , "ignore_missing": true } } - , {"convert": - {"field": "suricata.eve.app_proto" - ,"target_field": "network.protocol" - ,"type": "string" - ,"ignore_missing": true - } - } , { "lowercase": - { "field": "network.protocol" + { "field": "suricata.eve.app_proto" + , "target_field": "network.protocol" , "ignore_missing": true } } From 6ba3199bfd35e46766328f0ea59d4ca8514a47a3 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 14 Jan 2019 19:39:26 +0100 Subject: [PATCH 14/17] Cleanup painless script --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index d19c5147d60..702061c9203 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -171,7 +171,7 @@ } , { "script": { "lang": "painless" - , "source": "long g(def map, def key) { if(map!=null && map[key]!=null) { return map[key]; } return 0; } def n=ctx['network']; if (n==null){n=new HashMap(); ctx['network']=n;}def s=ctx['source'], d=ctx['destination']; def sp=g(s,'packets'), sb=g(s,'bytes'), dp=g(d,'packets'), db=g(d,'bytes'); if(sb+db>0)n['bytes']=sb+db; if(sp+dp>0)n['packets']=sp+dp;" + , "source": "long getOrZero(def map, def key) { if(map!=null && map[key]!=null) { return map[key]; } return 0; } def network=ctx['network'], source=ctx['source'], dest=ctx['destination']; def sp=getOrZero(source,'packets'), sb=getOrZero(source,'bytes'), dp=getOrZero(dest,'packets'), db=getOrZero(dest,'bytes'); if(sb+db+sp+dp > 0){if (network==null){network=new HashMap(); ctx['network']=network; } if(sb+db>0) network['bytes'] = sb+db; if(sp+dp>0) network['packets'] = sp+dp; }" } } , {"date": From c114f83af9eb61ec7648820af900aba36e1f4ede Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 16 Jan 2019 20:42:29 +0100 Subject: [PATCH 15/17] Fix golden data --- .../module/suricata/eve/test/eve-small.log-expected.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 0c36d4e677b..f2960c9c067 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -458,4 +458,4 @@ "suricata" ] } -] +] \ No newline at end of file From 5fed5a9dab8ad3de51dcb123e33de7df7e5cc284 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 16 Jan 2019 22:49:37 +0100 Subject: [PATCH 16/17] Fix golden data (2) --- .../eve/test/eve-alerts.log-expected.json | 20 +++++++++++++++++++ .../eve/test/eve-small.log-expected.json | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 888a61600c8..7ad628a7b53 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -33,6 +33,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -115,6 +116,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -197,6 +199,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -279,6 +282,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -361,6 +365,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -443,6 +448,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 347, "source.ip": "192.168.1.146", "source.packets": 4, @@ -525,6 +531,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 497, "source.ip": "192.168.1.146", "source.packets": 4, @@ -605,6 +612,7 @@ "network.packets": 7, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 487, "source.ip": "192.168.1.146", "source.packets": 4, @@ -685,6 +693,7 @@ "network.packets": 11, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 842, "source.ip": "192.168.1.146", "source.packets": 6, @@ -765,6 +774,7 @@ "network.packets": 126, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 4810, "source.ip": "192.168.1.146", "source.packets": 64, @@ -845,6 +855,7 @@ "network.packets": 185, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 6591, "source.ip": "192.168.1.146", "source.packets": 87, @@ -925,6 +936,7 @@ "network.packets": 377, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 11460, "source.ip": "192.168.1.146", "source.packets": 156, @@ -1005,6 +1017,7 @@ "network.packets": 131, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 4895, "source.ip": "192.168.1.146", "source.packets": 64, @@ -1085,6 +1098,7 @@ "network.packets": 210, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 6932, "source.ip": "192.168.1.146", "source.packets": 91, @@ -1165,6 +1179,7 @@ "network.packets": 412, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 11679, "source.ip": "192.168.1.146", "source.packets": 159, @@ -1245,6 +1260,7 @@ "network.packets": 504, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 13986, "source.ip": "192.168.1.146", "source.packets": 190, @@ -1325,6 +1341,7 @@ "network.packets": 916, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 23361, "source.ip": "192.168.1.146", "source.packets": 328, @@ -1405,6 +1422,7 @@ "network.packets": 921, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 23758, "source.ip": "192.168.1.146", "source.packets": 330, @@ -1484,6 +1502,7 @@ "network.packets": 1503, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 36819, "source.ip": "192.168.1.146", "source.packets": 524, @@ -1562,6 +1581,7 @@ "network.packets": 1654, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 40452, "source.ip": "192.168.1.146", "source.packets": 575, diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index f2960c9c067..6b48f369361 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -12,6 +12,7 @@ "input.type": "log", "log.offset": 0, "network.transport": "tcp", + "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 55406, "suricata.eve.dest_ip": "192.168.253.112", @@ -54,6 +55,7 @@ "network.packets": 7, "network.protocol": "tls", "network.transport": "tcp", + "service.type": "suricata", "source.bytes": 793, "source.ip": "192.168.86.85", "source.packets": 4, @@ -105,6 +107,7 @@ "input.type": "log", "log.offset": 985, "network.transport": "tcp", + "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 56119, "suricata.eve.dest_ip": "192.168.86.28", @@ -160,6 +163,7 @@ "log.offset": 1507, "network.protocol": "http", "network.transport": "tcp", + "service.type": "suricata", "source.ip": "192.168.86.28", "source.port": 8008, "suricata.eve.app_proto": "http", @@ -217,6 +221,7 @@ "input.type": "log", "log.offset": 2347, "network.transport": "udp", + "service.type": "suricata", "source.ip": "192.168.86.1", "source.port": 53, "suricata.eve.dest_ip": "192.168.86.85", @@ -249,6 +254,7 @@ "fileset.name": "eve", "input.type": "log", "log.offset": 2687, + "service.type": "suricata", "suricata.eve.event_type": "stats", "suricata.eve.stats.app_layer.flow.dcerpc_tcp": 0, "suricata.eve.stats.app_layer.flow.dcerpc_udp": 0, @@ -388,6 +394,7 @@ "input.type": "log", "log.offset": 4683, "network.transport": "tcp", + "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 56187, "suricata.eve.dest_ip": "17.142.164.13", @@ -431,6 +438,7 @@ "network.packets": 1, "network.protocol": "failed", "network.transport": "udp", + "service.type": "suricata", "source.bytes": 110, "source.ip": "fe80:0000:0000:0000:fada:0cff:fedc:87f1", "source.packets": 1, From af6841890ba7617a5e6dd9e614f8ca7282ada276 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 18 Jan 2019 21:55:21 +0100 Subject: [PATCH 17/17] Copy timestamp to event.end instead of parsing date again --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 702061c9203..ed700bc0689 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -181,11 +181,9 @@ ,"ignore_failure": true } } - , {"date": - {"field": "suricata.eve.timestamp" - ,"target_field": "event.end" - , "formats": ["ISO8601"] - ,"ignore_failure": true + , {"set": + {"field": "event.end" + ,"value": "{{@timestamp}}" } } , { "script":