diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 6b2b8ebd5f6..ed700bc0689 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -33,27 +33,58 @@ ,"ignore_missing": true } } + , {"lowercase": + {"field": "suricata.eve.http.http_method" + ,"target_field": "http.request.method" + ,"ignore_missing": true + } + } , {"convert": - {"field": "suricata.eve.http.http_method" - ,"target_field": "http.request.method" + {"field": "suricata.eve.http.status" + ,"target_field": "http.response.status_code" ,"type": "string" ,"ignore_missing": true } } , {"convert": - {"field": "suricata.eve.http.status" - ,"target_field": "http.response.status_code" + {"field": "suricata.eve.http.hostname" + ,"target_field": "url.domain" ,"type": "string" ,"ignore_missing": true } } + , { "grok": + { "field": "suricata.eve.http.url" + , "patterns": ["%{PATH:url.path}(?:\\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?"] + , "ignore_missing": true + , "pattern_definitions": + { "PATH": "[^?#]*" + , "QUERY": "[^#]*" + , "ANY": ".*" + } + } + } , {"convert": {"field": "suricata.eve.http.hostname" - ,"target_field": "url.hostname" + ,"target_field": "destination.domain" ,"type": "string" ,"ignore_missing": true } } + , {"convert": + {"field": "suricata.eve.http.http_refer" + ,"target_field": "http.request.referrer" + ,"type": "string" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.http.length" + ,"target_field": "http.response.body.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } , {"convert": {"field": "suricata.eve.fileinfo.filename" ,"target_field": "file.path" @@ -85,15 +116,93 @@ , { "lowercase": { "field": "suricata.eve.event_type" + , "target_field": "event.type" , "ignore_missing": true } } - , { "set": - { "field": "event.type" - , "value": "{{suricata.eve.event_type}}" + , {"convert": + {"field": "suricata.eve.alert.category" + ,"target_field": "message" + ,"type": "string" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.alert.action" + ,"target_field": "event.outcome" + ,"type": "string" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.alert.severity" + ,"target_field": "event.severity" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.pkts_toclient" + ,"target_field": "destination.packets" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.pkts_toserver" + ,"target_field": "source.packets" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.bytes_toclient" + ,"target_field": "destination.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } + , {"convert": + {"field": "suricata.eve.flow.bytes_toserver" + ,"target_field": "source.bytes" + ,"type": "integer" + ,"ignore_missing": true + } + } + , { "script": + { "lang": "painless" + , "source": "long getOrZero(def map, def key) { if(map!=null && map[key]!=null) { return map[key]; } return 0; } def network=ctx['network'], source=ctx['source'], dest=ctx['destination']; def sp=getOrZero(source,'packets'), sb=getOrZero(source,'bytes'), dp=getOrZero(dest,'packets'), db=getOrZero(dest,'bytes'); if(sb+db+sp+dp > 0){if (network==null){network=new HashMap(); ctx['network']=network; } if(sb+db>0) network['bytes'] = sb+db; if(sp+dp>0) network['packets'] = sp+dp; }" + } + } + , {"date": + {"field": "suricata.eve.flow.start" + ,"target_field": "event.start" + ,"formats": ["ISO8601"] + ,"ignore_failure": true + } + } + , {"set": + {"field": "event.end" + ,"value": "{{@timestamp}}" + } + } + , { "script": + { "lang": "painless" + , "source": "Instant ins(def d){try{return Instant.parse(d);}catch(Exception e){return null;}}def ev=ctx['event'];if(ev!=null){def start=ins(ev['start']); def end=ins(ev['end']); if(start!=null && end!=null && !start.isAfter(end)) {ev['duration'] = Duration.between(start,end).toNanos();}}" + } + } + , { "lowercase": + { "field": "suricata.eve.proto" + , "target_field": "network.transport" + , "ignore_missing": true + } + } + , { "lowercase": + { "field": "suricata.eve.app_proto" + , "target_field": "network.protocol" + , "ignore_missing": true } } - , { "user_agent": { "field": "user_agent.original" , "target_field": "user_agent" diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json index 5ac3f24248d..7ad628a7b53 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-alerts.log-expected.json @@ -1,6 +1,8 @@ [ { "@timestamp": "2018-10-03T14:42:44.836Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -9,18 +11,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 223000000, + "event.end": "2018-10-03T14:42:44.836Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T14:42:44.613Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 0, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32858, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -56,7 +72,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -67,6 +84,8 @@ }, { "@timestamp": "2018-10-03T16:16:26.711Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -75,18 +94,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 244000000, + "event.end": "2018-10-03T16:16:26.711Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T16:16:26.467Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 723, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32864, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -122,7 +155,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -133,6 +167,8 @@ }, { "@timestamp": "2018-10-03T16:44:50.813Z", + "destination.bytes": 1654, + "destination.domain": "example.net", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -141,18 +177,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 233000000, + "event.end": "2018-10-03T16:44:50.813Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T16:44:50.580Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1126, "http.response.status_code": "200", "input.type": "log", "log.offset": 1445, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32870, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -188,7 +238,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.net", + "url.domain": "example.net", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -199,6 +250,8 @@ }, { "@timestamp": "2018-10-03T16:45:09.267Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -207,18 +260,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 231000000, + "event.end": "2018-10-03T16:45:09.267Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T16:45:09.036Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 2168, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32872, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -254,7 +321,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -265,6 +333,8 @@ }, { "@timestamp": "2018-10-03T16:45:34.481Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -273,18 +343,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 229000000, + "event.end": "2018-10-03T16:45:34.481Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T16:45:34.252Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1121, "http.response.status_code": "200", "input.type": "log", "log.offset": 2889, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32876, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -320,7 +404,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -331,6 +416,8 @@ }, { "@timestamp": "2018-10-03T17:02:38.900Z", + "destination.bytes": 1654, + "destination.domain": "example.org", "destination.geo.city_name": "Norwell", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -339,18 +426,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "93.184.216.34", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 301000000, + "event.end": "2018-10-03T17:02:38.900Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 2, + "event.start": "2018-10-03T17:02:38.599Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1126, "http.response.status_code": "200", "input.type": "log", "log.offset": 3611, + "message": "Attempted Information Leak", + "network.bytes": 2001, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 347, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 32892, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Attempted Information Leak", @@ -386,7 +487,8 @@ "tags": [ "suricata" ], - "url.hostname": "example.org", + "url.domain": "example.org", + "url.path": "/", "user_agent.device": "Other", "user_agent.major": "7", "user_agent.minor": "58", @@ -397,6 +499,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.009Z", + "destination.bytes": 1654, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -405,18 +509,32 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 85000000, + "event.end": "2018-10-04T09:34:59.009Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1138, "http.response.status_code": "200", "input.type": "log", "log.offset": 4334, + "message": "Not Suspicious Traffic", + "network.bytes": 2151, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 497, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -451,7 +569,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -461,6 +580,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.168Z", + "destination.bytes": 417, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -469,18 +590,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 3, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 242000000, + "event.end": "2018-10-04T09:34:59.168Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 0, "http.response.status_code": "304", "input.type": "log", "log.offset": 5140, + "message": "Not Suspicious Traffic", + "network.bytes": 904, + "network.packets": 7, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 487, "source.ip": "192.168.1.146", + "source.packets": 4, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -515,7 +650,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -525,6 +661,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.288Z", + "destination.bytes": 3445, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -533,18 +671,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 5, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 362000000, + "event.end": "2018-10-04T09:34:59.288Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2601, "http.response.status_code": "200", "input.type": "log", "log.offset": 5931, + "message": "Not Suspicious Traffic", + "network.bytes": 4287, + "network.packets": 11, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 842, "source.ip": "192.168.1.146", + "source.packets": 6, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -579,7 +731,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -589,6 +742,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.289Z", + "destination.bytes": 90543, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -597,18 +752,32 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 62, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 365000000, + "event.end": "2018-10-04T09:34:59.289Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1241, "http.response.status_code": "200", "input.type": "log", "log.offset": 6734, + "message": "Not Suspicious Traffic", + "network.bytes": 95353, + "network.packets": 126, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 4810, "source.ip": "192.168.1.146", + "source.packets": 64, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -643,7 +812,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/main/source/by-hash/SHA256/f5ec03d97ca76c98162d9233c8b7c578c52897e2136428277baf2e7b633a8e72", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -653,6 +823,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.356Z", + "destination.bytes": 145014, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -661,18 +833,32 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 98, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 432000000, + "event.end": "2018-10-04T09:34:59.356Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 7630, + "message": "Not Suspicious Traffic", + "network.bytes": 151605, + "network.packets": 185, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 6591, "source.ip": "192.168.1.146", + "source.packets": 87, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -707,7 +893,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/main/binary-amd64/by-hash/SHA256/c5b8346a3221bc9a23a79ba4dc4e730a6319a77fc9d63872dfc56539a0810015", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -717,6 +904,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.456Z", + "destination.bytes": 330525, + "destination.domain": "security.ubuntu.com", "destination.geo.city_name": "London", "destination.geo.continent_name": "Europe", "destination.geo.country_iso_code": "GB", @@ -725,18 +914,32 @@ "destination.geo.region_iso_code": "GB-ENG", "destination.geo.region_name": "England", "destination.ip": "91.189.88.152", + "destination.packets": 221, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 532000000, + "event.end": "2018-10-04T09:34:59.456Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.924Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2688, "http.response.status_code": "200", "input.type": "log", "log.offset": 8533, + "message": "Not Suspicious Traffic", + "network.bytes": 341985, + "network.packets": 377, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 11460, "source.ip": "192.168.1.146", + "source.packets": 156, "source.port": 37742, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -771,7 +974,8 @@ "tags": [ "suricata" ], - "url.hostname": "security.ubuntu.com", + "url.domain": "security.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-security/universe/binary-amd64/by-hash/SHA256/e5cc957139a25a0fee47cbf2c0fac8ad5cab50346d6a74abe031748924c5b558", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -781,6 +985,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.747Z", + "destination.bytes": 96554, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -789,18 +995,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 67, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 821000000, + "event.end": "2018-10-04T09:34:59.747Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2601, "http.response.status_code": "200", "input.type": "log", "log.offset": 9443, + "message": "Not Suspicious Traffic", + "network.bytes": 101449, + "network.packets": 131, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 4895, "source.ip": "192.168.1.146", + "source.packets": 64, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -835,7 +1055,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-backports/InRelease", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -845,6 +1066,8 @@ }, { "@timestamp": "2018-10-04T09:34:59.953Z", + "destination.bytes": 174843, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -853,18 +1076,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 119, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1027000000, + "event.end": "2018-10-04T09:34:59.953Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 10252, + "message": "Not Suspicious Traffic", + "network.bytes": 181775, + "network.packets": 210, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 6932, "source.ip": "192.168.1.146", + "source.packets": 91, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -899,7 +1136,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/main/source/by-hash/SHA256/65f2e3a4e9d89d9d4b5e3d42e586bc96f48a24466b0ad0b4a707255e44a26b03", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -909,6 +1147,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.250Z", + "destination.bytes": 376452, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -917,18 +1157,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 253, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1324000000, + "event.end": "2018-10-04T09:35:00.250Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2688, "http.response.status_code": "200", "input.type": "log", "log.offset": 11147, + "message": "Not Suspicious Traffic", + "network.bytes": 388131, + "network.packets": 412, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 11679, "source.ip": "192.168.1.146", + "source.packets": 159, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -963,7 +1217,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/source/by-hash/SHA256/56cfd9cc2efa61dff7428dddf921c3cd6047ab8e6484a7f1888e4c3f7252f1ef", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -973,6 +1228,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.401Z", + "destination.bytes": 468170, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -981,18 +1238,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 314, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1475000000, + "event.end": "2018-10-04T09:35:00.401Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 12048, + "message": "Not Suspicious Traffic", + "network.bytes": 482156, + "network.packets": 504, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 13986, "source.ip": "192.168.1.146", + "source.packets": 190, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1027,7 +1298,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/main/binary-amd64/by-hash/SHA256/4360137dc8f98b47648da1fef5472ef234fb02115bc2b29873bcaeee62637e70", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1037,6 +1309,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.776Z", + "destination.bytes": 880323, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1045,18 +1319,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 588, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1850000000, + "event.end": "2018-10-04T09:35:00.776Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2691, "http.response.status_code": "200", "input.type": "log", "log.offset": 12951, + "message": "Not Suspicious Traffic", + "network.bytes": 903684, + "network.packets": 916, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 23361, "source.ip": "192.168.1.146", + "source.packets": 328, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1091,7 +1379,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/restricted/binary-amd64/by-hash/SHA256/c93fdc7f10cad1263349fd7b5bdd6a7f7163165b96ad263b3e12022e319d0d12", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1101,6 +1390,8 @@ }, { "@timestamp": "2018-10-04T09:35:00.897Z", + "destination.bytes": 884342, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1109,18 +1400,32 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 591, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1971000000, + "event.end": "2018-10-04T09:35:00.897Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 2687, "http.response.status_code": "200", "input.type": "log", "log.offset": 13860, + "message": "Not Suspicious Traffic", + "network.bytes": 908100, + "network.packets": 921, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 23758, "source.ip": "192.168.1.146", + "source.packets": 330, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1155,7 +1460,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/binary-amd64/by-hash/SHA256/5190f7afbee38b3cb32225db478fdbabd46f76eaa9c5921a13091891bf3e9bbc", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1165,6 +1471,8 @@ }, { "@timestamp": "2018-10-04T09:35:01.362Z", + "destination.bytes": 1467603, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1173,17 +1481,31 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 979, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 2436000000, + "event.end": "2018-10-04T09:35:01.362Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 0, "input.type": "log", "log.offset": 14767, + "message": "Not Suspicious Traffic", + "network.bytes": 1504422, + "network.packets": 1503, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 36819, "source.ip": "192.168.1.146", + "source.packets": 524, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1217,7 +1539,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/universe/i18n/by-hash/SHA256/9fe539b7036e51327cd85ca5e0a4dd4eb47f69168875de2ac9842a5e36ebd4a4", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", @@ -1227,6 +1550,8 @@ }, { "@timestamp": "2018-10-04T09:35:01.575Z", + "destination.bytes": 1618380, + "destination.domain": "archive.ubuntu.com", "destination.geo.city_name": "Boston", "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", @@ -1235,17 +1560,31 @@ "destination.geo.region_iso_code": "US-MA", "destination.geo.region_name": "Massachusetts", "destination.ip": "91.189.91.23", + "destination.packets": 1079, "destination.port": 80, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 2649000000, + "event.end": "2018-10-04T09:35:01.575Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 3, + "event.start": "2018-10-04T09:34:58.926Z", "event.type": "alert", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 0, "input.type": "log", "log.offset": 15651, + "message": "Not Suspicious Traffic", + "network.bytes": 1658832, + "network.packets": 1654, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 40452, "source.ip": "192.168.1.146", + "source.packets": 575, "source.port": 52340, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Not Suspicious Traffic", @@ -1279,7 +1618,8 @@ "tags": [ "suricata" ], - "url.hostname": "archive.ubuntu.com", + "url.domain": "archive.ubuntu.com", + "url.path": "/ubuntu/dists/bionic-updates/multiverse/binary-amd64/by-hash/SHA256/8ab8cb220c0e50521c589acc2bc2b43a3121210f0b035a0605972bcffd73dd16", "user_agent.device": "Other", "user_agent.major": "1", "user_agent.minor": "3", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index aa2b53b6c7e..6b48f369361 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -5,11 +5,13 @@ "destination.port": 22, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:01:09.820Z", "event.module": "suricata", "event.type": "ssh", "fileset.name": "eve", "input.type": "log", "log.offset": 0, + "network.transport": "tcp", "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 55406, @@ -32,17 +34,31 @@ }, { "@timestamp": "2018-07-05T19:07:20.910Z", + "destination.bytes": 343, "destination.ip": "192.168.156.70", + "destination.packets": 3, "destination.port": 443, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 1251000000, + "event.end": "2018-07-05T19:07:20.910Z", "event.module": "suricata", + "event.outcome": "allowed", + "event.severity": 1, + "event.start": "2018-07-05T19:07:19.659Z", "event.type": "alert", "fileset.name": "eve", "input.type": "log", "log.offset": 350, + "message": "Potential Corporate Privacy Violation", + "network.bytes": 1136, + "network.packets": 7, + "network.protocol": "tls", + "network.transport": "tcp", "service.type": "suricata", + "source.bytes": 793, "source.ip": "192.168.86.85", + "source.packets": 4, "source.port": 55641, "suricata.eve.alert.action": "allowed", "suricata.eve.alert.category": "Potential Corporate Privacy Violation", @@ -76,17 +92,21 @@ }, { "@timestamp": "2018-07-05T19:43:47.690Z", + "destination.domain": "192.168.86.28", "destination.ip": "192.168.86.28", "destination.port": 63963, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:43:47.690Z", "event.module": "suricata", "event.type": "http", "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1155, "http.response.status_code": "200", "input.type": "log", "log.offset": 985, + "network.transport": "tcp", "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 56119, @@ -111,7 +131,8 @@ "tags": [ "suricata" ], - "url.hostname": "192.168.86.28", + "url.domain": "192.168.86.28", + "url.path": "/dd.xml", "user_agent.device": "Other", "user_agent.major": "67", "user_agent.minor": "0", @@ -124,19 +145,24 @@ }, { "@timestamp": "2018-07-05T19:44:33.222Z", + "destination.domain": "192.168.86.28", "destination.ip": "192.168.86.85", "destination.port": 56118, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:44:33.222Z", "event.module": "suricata", "event.type": "fileinfo", "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", - "http.request.method": "GET", + "http.request.method": "get", + "http.response.body.bytes": 1071, "http.response.status_code": "200", "input.type": "log", "log.offset": 1507, + "network.protocol": "http", + "network.transport": "tcp", "service.type": "suricata", "source.ip": "192.168.86.28", "source.port": 8008, @@ -170,7 +196,8 @@ "tags": [ "suricata" ], - "url.hostname": "192.168.86.28", + "url.domain": "192.168.86.28", + "url.path": "/ssdp/device-desc.xml", "user_agent.device": "Other", "user_agent.major": "67", "user_agent.minor": "0", @@ -187,11 +214,13 @@ "destination.port": 39464, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:20.213Z", "event.module": "suricata", "event.type": "dns", "fileset.name": "eve", "input.type": "log", "log.offset": 2347, + "network.transport": "udp", "service.type": "suricata", "source.ip": "192.168.86.1", "source.port": 53, @@ -219,6 +248,7 @@ "@timestamp": "2018-07-05T19:51:23.009Z", "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:23.009Z", "event.module": "suricata", "event.type": "stats", "fileset.name": "eve", @@ -357,11 +387,13 @@ "destination.port": 443, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.end": "2018-07-05T19:51:50.666Z", "event.module": "suricata", "event.type": "tls", "fileset.name": "eve", "input.type": "log", "log.offset": 4683, + "network.transport": "tcp", "service.type": "suricata", "source.ip": "192.168.86.85", "source.port": 56187, @@ -388,17 +420,28 @@ }, { "@timestamp": "2018-07-05T19:51:54.001Z", + "destination.bytes": 0, "destination.ip": "ff02:0000:0000:0000:0000:0000:0001:0002", + "destination.packets": 0, "destination.port": 547, "ecs.version": "1.0.0-beta2", "event.dataset": "suricata.eve", + "event.duration": 30548000000, + "event.end": "2018-07-05T19:51:54.001Z", "event.module": "suricata", + "event.start": "2018-07-05T19:51:23.453Z", "event.type": "flow", "fileset.name": "eve", "input.type": "log", "log.offset": 5308, + "network.bytes": 110, + "network.packets": 1, + "network.protocol": "failed", + "network.transport": "udp", "service.type": "suricata", + "source.bytes": 110, "source.ip": "fe80:0000:0000:0000:fada:0cff:fedc:87f1", + "source.packets": 1, "source.port": 546, "suricata.eve.app_proto": "failed", "suricata.eve.dest_ip": "ff02:0000:0000:0000:0000:0000:0001:0002",