From af260377fe5dc8f53010ed1acbb7e4ec247786bd Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 5 Jan 2021 14:54:10 -0500 Subject: [PATCH 1/2] Improve Suricata error handling (#23236) This fixes several errors processing EVE logs. Fix null dereference of suricata.eve.http.status. Even for http events it can be null. Remove unused field mapping for suricata.eve.flow.end Improve error.message details in pipeline on_failure handlers. Add tags to script processors to improve error messages. Rename suricata.eve.http.http_port to url.port. Add remove processor to on_failure handler to remove any alias field to prevent indexing errors. Ignore errors parsing TLS version. The pipeline got a value of "UNDETERMINED" that caused an error. Ignore errors parsing TLS subject/issuer DNs with kv. If a value contain the value_split string like "Nutanix, Inc." it can fail. (cherry picked from commit 11c5367b33fb0a92d93c24fb6d382c1795aed5a5) --- CHANGELOG.next.asciidoc | 1 + .../module/suricata/eve/_meta/fields.yml | 3 - .../suricata/eve/ingest/dns-answer-v1.yml | 8 +- .../suricata/eve/ingest/dns-answer-v2.yml | 8 +- .../module/suricata/eve/ingest/dns.yml | 8 +- .../module/suricata/eve/ingest/pipeline.yml | 62 ++++- .../module/suricata/eve/ingest/tls.yml | 12 +- .../module/suricata/eve/test/eve-small.log | 4 + .../eve/test/eve-small.log-expected.json | 217 ++++++++++++++++++ x-pack/filebeat/module/suricata/fields.go | 2 +- 10 files changed, 303 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be77ada1edf..52be80b60a7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -255,6 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273] - Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777] +- Fix various processing errors in the Suricata module. {pull}23236[23236] *Heartbeat* diff --git a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml index dffb86e7ebe..0654abd3141 100644 --- a/x-pack/filebeat/module/suricata/eve/_meta/fields.yml +++ b/x-pack/filebeat/module/suricata/eve/_meta/fields.yml @@ -714,9 +714,6 @@ type: alias path: source.packets - - name: end - type: date - - name: alerted type: boolean diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml index e915537365d..b44d3ce8dd0 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml @@ -35,5 +35,9 @@ processors: } on_failure: - append: - field: error.message - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: >- + error in DNS v1 pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml index a9e77c28549..189798e11de 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml @@ -38,5 +38,9 @@ processors: } on_failure: - append: - field: error.message - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: >- + error in DNS v2 pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml index f2c1127ddee..edd15c1d918 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/dns.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml @@ -89,5 +89,9 @@ processors: ignore_missing: true on_failure: - append: - field: error.message - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: >- + error in DNS pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index 04c9d0e0557..92bf9800d14 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -11,6 +11,7 @@ processors: field: suricata.eve.event_type ignore_missing: true - script: + tag: suricata_ecs_categorize lang: painless ignore_failure: true params: @@ -120,11 +121,17 @@ processors: ignore_empty_value: true ## HTTP - set: - if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400 + if: >- + ctx?.suricata?.eve?.event_type == "http" && + ctx?.suricata?.eve?.http?.status != null && + ctx?.suricata?.eve?.http?.status < 400 field: event.outcome value: success - set: - if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status >= 400 + if: >- + ctx?.suricata?.eve?.event_type == "http" && + ctx?.suricata?.eve?.http?.status != null && + ctx?.suricata?.eve?.http?.status >= 400 field: event.outcome value: failure ## DNS @@ -179,7 +186,7 @@ processors: } ignore_failure: true - set: - if: "ctx?.network?.protocol == 'http'" + if: ctx?.network?.protocol == 'http' field: url.domain value: '{{destination.domain}}' ignore_empty_value: true @@ -196,6 +203,10 @@ processors: field: suricata.eve.http.url target_field: url.original ignore_missing: true + - rename: + field: suricata.eve.http.http_port + target_field: url.port + ignore_missing: true - rename: field: suricata.eve.http.http_refer target_field: http.request.referrer @@ -233,13 +244,13 @@ processors: value: "{{suricata.eve.alert.signature}}" ignore_empty_value: true - set: + if: ctx?.suricata?.eve?.alert?.action == 'blocked' field: suricata.eve.alert.action value: denied - if: "ctx?.suricata?.eve?.alert?.action == 'blocked'" - append: + if: ctx?.suricata?.eve?.alert?.action != null field: event.type value: "{{suricata.eve.alert.action}}" - if: "ctx?.suricata?.eve?.alert?.action != null" - remove: field: suricata.eve.alert.action ignore_failure: true @@ -264,6 +275,7 @@ processors: target_field: source.bytes ignore_missing: true - script: + tag: suricata_network_bytes_packets lang: painless source: > long getOrZero(def map, def key) { @@ -299,6 +311,7 @@ processors: - ISO8601 ignore_failure: true - script: + tag: suricata_event_duration lang: painless source: > Instant ins(def d) { @@ -324,12 +337,12 @@ processors: field: suricata.eve.http.http_user_agent ignore_missing: true - geoip: - if: ctx.source?.geo == null + if: ctx?.source?.geo == null field: source.ip target_field: source.geo ignore_missing: true - geoip: - if: ctx.destination?.geo == null + if: ctx?.destination?.geo == null field: destination.ip target_field: destination.geo ignore_missing: true @@ -366,9 +379,9 @@ processors: target_field: destination.as.organization.name ignore_missing: true - append: + if: ctx?.url?.domain != null && ctx.url.domain != '' field: related.hosts value: '{{url.domain}}' - if: ctx.url?.domain != null && ctx.url?.domain != '' allow_duplicates: false - append: if: ctx?.source?.ip != null @@ -389,6 +402,35 @@ processors: - suricata.eve.http.http_user_agent ignore_missing: true on_failure: - - set: + - append: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: >- + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} + - remove: + field: + # Remove all alias fields to ensure indexing succeeds. + - suricata.eve.alert.action + - suricata.eve.alert.severity + - suricata.eve.app_proto + - suricata.eve.dest_ip + - suricata.eve.dest_port + - suricata.eve.fileinfo.filename + - suricata.eve.fileinfo.size + - suricata.eve.flow.bytes_toclient + - suricata.eve.flow.bytes_toserver + - suricata.eve.flow.pkts_toclient + - suricata.eve.flow.pkts_toserver + - suricata.eve.flow.start + - suricata.eve.http.hostname + - suricata.eve.http.http_method + - suricata.eve.http.http_refer + - suricata.eve.http.http_user_agent + - suricata.eve.http.length + - suricata.eve.http.status + - suricata.eve.http.url + - suricata.eve.proto + - suricata.eve.src_ip + - suricata.eve.src_port + ignore_missing: true diff --git a/x-pack/filebeat/module/suricata/eve/ingest/tls.yml b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml index 2c84e0c1cb7..dd61fd93f6c 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/tls.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml @@ -3,9 +3,11 @@ description: Pipeline for Suricata TLS Events processors: - dissect: + if: ctx?.suricata?.eve?.tls?.version != "UNDETERMINED" field: suricata.eve.tls.version pattern: '%{tls.version_protocol} %{tls.version}' ignore_missing: true + ignore_failure: true - lowercase: field: tls.version_protocol ignore_missing: true @@ -35,6 +37,7 @@ processors: value_split: '=' target_field: suricata.eve.tls.kv_subject ignore_missing: true + ignore_failure: true - rename: field: suricata.eve.tls.kv_subject.C target_field: tls.server.x509.subject.country @@ -75,6 +78,7 @@ processors: value_split: '=' target_field: suricata.eve.tls.kv_issuerdn ignore_missing: true + ignore_failure: true - rename: field: suricata.eve.tls.kv_issuerdn.C target_field: tls.server.x509.issuer.country @@ -184,5 +188,9 @@ processors: ignore_missing: true on_failure: - append: - field: error.message - value: "{{ _ingest.on_failure_message }}" + field: error.message + value: >- + error in TLS pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log b/x-pack/filebeat/module/suricata/eve/test/eve-small.log index 45163a617e9..9cc157a9e75 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log @@ -6,3 +6,7 @@ {"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}} {"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}} {"timestamp":"2018-07-05T15:51:54.001329-0400","flow_id":1828507008887644,"event_type":"flow","src_ip":"fe80:0000:0000:0000:fada:0cff:fedc:87f1","src_port":546,"dest_ip":"ff02:0000:0000:0000:0000:0000:0001:0002","dest_port":547,"proto":"UDP","app_proto":"failed","flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":110,"bytes_toclient":0,"start":"2018-07-05T15:51:23.453468-0400","end":"2018-07-05T15:51:23.453468-0400","age":0,"state":"new","reason":"timeout","alerted":false}} +{"timestamp":"2020-12-09T16:02:43.000505+0000","flow_id":913701662641234,"in_iface":"eno6","event_type":"http","src_ip":"192.168.50.1","src_port":57134,"dest_ip":"192.168.50.1","dest_port":8080,"proto":"TCP","tx_id":0,"http":{"hostname":"ctldl.windowsupdate.com","url":"http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111","http_user_agent":"Microsoft-CryptoAPI/10.0","http_method":"GET","protocol":"HTTP/1.1","length":0}} +{"timestamp":"2020-12-09T16:02:58.005716+0000","flow_id":1298574590709840,"in_iface":"eno6","event_type":"tls","src_ip":"192.168.50.1","src_port":60614,"dest_ip":"192.168.50.1","dest_port":443,"proto":"TCP","tls":{"subject":"C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com","issuerdn":"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018","serial":"0D:CE:DC:BC:AF:92:56:B4:C5:41:40:71:26:5B:1D:53","fingerprint":"18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25","version":"TLS 1.2","notbefore":"2020-11-24T00:00:00","notafter":"2021-12-25T23:59:59","ja3":{},"ja3s":{"hash":"adc06261ef82c2e4688b3cf08c1b2f24","string":"771,159,65281"}}} +{"timestamp":"2020-12-09T16:03:00.179037+0000","flow_id":1097935193623328,"in_iface":"eno6","event_type":"http","src_ip":"192.168.50.1","src_port":50898,"dest_ip":"192.168.50.1","dest_port":8081,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.50.1","http_port":8081,"url":"/uuid","http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0","http_method":"POST","protocol":"HTTP/1.1","length":0}} +{"timestamp":"2020-12-09T16:03:50.083307+0000","flow_id":289459143040794,"in_iface":"eno6","event_type":"tls","src_ip":"192.168.50.1","src_port":12509,"dest_ip":"192.168.50.1","dest_port":443,"proto":"TCP","tls":{"sni":"www.example.com","version":"UNDETERMINED","ja3":{"hash":"44d502d471cfdb99c59bdfb0f220e5a8","string":"771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0"},"ja3s":{}}} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index e4de4700be8..66204334c43 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -536,5 +536,222 @@ "tags": [ "suricata" ] + }, + { + "@timestamp": "2020-12-09T16:02:43.000Z", + "destination.address": "192.168.50.1", + "destination.domain": "ctldl.windowsupdate.com", + "destination.ip": "192.168.50.1", + "destination.port": 8080, + "event.category": [ + "network", + "web" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"timestamp\":\"2020-12-09T16:02:43.000505+0000\",\"flow_id\":913701662641234,\"in_iface\":\"eno6\",\"event_type\":\"http\",\"src_ip\":\"192.168.50.1\",\"src_port\":57134,\"dest_ip\":\"192.168.50.1\",\"dest_port\":8080,\"proto\":\"TCP\",\"tx_id\":0,\"http\":{\"hostname\":\"ctldl.windowsupdate.com\",\"url\":\"http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111\",\"http_user_agent\":\"Microsoft-CryptoAPI/10.0\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"length\":0}}", + "event.type": [ + "access", + "protocol" + ], + "fileset.name": "eve", + "http.request.method": "GET", + "http.response.body.bytes": 0, + "input.type": "log", + "log.offset": 5796, + "network.community_id": "1:Y8m38aDR9cy/emlD86XGhosniqY=", + "network.protocol": "http", + "network.transport": "tcp", + "related.hosts": [ + "ctldl.windowsupdate.com" + ], + "related.ip": [ + "192.168.50.1" + ], + "service.type": "suricata", + "source.address": "192.168.50.1", + "source.ip": "192.168.50.1", + "source.port": 57134, + "suricata.eve.event_type": "http", + "suricata.eve.flow_id": 913701662641234, + "suricata.eve.http.protocol": "HTTP/1.1", + "suricata.eve.in_iface": "eno6", + "suricata.eve.tx_id": 0, + "tags": [ + "suricata" + ], + "url.domain": "ctldl.windowsupdate.com", + "url.original": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111", + "url.path": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", + "url.query": "111111111111", + "user_agent.device.name": "Other", + "user_agent.name": "Microsoft-CryptoAPI", + "user_agent.original": "Microsoft-CryptoAPI/10.0", + "user_agent.version": "10.0" + }, + { + "@timestamp": "2020-12-09T16:02:58.005Z", + "destination.address": "192.168.50.1", + "destination.ip": "192.168.50.1", + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"timestamp\":\"2020-12-09T16:02:58.005716+0000\",\"flow_id\":1298574590709840,\"in_iface\":\"eno6\",\"event_type\":\"tls\",\"src_ip\":\"192.168.50.1\",\"src_port\":60614,\"dest_ip\":\"192.168.50.1\",\"dest_port\":443,\"proto\":\"TCP\",\"tls\":{\"subject\":\"C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com\",\"issuerdn\":\"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018\",\"serial\":\"0D:CE:DC:BC:AF:92:56:B4:C5:41:40:71:26:5B:1D:53\",\"fingerprint\":\"18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25\",\"version\":\"TLS 1.2\",\"notbefore\":\"2020-11-24T00:00:00\",\"notafter\":\"2021-12-25T23:59:59\",\"ja3\":{},\"ja3s\":{\"hash\":\"adc06261ef82c2e4688b3cf08c1b2f24\",\"string\":\"771,159,65281\"}}}", + "event.type": [ + "protocol" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 6267, + "network.community_id": "1:SKXuhLNyv4gfe01gqILs5v+qx40=", + "network.protocol": "tls", + "network.transport": "tcp", + "related.hash": [ + "183C114546E926C787640FED47861B31BF0F8425" + ], + "related.ip": [ + "192.168.50.1" + ], + "service.type": "suricata", + "source.address": "192.168.50.1", + "source.ip": "192.168.50.1", + "source.port": 60614, + "suricata.eve.event_type": "tls", + "suricata.eve.flow_id": 1298574590709840, + "suricata.eve.in_iface": "eno6", + "suricata.eve.tls.fingerprint": "18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25", + "suricata.eve.tls.issuerdn": "C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018", + "suricata.eve.tls.ja3s.hash": "adc06261ef82c2e4688b3cf08c1b2f24", + "suricata.eve.tls.ja3s.string": "771,159,65281", + "suricata.eve.tls.notafter": "2021-12-25T23:59:59", + "suricata.eve.tls.notbefore": "2020-11-24T00:00:00", + "suricata.eve.tls.serial": "0D:CE:DC:BC:AF:92:56:B4:C5:41:40:71:26:5B:1D:53", + "suricata.eve.tls.subject": "C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com", + "suricata.eve.tls.version": "TLS 1.2", + "tags": [ + "suricata" + ], + "tls.server.hash.sha1": "183C114546E926C787640FED47861B31BF0F8425", + "tls.server.issuer": "C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018", + "tls.server.ja3s": "adc06261ef82c2e4688b3cf08c1b2f24", + "tls.server.not_after": "2021-12-25T23:59:59.000Z", + "tls.server.not_before": "2020-11-24T00:00:00.000Z", + "tls.server.subject": "C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com", + "tls.server.x509.issuer.common_name": "GeoTrust RSA CA 2018", + "tls.server.x509.issuer.country": "US", + "tls.server.x509.issuer.organization": "DigiCert Inc", + "tls.server.x509.issuer.organizational_unit": "www.digicert.com", + "tls.server.x509.not_after": "2021-12-25T23:59:59.000Z", + "tls.server.x509.not_before": "2020-11-24T00:00:00.000Z", + "tls.server.x509.serial_number": "0DCEDCBCAF9256B4C5414071265B1D53", + "tls.server.x509.subject.country": "US", + "tls.server.x509.subject.locality": "New York City", + "tls.server.x509.subject.organization": "Acme U.S.A.", + "tls.server.x509.subject.state_or_province": "New York", + "tls.version": "1.2", + "tls.version_protocol": "tls" + }, + { + "@timestamp": "2020-12-09T16:03:00.179Z", + "destination.address": "192.168.50.1", + "destination.domain": "192.168.50.1", + "destination.ip": "192.168.50.1", + "destination.port": 8081, + "event.category": [ + "network", + "web" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"timestamp\":\"2020-12-09T16:03:00.179037+0000\",\"flow_id\":1097935193623328,\"in_iface\":\"eno6\",\"event_type\":\"http\",\"src_ip\":\"192.168.50.1\",\"src_port\":50898,\"dest_ip\":\"192.168.50.1\",\"dest_port\":8081,\"proto\":\"TCP\",\"tx_id\":0,\"http\":{\"hostname\":\"192.168.50.1\",\"http_port\":8081,\"url\":\"/uuid\",\"http_user_agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0\",\"http_method\":\"POST\",\"protocol\":\"HTTP/1.1\",\"length\":0}}", + "event.type": [ + "access", + "protocol" + ], + "fileset.name": "eve", + "http.request.method": "POST", + "http.response.body.bytes": 0, + "input.type": "log", + "log.offset": 6958, + "network.community_id": "1:UHWPAQmxXu8t7EWZzPx9jl6b6TM=", + "network.protocol": "http", + "network.transport": "tcp", + "related.hosts": [ + "192.168.50.1" + ], + "related.ip": [ + "192.168.50.1" + ], + "service.type": "suricata", + "source.address": "192.168.50.1", + "source.ip": "192.168.50.1", + "source.port": 50898, + "suricata.eve.event_type": "http", + "suricata.eve.flow_id": 1097935193623328, + "suricata.eve.http.protocol": "HTTP/1.1", + "suricata.eve.in_iface": "eno6", + "suricata.eve.tx_id": 0, + "tags": [ + "suricata" + ], + "url.domain": "192.168.50.1", + "url.original": "/uuid", + "url.path": "/uuid", + "url.port": 8081, + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "84.0." + }, + { + "@timestamp": "2020-12-09T16:03:50.083Z", + "destination.address": "192.168.50.1", + "destination.domain": "www.example.com", + "destination.ip": "192.168.50.1", + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "suricata.eve", + "event.kind": "event", + "event.module": "suricata", + "event.original": "{\"timestamp\":\"2020-12-09T16:03:50.083307+0000\",\"flow_id\":289459143040794,\"in_iface\":\"eno6\",\"event_type\":\"tls\",\"src_ip\":\"192.168.50.1\",\"src_port\":12509,\"dest_ip\":\"192.168.50.1\",\"dest_port\":443,\"proto\":\"TCP\",\"tls\":{\"sni\":\"www.example.com\",\"version\":\"UNDETERMINED\",\"ja3\":{\"hash\":\"44d502d471cfdb99c59bdfb0f220e5a8\",\"string\":\"771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0\"},\"ja3s\":{}}}", + "event.type": [ + "protocol" + ], + "fileset.name": "eve", + "input.type": "log", + "log.offset": 7401, + "network.community_id": "1:0dSnqQKCiJXvy6HxZlV+50/b68k=", + "network.protocol": "tls", + "network.transport": "tcp", + "related.ip": [ + "192.168.50.1" + ], + "service.type": "suricata", + "source.address": "192.168.50.1", + "source.ip": "192.168.50.1", + "source.port": 12509, + "suricata.eve.event_type": "tls", + "suricata.eve.flow_id": 289459143040794, + "suricata.eve.in_iface": "eno6", + "suricata.eve.tls.ja3.hash": "44d502d471cfdb99c59bdfb0f220e5a8", + "suricata.eve.tls.ja3.string": "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0", + "suricata.eve.tls.sni": "www.example.com", + "suricata.eve.tls.version": "UNDETERMINED", + "tags": [ + "suricata" + ], + "tls.client.ja3": "44d502d471cfdb99c59bdfb0f220e5a8", + "tls.client.server_name": "www.example.com" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/suricata/fields.go b/x-pack/filebeat/module/suricata/fields.go index 105704a1cc3..41b62db2d7a 100644 --- a/x-pack/filebeat/module/suricata/fields.go +++ b/x-pack/filebeat/module/suricata/fields.go @@ -19,5 +19,5 @@ func init() { // AssetSuricata returns asset data. // This is the base64 encoded gzipped contents of module/suricata. func AssetSuricata() string { - return "eJzsXEuP47gRvs+v0G1Pa2Q32UXQh9w2hwBJDgvkSpTJksQxX0NSdju/PpDsdssWKYsPdLAz06eGu/mxXqwXi/6xOeD5pXGD5RQ8fGoaz73Al+b3908YOmq58Vyrl+Zvn5qmaf6p2SCwabVtelBMcNU1vsfmt//81vzj93//qxG6c42xmg0UWbM/3/B2n5qm5SiYe5mQfmwUSLyjYPzxZ4MvTWf1YK6fBKgYf/4+YTWt1XKi4G2fiRShu6blAnfXf59vPN8cj3j7LLT3yv4zGvDVaOsv7C6EMVvwSMUDJcqTkYK7P78RdcDzSVt2+1sQA4whxmqviba8y8fx1AQXP0omxtMdEmkF3ElhMznvMMb1EYC91gJBPQO40UE8LSMF6KGMFHdWhQAe/KORJDIxE0ehZqzzZdy0PEsc78sFctXqSvbqevipTCAjQeNvERQQHJYiN+D7y9Ld+OtT9b0SziIbCK26D7Ag57XFGA0bVd+BiVnf1qPQw8+//FrGiWS/FIqC/zdb23drgwbOqTTxsBDX9tt6hs6TMTwF14cIvJA3ruMKxqi3u1se3MVZmrOJ04OluAF/imqJ4Ar9SdvDzltQbsMWFAyhKszCczmPEuDhsPmU/9m6sA7V4/LseLw0pFSHb2tgrHjIrRjsPWfMhCh1o96LgtWWalYogwzqb2FK6NOS+4QcESXwR/azo64HP2QlIndeLvn4zX3cszM4+eGAyrYKvPe+VlK9Kq61kDPSsLPojFYOdxeYe55itoqMW6SxVG+juY7bk8GhJdDhwtFuYeB98W6scLgC8WzTKXJQHTupKaRbbNHmi/3LgM7vJhA7w4nsONgYyasCsmKzZHrtfGamOm7DtAT+NEMTqDofK9622+pes/Nuf/boNmlKou91zDluVtUDytqGVCsfqd+T/BhXhLdAS3oAAiNpWLqjoeCx0/ZcmB/jES33MZQ1dUxNkd0CIOqljgXxuCsrqXinwA+2MKIDHYNRtqT04KmWT935jdic/OcGsujJZJuZ4PFwEIKLAT54fXJE60Li3KiUGa+69SewWIr4fiTsMRpKviKO5xlLrRKGglk5Z3nCO6BVKIgBesBAR2yLI1iAMasD7Y0sKN7mg93yCeN5NNZvAWDo4xlgnthDsSqNqEBWXUaSRElhuSxNbRLl4JayTmGs5QLJ1GSryp42qMiIXWZMDFsLjxcMhYKHVzKCkp4Xn0Bujn+JQoTJixM4q/e5RD0EqdtG30y9FjqJqgqWRXAO5V4E+rFb0WaSW7ZTv0vu+XEV+lT1OHg61pu5XmTm9lkFZ1buEVGi7YjUDAkqj6Grg2xAHUql0tCWF56pCJzKtaOTABL3XNtAnIFA0PjwABYSaVF+63BgmrTAw6c1SUROkYtZT1laqdaUQ+sJAw+TLQowE5mlRKIbE+/SWHiRWyktZxW6986Xk9JWgqgipsB1+ocZ+5y5IwjOCO2RHtwgi5U/BbmKVnq1zDo6FNz5GsoLTQ8kCspbBEkYGt8Ti0D7Yu9wywrOpIppzPC6YgW8ORjG25YEr8DS8JQmwdwlrRqgmlVuZ8CxI+bgSfByPY3F+7ZxHkZ5gmAhT8gzT25KaTgKUOQLV18KcX5QgxA/FIIIP6JEQfKLjUG5wVwG8sL9+K2UzkR/8MRrTZyEIMlZsaJQfl1xlrdaJleoFzea9aFGvf8r4YpUoIabtYOab5Rv+QE3Kx3VZH+g///lIVtcVjUVxFX3wI1+rzTPoIFGY2KeCa+1QtpUQZcm9dKI0oPnVhz4H61uRutMsZmg79EqLM1qR89cy6Eh4l//9PNPsLyET8rxAoVoeZ+ddELvodSGrljhqdaP79wLfSKyq5sOW31yZD+45SV6Gn8jcY5c27FVsJSugzZx6A7cmOICjgrtkBFjB1WMpfBUB+giLYtSH4ux9mczlpSVWJwGhOuxeDUHwlV56XxBnHoq5WX9CCXhVWCpk7eViUJpAtMxKT4HjCECzpVr8NXGQMFlUDDtSL4HCuZjqSjB++MMYqauNAln06lgTLmKSOHORSqSkzXlVIcm6fJrqLUxphwULoOdvWSFUbSG1tL+BayWAeyzUW4H//W7N/nGHUCd0/b1HZO4NcfPx/qYnNJ+j218fojNi6UIBnduQMti86hbhx0VLwOINczSqLhcpxKLbpClLxdbrjq0xvLo2Or2QVAeqH+TMJT20Ppo7rdF0W7Yfy5+3vEZ/pzWI2DYwiA8mQx5TA5EoGB4HgGct1wth9Hi9C8helj1S9sF8A3yP69BLkPIkVHfTSiBmiPXA073fsTr1fHytZn6+fOwTW9PnIfAPOvzja7PHO5WR7YwB1+Np8dx58iW0JWMDVd4a/6myNWp+TWmr89sN+nQIrjSWHNVUim9G/WDKhbNtrj+aQY7Lx4uTn7w4D5/oL14ohdOjQKvdbemVTPf9Pj1Hwm+KZAD5/omS40nC4ElmpkELkhr9XLYKAmmR5FFyFK4+GqQLo0pyf0vvyvmuYz/FwAA//8GEN89" + return "eJzsXEuP47gRvs+v0G1Pa2Q32UXQh9w2hwBJDgvkSpTJksQxX0NSdju/PpDsdssWKYsPdLAz06eGu/mxXqwXi/6xOeD5pXGD5RQ8fGoaz73Al+b3908YOmq58Vyrl+Zvn5qmaf6p2SCwabVtelBMcNU1vsfmt//81vzj93//qxG6c42xmg0UWbM/3/B2n5qm5SiYe5mQfmwUSLyjYPzxZ4MvTWf1YK6fBKgYf/4+YTWt1XKi4G2fiRShu6blAnfXf59vPN8cj3j7LLT3yv4zGvDVaOsv7C6EMVvwSMUDJcqTkYK7P78RdcDzSVt2+1sQA4whxmqviba8y8fx1AQXP0omxtMdEmkF3ElhMznvMMb1EYC91gJBPQO40UE8LSMF6KGMFHdWhQAe/KORJDIxE0ehZqzzZdy0PEsc78sFctXqSvbqevipTCAjQeNvERQQHJYiN+D7y9Ld+OtT9b0SziIbCK26D7Ag57XFGA0bVd+BiVnf1qPQw8+//FrGiWS/FIqC/zdb23drgwbOqTTxsBDX9tt6hs6TMTwF14cIvJA3ruMKxqi3u1se3MVZmrOJ04OluAF/imqJ4Ar9SdvDzltQbsMWFAyhKszCczmPEuDhsPmU/9m6sA7V4/LseLw0pFSHb2tgrHjIrRjsPWfMhCh1o96LgtWWalYogwzqb2FK6NOS+4QcESXwR/azo64HP2QlIndeLvn4zX3cszM4+eGAyrYKvPe+VlK9Kq61kDPSsLPojFYOdxeYe55itoqMW6SxVG+juY7bk8GhJdDhwtFuYeB98W6scLgC8WzTKXJQHTupKaRbbNHmi/3LgM7vJhA7w4nsONgYyasCsmKzZHrtfGamOm7DtAT+NEMTqDofK9622+pes/Nuf/boNmlKou91zDluVtUDytqGVCsfqd+T/BhXhLdAS3oAAiNpWLqjoeCx0/ZcmB/jES33MZQ1dUxNkd0CIOqljgXxuCsrqXinwA+2MKIDHYNRtqT04KmWT935jdic/OcGsujJZJuZ4PFwEIKLAT54fXJE60Li3KiUGa+69SewWIr4fiTsMRpKviKO5xlLrRKGglk5Z3nCO6BVKIgBesBAR2yLI1iAMasD7Y0sKN7mg93yCeN5NNZvAWDo4xlgnthDsSqNqEBWXUaSRElhuSxNbRLl4JayTmGs5QLJ1GSryp42qMiIXWZMDFsLjxcMhYKHVzKCkp4Xn0Bujn+JQoTJixM4q/e5RD0EqdtG30y9FjqJqgqWRXAO5V4E+rFb0WaSW7ZTv0vu+XEV+lT1OHg61pu5XmTm9lkFZ1buEVGi7YjUDAkqj6Grg2xAHUql0tCWF56pCJzKtaOTABL3XNtAnIFA0PjwABYSaVF+63BgmrTAw6c1SUROkYtZT1laqdaUQ+sJAw+TLQowE5mlRKIbE+/SWHiRWyktZxW6986Xk9JWgqgipsB1+ocZ+5y5IwjOCO2RHtwgi5U/BbmKVnq1zDo6FNz5GsoLTQ8kCspbBEkYGt8Ti0D7Yu9wywrOpIppzPC6YgW8ORjG25YEr8DS8JQmwdwlrRqgmlVuZ8CxI+bgSfByPY3F+7ZxHkZ5gmAhT8gzT25KaTgKUOQLV18KcX5QgxA/FIIIP6JEQfKLjUG5wVwG8sL9+K2UzkR/8MRrTZyEIMlZsaJQfl1xlrdaJleoFzea9aFGvf8r4YpUoIabtYOab5Rv+QE3Kx3VZH+g///lIVtcVjUVxFX3wI1+rzTPoIFGY2KeCa+1QtpUQZcm9dKI0oPnVhz4H61uRutMsZmg79EqLM1qR89cy6Eh4l//9PNPsLyET8rxAoVoeZ+ddELvodSGrljhqdaP79wLfSKyq5sOW31yZD+45SV6Gn8jcY5c27FVsJSugzZx6A7cmOICjgrtkBFjB1WMpfBUB+giLYtSH4ux9mczlpSVWJwGhOuxeDUHwlV56XxBnHoq5WX9CCXhVWCpk7eViUJpAtMxKT4HjCECzpVr8NXGQMFlUDDtSL4HCuZjqSjB++MMYqauNAln06lgTLmKSOHORSqSkzXlVIcm6fJrqLUxphwULoOdvWSFUbSG1tL+BayWAeyzUW4H//W7N/nGHUCd0/b1HZO4NcfPx/qYnNJ+j218fojNi6UIBnduQMti86hbhx0VLwOINczSqLhcpxKLbpClLxdbrjq0xvLo2Or2QVAeqH+TMJT20Ppo7rdF0W7Yfy5+3vEZ/pzWI2DYwiA8mQx5TA5EoGB4HgGct1wth9Hi9C8helj1S9sF8A3yP69BLkPIkVHfTSiBmiPXA073fsTr1fHytZn6+fOwTW9PnIfAPOvzja7PHO5WR7YwB1+Np8dx58iW0JWMDVd4a/6myNWp+TWmr89sN+nQIrjSWHNVUim9W/Uj0Pq8iLY4u8Gj9/yJ9eKRXTi5Cby33ZoYzbzL4xd4JHiXQBab610sNZ4sBJZoKBK4IK3Vy3GhJJgeRRYhS+Hiq0G6NKYkB778tpfnMv5fAAAA//9J19Mm" } From 4041a4f2966e846b7c483221e10903be708bc387 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 2 Feb 2021 17:07:04 -0500 Subject: [PATCH 2/2] Generate docs --- filebeat/docs/fields.asciidoc | 7 ------- 1 file changed, 7 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 34bdd395d90..9a164a9be31 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -142044,13 +142044,6 @@ alias to: source.packets -- -*`suricata.eve.flow.end`*:: -+ --- -type: date - --- - *`suricata.eve.flow.alerted`*:: + --