Cherry-pick #23236 to 7.11: [Filebeat] Improve Suricata error handling

CHANGELOG.next.asciidoc

@@ -255,6 +255,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273]
 - Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534]
 - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
+- Fix various processing errors in the Suricata module. {pull}23236[23236]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -142044,13 +142044,6 @@ alias to: source.packets
 
 --
 
-*`suricata.eve.flow.end`*::
-+
---
-type: date
-
---
-
 *`suricata.eve.flow.alerted`*::
 +
 --

x-pack/filebeat/module/suricata/eve/_meta/fields.yml

@@ -714,9 +714,6 @@
         type: alias
         path: source.packets
 
-      - name: end
-        type: date
-
       - name: alerted
         type: boolean
 

x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml

@@ -35,5 +35,9 @@ processors:
         }
 on_failure:
   - append:
-        field: error.message
-        value: "{{ _ingest.on_failure_message }}"
+      field: error.message
+      value: >-
+        error in DNS v1 pipeline:
+        error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}}
+        with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}}
+        {{ _ingest.on_failure_message }}

x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml

@@ -38,5 +38,9 @@ processors:
         }
 on_failure:
   - append:
-        field: error.message
-        value: "{{ _ingest.on_failure_message }}"
+      field: error.message
+      value: >-
+        error in DNS v2 pipeline:
+        error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}}
+        with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}}
+        {{ _ingest.on_failure_message }}

x-pack/filebeat/module/suricata/eve/ingest/dns.yml

@@ -89,5 +89,9 @@ processors:
       ignore_missing: true
 on_failure:
   - append:
-        field: error.message
-        value: "{{ _ingest.on_failure_message }}"
+      field: error.message
+      value: >-
+        error in DNS pipeline:
+        error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}}
+        with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}}
+        {{ _ingest.on_failure_message }}

x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml

@@ -11,6 +11,7 @@ processors:
       field: suricata.eve.event_type
       ignore_missing: true
   - script:
+      tag: suricata_ecs_categorize
       lang: painless
       ignore_failure: true
       params:
@@ -120,11 +121,17 @@ processors:
       ignore_empty_value: true
   ## HTTP
   - set:
-      if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400
+      if: >-
+        ctx?.suricata?.eve?.event_type == "http" &&
+        ctx?.suricata?.eve?.http?.status != null &&
+        ctx?.suricata?.eve?.http?.status < 400
       field: event.outcome
       value: success
   - set:
-      if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status >= 400
+      if: >-
+        ctx?.suricata?.eve?.event_type == "http" &&
+        ctx?.suricata?.eve?.http?.status != null &&
+        ctx?.suricata?.eve?.http?.status >= 400
       field: event.outcome
       value: failure
   ## DNS
@@ -179,7 +186,7 @@ processors:
         }
       ignore_failure: true
   - set:
-      if: "ctx?.network?.protocol == 'http'"
+      if: ctx?.network?.protocol == 'http'
       field: url.domain
       value: '{{destination.domain}}'
       ignore_empty_value: true
@@ -196,6 +203,10 @@ processors:
       field: suricata.eve.http.url
       target_field: url.original
       ignore_missing: true
+  - rename:
+      field: suricata.eve.http.http_port
+      target_field: url.port
+      ignore_missing: true
   - rename:
       field: suricata.eve.http.http_refer
       target_field: http.request.referrer
@@ -233,13 +244,13 @@ processors:
       value: "{{suricata.eve.alert.signature}}"
       ignore_empty_value: true
   - set:
+      if:  ctx?.suricata?.eve?.alert?.action == 'blocked'
       field: suricata.eve.alert.action
       value: denied
-      if:  "ctx?.suricata?.eve?.alert?.action == 'blocked'"
   - append:
+      if: ctx?.suricata?.eve?.alert?.action != null
       field: event.type
       value: "{{suricata.eve.alert.action}}"
-      if: "ctx?.suricata?.eve?.alert?.action != null"
   - remove:
       field: suricata.eve.alert.action
       ignore_failure: true
@@ -264,6 +275,7 @@ processors:
       target_field: source.bytes
       ignore_missing: true
   - script:
+      tag: suricata_network_bytes_packets
       lang: painless
       source: >
         long getOrZero(def map, def key) {
@@ -299,6 +311,7 @@ processors:
         - ISO8601
       ignore_failure: true
   - script:
+      tag: suricata_event_duration
       lang: painless
       source: >
         Instant ins(def d) {
@@ -324,12 +337,12 @@ processors:
       field: suricata.eve.http.http_user_agent
       ignore_missing: true
   - geoip:
-      if: ctx.source?.geo == null
+      if: ctx?.source?.geo == null
       field: source.ip
       target_field: source.geo
       ignore_missing: true
   - geoip:
-      if: ctx.destination?.geo == null
+      if: ctx?.destination?.geo == null
       field: destination.ip
       target_field: destination.geo
       ignore_missing: true
@@ -366,9 +379,9 @@ processors:
       target_field: destination.as.organization.name
       ignore_missing: true
   - append:
+      if: ctx?.url?.domain != null && ctx.url.domain != ''
       field: related.hosts
       value: '{{url.domain}}'
-      if: ctx.url?.domain != null && ctx.url?.domain != ''
       allow_duplicates: false
   - append:
       if: ctx?.source?.ip != null
@@ -389,6 +402,35 @@ processors:
         - suricata.eve.http.http_user_agent
       ignore_missing: true
 on_failure:
-  - set:
+  - append:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: >-
+        error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}}
+        with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}}
+        {{ _ingest.on_failure_message }}
+  - remove:
+      field:
+        # Remove all alias fields to ensure indexing succeeds.
+        - suricata.eve.alert.action
+        - suricata.eve.alert.severity
+        - suricata.eve.app_proto
+        - suricata.eve.dest_ip
+        - suricata.eve.dest_port
+        - suricata.eve.fileinfo.filename
+        - suricata.eve.fileinfo.size
+        - suricata.eve.flow.bytes_toclient
+        - suricata.eve.flow.bytes_toserver
+        - suricata.eve.flow.pkts_toclient
+        - suricata.eve.flow.pkts_toserver
+        - suricata.eve.flow.start
+        - suricata.eve.http.hostname
+        - suricata.eve.http.http_method
+        - suricata.eve.http.http_refer
+        - suricata.eve.http.http_user_agent
+        - suricata.eve.http.length
+        - suricata.eve.http.status
+        - suricata.eve.http.url
+        - suricata.eve.proto
+        - suricata.eve.src_ip
+        - suricata.eve.src_port
+      ignore_missing: true

x-pack/filebeat/module/suricata/eve/ingest/tls.yml

@@ -3,9 +3,11 @@ description: Pipeline for Suricata TLS Events
 
 processors:
   - dissect:
+      if: ctx?.suricata?.eve?.tls?.version != "UNDETERMINED"
       field: suricata.eve.tls.version
       pattern: '%{tls.version_protocol} %{tls.version}'
       ignore_missing: true
+      ignore_failure: true
   - lowercase:
       field: tls.version_protocol
       ignore_missing: true
@@ -35,6 +37,7 @@ processors:
       value_split: '='
       target_field: suricata.eve.tls.kv_subject
       ignore_missing: true
+      ignore_failure: true
   - rename:
       field: suricata.eve.tls.kv_subject.C
       target_field: tls.server.x509.subject.country
@@ -75,6 +78,7 @@ processors:
       value_split: '='
       target_field: suricata.eve.tls.kv_issuerdn
       ignore_missing: true
+      ignore_failure: true
   - rename:
       field: suricata.eve.tls.kv_issuerdn.C
       target_field: tls.server.x509.issuer.country
@@ -184,5 +188,9 @@ processors:
       ignore_missing: true
 on_failure:
   - append:
-        field: error.message
-        value: "{{ _ingest.on_failure_message }}"
+      field: error.message
+      value: >-
+        error in TLS pipeline:
+        error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}}
+        with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}}
+        {{ _ingest.on_failure_message }}

x-pack/filebeat/module/suricata/eve/test/eve-small.log

@@ -6,3 +6,7 @@
 {"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}}
 {"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}}
 {"timestamp":"2018-07-05T15:51:54.001329-0400","flow_id":1828507008887644,"event_type":"flow","src_ip":"fe80:0000:0000:0000:fada:0cff:fedc:87f1","src_port":546,"dest_ip":"ff02:0000:0000:0000:0000:0000:0001:0002","dest_port":547,"proto":"UDP","app_proto":"failed","flow":{"pkts_toserver":1,"pkts_toclient":0,"bytes_toserver":110,"bytes_toclient":0,"start":"2018-07-05T15:51:23.453468-0400","end":"2018-07-05T15:51:23.453468-0400","age":0,"state":"new","reason":"timeout","alerted":false}}
+{"timestamp":"2020-12-09T16:02:43.000505+0000","flow_id":913701662641234,"in_iface":"eno6","event_type":"http","src_ip":"192.168.50.1","src_port":57134,"dest_ip":"192.168.50.1","dest_port":8080,"proto":"TCP","tx_id":0,"http":{"hostname":"ctldl.windowsupdate.com","url":"http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111","http_user_agent":"Microsoft-CryptoAPI/10.0","http_method":"GET","protocol":"HTTP/1.1","length":0}}
+{"timestamp":"2020-12-09T16:02:58.005716+0000","flow_id":1298574590709840,"in_iface":"eno6","event_type":"tls","src_ip":"192.168.50.1","src_port":60614,"dest_ip":"192.168.50.1","dest_port":443,"proto":"TCP","tls":{"subject":"C=US, ST=New York, L=New York City, O=Acme U.S.A., INC., CN=update.acme.com","issuerdn":"C=US, O=DigiCert Inc, OU=www.digicert.com, CN=GeoTrust RSA CA 2018","serial":"0D:CE:DC:BC:AF:92:56:B4:C5:41:40:71:26:5B:1D:53","fingerprint":"18:3c:11:45:46:e9:26:c7:87:64:0f:ed:47:86:1b:31:bf:0f:84:25","version":"TLS 1.2","notbefore":"2020-11-24T00:00:00","notafter":"2021-12-25T23:59:59","ja3":{},"ja3s":{"hash":"adc06261ef82c2e4688b3cf08c1b2f24","string":"771,159,65281"}}}
+{"timestamp":"2020-12-09T16:03:00.179037+0000","flow_id":1097935193623328,"in_iface":"eno6","event_type":"http","src_ip":"192.168.50.1","src_port":50898,"dest_ip":"192.168.50.1","dest_port":8081,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.50.1","http_port":8081,"url":"/uuid","http_user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:84.0) Gecko/20100101 Firefox/84.0","http_method":"POST","protocol":"HTTP/1.1","length":0}}
+{"timestamp":"2020-12-09T16:03:50.083307+0000","flow_id":289459143040794,"in_iface":"eno6","event_type":"tls","src_ip":"192.168.50.1","src_port":12509,"dest_ip":"192.168.50.1","dest_port":443,"proto":"TCP","tls":{"sni":"www.example.com","version":"UNDETERMINED","ja3":{"hash":"44d502d471cfdb99c59bdfb0f220e5a8","string":"771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-41,29-23-24,0"},"ja3s":{}}}