[filebeat][threatintel] MISP splitting fix for empty responses (#38917)

Two fixes related to empty server responses:

- Set `response.split.ignore_empty_value` to `true` to avoid indexing
  `{response:[]}`, which is the MISP server's empty response body.
  According to the `response.split` [documentation][1], "If the split
  target is empty the parent document will be kept. If documents with
  empty splits should be dropped, the `ignore_empty_value` option should
  be set to `true`."

- Use the null-safe operator for a chained method invocation following a
  null-safe field access. All other null-safe operator usages were also
  reviewed and corrected where necessary.

[1]: https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split

---------

Co-authored-by: Dan Kortschak <dan.kortschak@elastic.co>
(cherry picked from commit 692658c)

CHANGELOG.next.asciidoc

@@ -75,6 +75,17 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Filebeat Cisco module with missing escape character {issue}36325[36325] {pull}36326[36326]
 - Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496]
 - Fix indexing failures by re-enabling event normalisation in netflow input. {issue}38703[38703] {pull}38780[38780]
+- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416]
+- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556]
+- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488]
+- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917]
+
+*Heartbeat*
+
+- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702]
+- Fix setuid root when running under cgroups v2. {pull}37794[37794]
+- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981]
+- Reset prctl dumpable flag after cap drop. {pull}38269[38269]
 
 *Heartbeat*
 

x-pack/filebeat/module/threatintel/misp/config/config.yml

@@ -49,6 +49,7 @@ request.transforms:
 
 response.split:
   target: body.response
+  ignore_empty_value: true
   split:
     target: body.Event.Attribute
     ignore_empty_value: true

x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml

@@ -63,12 +63,12 @@ processors:
       field:
         - misp.Attribute
       ignore_missing: true
-      if: ctx.misp?.Attribute.size() == 0
+      if: ctx.misp?.Attribute?.size() == 0
   - remove:
       field:
         - misp.Object
       ignore_missing: true
-      if: ctx.misp?.Object.size() == 0
+      if: ctx.misp?.Object?.size() == 0
   - date:
       field: misp.timestamp
       formats:
@@ -144,12 +144,12 @@ processors:
   - set:
       field: threat.indicator.type
       value: file
-      if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
+      if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true"
   - rename:
       field: misp.attribute.value
       target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
       ignore_missing: true
-      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
+      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.file.name
@@ -160,23 +160,23 @@ processors:
       patterns:
         - "%{WORD}\\|%{WORD:_tmp.hashtype}"
       ignore_missing: true
-      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
+      if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
   - grok:
       field: misp.attribute.value
       patterns:
         - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
       ignore_missing: true
-      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
+      if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
   - set:
       field: threat.indicator.file.hash.{{_tmp.hashtype}}
       value: "{{_tmp.hashvalue}}"
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
+      if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null"
 
   ## URL/URI indicator operations
   - set:
       field: threat.indicator.type
       value: url
-      if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
+      if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
   - uri_parts:
       field: misp.attribute.value
       target_field: threat.indicator.url
@@ -193,7 +193,7 @@ processors:
   - set:
       field: threat.indicator.type
       value: windows-registry-key
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
+      if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.registry.key
@@ -210,7 +210,7 @@ processors:
   - set:
       field: threat.indicator.type
       value: autonomous-system
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
+      if: "ctx.misp?.attribute?.type == 'AS'"
   - convert:
       field: misp.attribute.value
       type: long
@@ -222,11 +222,11 @@ processors:
   - set:
       field: threat.indicator.type
       value: domain-name
-      if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
+      if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true"
   - set:
       field: threat.indicator.type
       value: ipv4-addr
-      if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
+      if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.url.domain
@@ -257,11 +257,11 @@ processors:
   - set:
       field: threat.indicator.type
       value: email-addr
-      if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
+      if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
   - set:
       field: threat.indicator.type
       value: email-message
-      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
+      if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.email.address
@@ -274,13 +274,13 @@ processors:
   - append:
       field: user.roles
       value: "reporting_user"
-      if: ctx?.user?.email != null
+      if: ctx.user?.email != null
 
   ## MAC Address indicator operations
   - set:
       field: threat.indicator.type
       value: mac-addr
-      if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
+      if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
   - rename:
       field: misp.attribute.value
       target_field: threat.indicator.mac
@@ -337,12 +337,12 @@ processors:
   ######################
   - remove:
       field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
       ignore_failure: true
       ignore_missing: true
   - script:
       lang: painless
-      if: ctx?.misp != null
+      if: ctx.misp != null
       source: |
         void handleMap(Map map) {
           for (def x : map.values()) {