Cherry-pick #24661 to 7.x: [Filebeat][Cisco AMP] Pipeline fixes for r…

…elated mac/ip and mitre tactics fields. (#24701)

* [Filebeat][Cisco AMP] Pipeline fixes for related mac/ip and mitre tactics fields. (#24661)

* changing foreach processors to remove null value fields from related fields

* splitting up test files and fixing document_id generation, and some field typos

* removing lines from painless script that isn't really necessary

* mage fmt update to format python code

* reverting CI changes for now, will be moved to separate PR

* reduce test logs and add fixes to pipeline

* update changelog

(cherry picked from commit 45cd394)

* Update CHANGELOG.next.asciidoc

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>

CHANGELOG.next.asciidoc

@@ -499,6 +499,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline {issue}23118[23118] {pull}23929[23929]
 - Update zoom module to ECS 1.8. {pull}23904[23904] {issue}23118[23118]
 - Support X-Forwarder-For in IIS logs. {pull}19142[192142]
+- Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661]
 - Added NTP fileset to Zeek module {pull}24224[24224]
 
 *Heartbeat*

filebeat/docs/fields.asciidoc

@@ -21078,7 +21078,7 @@ type: keyword
 
 --
 
-*`cisco.amp.file.archived_file.identify.sha256`*::
+*`cisco.amp.file.archived_file.identity.sha256`*::
 +
 --
 SHA256 hash of the archived file related to the malicious event.
@@ -21288,12 +21288,52 @@ type: flattened
 
 --
 
+*`cisco.amp.mitre_tactics`*::
++
+--
+Array of all related mitre tactic ID's
+
+
+type: keyword
+
+--
+
 *`cisco.amp.techniques`*::
 +
 --
 List of all MITRE techniques related to the incident found.
 
 
+type: flattened
+
+--
+
+*`cisco.amp.mitre_techniques`*::
++
+--
+Array of all related mitre technique ID's
+
+
+type: keyword
+
+--
+
+*`cisco.amp.command_line.arguments`*::
++
+--
+The CLI arguments related to the Cloud Threat IOC reported by Cisco.
+
+
+type: keyword
+
+--
+
+*`cisco.amp.bp_data`*::
++
+--
+Endpoint isolation information
+
+
 type: flattened
 
 --

x-pack/filebeat/module/cisco/amp/_meta/fields.yml

@@ -160,7 +160,7 @@
       description: >
         SHA1 hash of the archived file related to the malicious event.
 
-    - name: file.archived_file.identify.sha256
+    - name: file.archived_file.identity.sha256
       type: keyword
       description: >
         SHA256 hash of the archived file related to the malicious event.
@@ -265,7 +265,27 @@
       description: >
         List of all MITRE tactics related to the incident found.
 
+    - name: mitre_tactics
+      type: keyword
+      description: >
+        Array of all related mitre tactic ID's
+
     - name: techniques
       type: flattened
       description: >
-        List of all MITRE techniques related to the incident found.
+        List of all MITRE techniques related to the incident found.
+
+    - name: mitre_techniques
+      type: keyword
+      description: >
+        Array of all related mitre technique ID's
+
+    - name: command_line.arguments
+      type: keyword
+      description: >
+        The CLI arguments related to the Cloud Threat IOC reported by Cisco.
+
+    - name: bp_data
+      type: flattened
+      description: >
+        Endpoint isolation information

x-pack/filebeat/module/cisco/amp/config/config.yml

@@ -62,14 +62,14 @@ processors:
       fields: [message]
       target: json
   - if:
-      has_fields: ["json.data.id"]
+      has_fields: ["json.data.detection_id"]
     then:
       - fingerprint:
-          fields: ["json.data.id"]
+          fields: ["json.data.detection_id"]
           target_field: "@metadata._id"
     else:
       - fingerprint:
-          fields: ["json.data.timestamp", "json.data.event_type_id", "json.data.connector_guid"]
+          fields: ["json.data.timestamp", "json.data.timestamp_nanoseconds", "json.data.event_type_id", "json.data.connector_guid"]
           target_field: "@metadata._id"
   - add_fields:
       target: ''

x-pack/filebeat/module/cisco/amp/ingest/pipeline.yml

@@ -76,6 +76,15 @@ processors:
     ignore_failure: true
     if: ctx?.cisco?.amp?.start_timestamp != null
 
+- rename:
+    field: cisco.amp.techniques
+    target_field: cisco.amp.mitre_techniques
+    if: "ctx?.cisco?.amp?.techniques != null && ctx?.cisco?.amp?.techniques.length > 0 && ctx?.cisco?.amp?.techniques[0] instanceof String"
+- rename:
+    field: cisco.amp.tactics
+    target_field: cisco.amp.mitre_tactics
+    if: "ctx?.cisco?.amp?.tactics != null && ctx?.cisco?.amp?.tactics.length > 0 && ctx?.cisco?.amp?.tactics[0] instanceof String"
+
 ######################
 ## ECS Host Mapping ##
 ######################
@@ -189,6 +198,10 @@ processors:
     field: cisco.amp.file.parent.process_id
     target_field: process.pid
     ignore_missing: true
+- rename:
+    field: cisco.amp.network_info.parent.process_id
+    target_field: process.pid
+    ignore_missing: true
 - rename:
     field: cisco.amp.file.parent.file_name
     target_field: process.name
@@ -205,10 +218,9 @@ processors:
     field: cisco.amp.file.parent.identity.md5
     target_field: process.hash.md5
     ignore_missing: true
-
 - rename:
-    field: cisco.amp.network_info.parent.process_id
-    target_field: process.pid
+    field: cisco.amp.file.parent.identity.md5
+    target_field: process.hash.md5
     ignore_missing: true
 - rename:
     field: cisco.amp.network_info.parent.file_name
@@ -300,21 +312,39 @@ processors:
     value: "{{ cisco.amp.computer.external_ip }}"
     if: ctx?.cisco?.amp?.computer?.external_ip != null
     allow_duplicates: false
-- foreach:
-    field: cisco.amp.computer.network_addresses
-    processor:
-      append:
-        field: related.ip
-        value: "{{ _ingest._value.ip }}"
-        allow_duplicates: false
+- script:
+    lang: painless
+    source: |
+        if (ctx?.related == null) {
+            ctx.related = new HashMap();
+        }
+        if (ctx?.related?.ip == null) {
+            ctx.related.ip = new ArrayList();
+        }
+        for (addr in ctx?.cisco?.amp?.computer?.network_addresses) {
+            if (addr.ip != null && !addr.ip.isEmpty()) {
+                if (!ctx?.related?.ip.contains(addr.ip)) {
+                    ctx?.related?.ip.add(addr.ip);  
+                }
+            }
+        }
     if: ctx?.cisco?.amp?.computer?.network_addresses != null
-- foreach:
-    field: cisco.amp.computer.network_addresses
-    processor:
-      append:
-        field: cisco.amp.related.mac
-        value: "{{ _ingest._value.mac }}"
-        allow_duplicates: false
+- script:
+    lang: painless
+    source: |
+        if (ctx?.cisco?.amp?.related == null) {
+            ctx.cisco.amp.related = new HashMap();
+        }
+        if (ctx?.cisco?.amp?.related?.mac == null) {
+            ctx.cisco.amp.related.mac = new ArrayList();
+        }
+        for (addr in ctx?.cisco?.amp?.computer?.network_addresses) {
+            if (addr.mac != null && !addr.mac.isEmpty()) {
+                if (!ctx?.cisco?.amp?.related?.mac.contains(addr.mac)) {
+                    ctx?.cisco?.amp?.related?.mac.add(addr.mac);  
+                }
+            }
+        }
     if: ctx?.cisco?.amp?.computer?.network_addresses != null
 - foreach:
     field: cisco.amp.vulnerabilities