[Filebeat] Update Fortinet Ingest Pipeline (#24816)

* 22136: Update Fortinet Ingest Pipeline * Update Pipelines * Additional updates * Set virus/ips subtypes to event.kind: alert * update fields * Consolidate processors to script * Update event.outcome logic * replace hashmap * update event.outcome * cleanup * Added Changes for #25254 * regenerate data * update changelog * remove extra items in changelog
elastic · Jun 29, 2021 · 890e473 · 890e473
1 parent 2a56cd7
commit 890e473
Show file tree

Hide file tree

Showing 20 changed files with 4,630 additions and 2,435 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
+- Change source field for `event.action` in `fortinet.firewall` module to `fortinet.firewall.action` instead of `fortinet.firewall.eventtype`. {pull}24816[24816]
 
 *Heartbeat*
 - Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
@@ -839,6 +840,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
 - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
+- Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -62883,6 +62883,16 @@ type: keyword
 ESP Transform
 
 
+type: keyword
+
+--
+
+*`fortinet.firewall.eventtype`*::
++
+--
+UTM Event Type
+
+
 type: keyword
 
 --
@@ -65363,6 +65373,16 @@ type: integer
 Security action performed by UTM
 
 
+type: keyword
+
+--
+
+*`fortinet.firewall.utmref`*::
++
+--
+Reference to UTM
+
+
 type: keyword
 
 --

diff --git a/x-pack/filebeat/module/fortinet/fields.go b/x-pack/filebeat/module/fortinet/fields.go
diff --git a/x-pack/filebeat/module/fortinet/firewall/_meta/fields.yml b/x-pack/filebeat/module/fortinet/firewall/_meta/fields.yml
@@ -766,6 +766,11 @@
           description: >
             ESP Transform
 
+        - name: eventtype
+          type: keyword
+          description: >
+            UTM Event Type
+
         - name: exch
           type: keyword
           description: >
@@ -2006,6 +2011,11 @@
           description: >
             Security action performed by UTM
 
+        - name: utmref
+          type: keyword
+          description: >
+            Reference to UTM
+
         - name: vap
           type: keyword
           description: >

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml
@@ -3,6 +3,10 @@ processors:
 - set:
     field: event.kind
     value: event
+- set:
+    field: event.action
+    value: "{{fortinet.firewall.action}}"
+    ignore_empty_value: true
 - set:
     field: event.outcome
     value: failure
@@ -16,24 +20,29 @@ processors:
     value:
     - user
     - start
+    allow_duplicates: false
     if: "['FSSO-logon', 'auth-logon'].contains(ctx.fortinet?.firewall?.action)"
 - append:
     field: event.type
     value:
     - user
     - end
+    allow_duplicates: false
     if: "['FSSO-logoff', 'auth-logout'].contains(ctx.fortinet?.firewall?.action)"
 - append:
     field: event.type
     value: connection
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'vpn'"
 - append:
     field: event.category
     value: network
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'vpn'"
 - append:
     field: event.type
     value: info
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.action == 'perf-stats'"
 - append:
     field: event.category
@@ -42,16 +51,19 @@ processors:
 - append:
     field: event.type
     value: info
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'update'"
 - append:
     field: event.category
     value:
     - host
     - malware
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'update'"
 - append:
     field: event.category
     value: authentication
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'user'"
 - rename:
     field: fortinet.firewall.dstip
@@ -95,10 +107,6 @@ processors:
     target_field: destination.domain
     ignore_missing: true
     if: "ctx.destination?.address == null"
-- rename:
-    field: fortinet.firewall.group
-    target_field: source.user.group.name
-    ignore_missing: true
 - convert:
     field: fortinet.firewall.sentbyte
     target_field: source.bytes
@@ -144,10 +152,6 @@ processors:
     field: fortinet.firewall.saddr
     target_field: source.address
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.agent
-    target_field: user_agent.original
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.file
     target_field: file.name
@@ -167,18 +171,10 @@ processors:
     target_field: event.code
     ignore_missing: true
     if: "ctx.event?.code == null"
-- rename:
-    field: fortinet.firewall.msg
-    target_field: message
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.policyid
     target_field: rule.id
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.proto
-    target_field: network.iana_number
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.dir
     target_field: network.direction
@@ -207,21 +203,10 @@ processors:
         return
       }
       ctx.network.direction = k;
-- rename:
-    field: fortinet.firewall.service
-    target_field: network.protocol
-    ignore_missing: true
-- lowercase:
-    field: network.protocol
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.error_num
     target_field: error.code
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.hostname
-    target_field: url.domain
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.logdesc
     target_field: rule.description

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml
@@ -24,7 +24,7 @@ processors:
     source: |
       def fw = ctx?.fortinet?.firewall;
       if (fw != null) {
-        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A");
+        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A" || entry.getValue() == "undefined");
       }
 - set:
     field: observer.vendor
@@ -35,12 +35,6 @@ processors:
 - set:
     field: observer.type
     value: firewall
-- set:
-    field: event.module
-    value: fortinet
-- set:
-    field: event.dataset
-    value: fortinet.firewall
 - set:
     field: event.timezone
     value: "{{fortinet.firewall.tz}}"
@@ -198,18 +192,7 @@ processors:
       )
 - remove:
     field:
-      - _temp.time
-      - _temp
       - message
-      - syslog5424_sd
-      - syslog5424_pri
-      - fortinet.firewall.tz
-      - fortinet.firewall.date
-      - fortinet.firewall.devid
-      - fortinet.firewall.eventtime
-      - fortinet.firewall.time
-      - fortinet.firewall.duration
-      - host
     ignore_missing: true
 - pipeline:
     name: '{< IngestPipeline "event" >}'
@@ -220,6 +203,73 @@ processors:
 - pipeline:
     name: '{< IngestPipeline "utm" >}'
     if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
+- rename:
+    field: fortinet.firewall.reason
+    target_field: event.reason
+    ignore_missing: true
+- rename:
+    field: fortinet.firewall.msg
+    target_field: message
+    ignore_missing: true
+- rename:
+    field: fortinet.firewall.proto
+    target_field: network.iana_number
+    ignore_missing: true
+- script:
+    lang: painless
+    ignore_failure: true
+    if: ctx?.network?.iana_number != null
+    source: |
+      if (ctx?.network == null) {
+          ctx.network = new HashMap();
+      }
+      def iana_number = ctx.network.iana_number;
+      if (iana_number == '1') {
+        ctx.network.transport = 'icmp';
+      } else if (iana_number == '2') {
+        ctx.network.transport = 'igmp';
+      } else if (iana_number == '6') {
+        ctx.network.transport = 'tcp';
+      } else if (iana_number == '17') {
+        ctx.network.transport = 'udp';
+      } else if (iana_number == '58') {
+        ctx.network.transport = 'ipv6-icmp';
+      }
+- rename:
+    field: fortinet.firewall.group
+    target_field: source.user.group.name
+    ignore_missing: true
+- uri_parts:
+    field: fortinet.firewall.url
+    remove_if_successful: true
+    ignore_failure: true
+    if: "ctx.fortinet?.firewall?.url != null"
+- set:
+    field: url.domain
+    value: "{{fortinet.firewall.hostname}}"
+    ignore_empty_value: true
+    if: "ctx?.url?.domain == null"
+- rename:
+    field: fortinet.firewall.service
+    target_field: network.protocol
+    ignore_missing: true
+- lowercase:
+    field: network.protocol
+    ignore_missing: true
+- set:
+    field: network.type
+    value: ipv4
+    if: (ctx.source?.ip != null && ctx.source?.ip.contains('.')) || (ctx.destination?.ip != null && ctx.destination?.ip.contains('.'))
+- set:
+    field: network.type
+    value: ipv6
+    if: ctx.source?.ip != null && ctx.source?.ip.contains(':') || (ctx.destination?.ip != null && ctx.destination?.ip.contains(':'))
+- community_id:
+    ignore_missing: true
+    ignore_failure: true
+- user_agent:
+    field: fortinet.firewall.agent
+    ignore_missing: true
 - convert:
     field: fortinet.firewall.quotamax
     type: long
@@ -315,30 +365,37 @@ processors:
 - append:
     field: related.ip
     value: "{{source.ip}}"
+    allow_duplicates: false
     if: "ctx.source?.ip != null"
 - append:
     field: related.ip
     value: "{{destination.ip}}"
+    allow_duplicates: false
     if: "ctx.destination?.ip != null"
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 - append:
     field: related.user
     value: "{{destination.user.name}}"
+    allow_duplicates: false
     if: "ctx.destination?.user?.name != null"
 - append:
     field: related.hosts
     value: "{{destination.address}}"
+    allow_duplicates: false
     if: "ctx.destination?.address != null"
 - append:
     field: related.hosts
     value: "{{source.address}}"
+    allow_duplicates: false
     if: "ctx.source?.address != null"
 - append:
     field: related.hosts
     value: "{{dns.question.name}}"
+    allow_duplicates: false
     if: "ctx.dns?.question?.name != null"
 - script:
     lang: painless
@@ -354,6 +411,45 @@ processors:
           }
         }
       }
+- remove:
+    field:
+      - _temp
+      - syslog5424_sd
+      - syslog5424_pri
+      - fortinet.firewall.tz
+      - fortinet.firewall.date
+      - fortinet.firewall.devid
+      - fortinet.firewall.eventtime
+      - fortinet.firewall.time
+      - fortinet.firewall.duration
+      - host
+      - fortinet.firewall.hostname
+      - fortinet.firewall.agent
+    ignore_missing: true
+- script:
+    lang: painless
+    description: This script processor iterates over the whole document to remove fields with null values.
+    source: |
+      void handleMap(Map map) {
+        for (def x : map.values()) {
+          if (x instanceof Map) {
+              handleMap(x);
+          } else if (x instanceof List) {
+              handleList(x);
+          }
+        }
+        map.values().removeIf(v -> v == null);
+      }
+      void handleList(List list) {
+        for (def x : list) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+        }
+      }
+      handleMap(ctx);
 on_failure:
 - set:
     field: error.message