[Filebeat][Fortinet] Fixing kv split for when assign-ip is not an IP and for date checking when eventtime is missing

CHANGELOG.next.asciidoc

@@ -290,6 +290,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 - Fix syslog RFC 5424 parsing in the CheckPoint module. {pull}21854[21854]
 - Fix incorrect connection state mapping in zeek connection pipeline. {pull}22151[22151] {issue}22149[22149]
+- Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361]
 - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
 
 *Heartbeat*

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -11,10 +11,18 @@ processors:
     field: syslog5424_sd
     field_split: " (?=[a-z\\_\\-]+=)"
     value_split: "="
-    prefix: "fortinet.firewall."
+    prefix: "fortinet.tmp."
     ignore_missing: true
     ignore_failure: false
     trim_value: "\""
+- remove:
+    field: fortinet.tmp.assignip
+    if: "ctx.fortinet?.tmp?.assignip == 'N/A'"
+    ignore_missing: true
+- rename:
+    field: fortinet.tmp
+    target_field: fortinet.firewall
+    ignore_missing: true
 - set:
     field: observer.vendor
     value: Fortinet
@@ -65,41 +73,41 @@ processors:
     field: fortinet.firewall.eventtime
     pattern: "\\d{6}$"
     replacement: ""
-    if: "(ctx.fortinet?.firewall?.eventtime).length() > 18"
+    if: "ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 18"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
     timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
     timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
-    if: "ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    if: "ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
-- rename:
-    field: fortinet.firewall.devname
-    target_field: observer.name
-    ignore_missing: true
+    if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - script:
     lang: painless
     source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"
     if: "ctx.fortinet?.firewall?.duration != null"
+- rename:
+    field: fortinet.firewall.devname
+    target_field: observer.name
+    ignore_missing: true
 - rename:
     field: fortinet.firewall.devid
     target_field: observer.serial_number
@@ -126,9 +134,6 @@ processors:
     field: fortinet.firewall.level
     target_field: log.level
     ignore_missing: true
-- remove:
-    field: fortinet.firewall.assignip
-    if: "ctx.fortinet?.firewall?.assignip == 'N/A'"
 - remove:
     field: fortinet.firewall.dstip
     if: "ctx.fortinet?.firewall?.dstip == 'N/A'"

x-pack/filebeat/module/fortinet/firewall/test/fortinet.log

@@ -29,3 +29,4 @@
 <188>date=2020-04-23 time=12:14:39 devname="firewall3" devid="oldfwid" logid="0000000011" type="traffic" subtype="forward" level="warning" vd="root" eventtime=1587230079841464445 tz="-0500" srcip=192.168.1.1 srcport=62493 srcintf="port1" srcintfrole="lan" dstip=192.168.100.100 dstport=1235 dstintf="newinterface" dstintfrole="undefined" sessionid=54234 proto=17 action="ip-conn" policyid=49 policytype="policy" poluuid="654cc-b6542-53467u8-e45234-1566casd35f7836" policyname="oldpolicyname" user="elasticsuper" authserver="FSSO_newfsso" service="udp/12302" dstcountry="Reserved" srccountry="Reserved" appcat="unscanned" crscore=5 craction=63332144 crlevel="low"
 <189>date=2020-04-23 time=12:14:28 devname="firewall3" devid="oldfwid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="root" eventtime=1587230069291463928 tz="-0500" srcip=192.168.50.50 srcport=56603 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=442 dstintf="wan1" dstintfrole="wan" sessionid=2345 proto=6 action="close" policyid=2365 policytype="policy" poluuid="654644c-b064-fdgdf3425-f003-1234ghdf682e05f" policyname="someoldpolicyname" user="elasticuser" group="testgroup" authserver="FSSO_something" service="HTTPS" dstcountry="Netherlands" srccountry="Reserved" trandisp="snat" transip=23.23.23.23 transport=603 appid=43540 app="Skype.Portals" appcat="Collaboration" apprisk="elevated" applist="someapplist" appact="detected" duration=126 sentbyte=923 rcvdbyte=77654 sentpkt=113 rcvdpkt=70 vwlid=4 vwlquality="Seq_num(3), alive, selected" wanin=1130 wanout=6671 lanin=1406 lanout=146506 utmaction="block" countweb=1 countapp=1 crscore=5 craction=6144 crlevel="low"
 <190>date=2019-05-15 time=18:03:36 logid="1059028704" type="utm" subtype="app-ctrl" eventtype="app-ctrl-all" level="information" vd="root" eventtime=1557968615 appid=40568 srcip=10.1.100.22 dstip=195.8.215.136 srcport=50798 dstport=443 srcintf="port10" srcintfrole="lan" dstintf="port9" dstintfrole="wan" proto=6 service="HTTPS" direction="outgoing" policyid=1 sessionid=4414 applist="block-social.media" appcat="Web.Client" app="HTTPS.BROWSER" action="pass" hostname="www.dailymotion.com" incidentserialno=1962906680 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="*.dailymotion.com" scertissuer="DigiCert SHA2 High Assurance Server CA"
+<190>date=2020-11-02 time=08:11:38 devname=testfirewall devid=newrouterid logid=0101037127 type="event" subtype=vpn level=notice vd=root logdesc="Progress IPsec phase 1" msg="progress IPsec phase 1" action=negotiate remip=8.8.8.8 locip=10.10.10.10 remport=500 locport=500 outintf="port1" cookies="125cbf9ee8349965/0000000000000000" user="N/A" group="N/A" xauthuser="N/A" xauthgroup="N/A" assignip=N/A vpntunnel="P1_Test" status=success init=local mode=aggressive dir=outbound stage=1 role=initiator result=OK

x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json

@@ -1968,5 +1968,66 @@
         "tls.server.x509.subject.common_name": "*.dailymotion.com",
         "url.domain": "www.dailymotion.com",
         "url.path": "/"
+    },
+    {
+        "@timestamp": "2020-11-02T08:11:38.000Z",
+        "destination.as.number": 15169,
+        "destination.as.organization.name": "Google LLC",
+        "destination.geo.continent_name": "North America",
+        "destination.geo.country_iso_code": "US",
+        "destination.geo.country_name": "United States",
+        "destination.geo.location.lat": 37.751,
+        "destination.geo.location.lon": -97.822,
+        "destination.ip": "8.8.8.8",
+        "destination.port": 500,
+        "event.category": [
+            "network"
+        ],
+        "event.code": "0101037127",
+        "event.dataset": "fortinet.firewall",
+        "event.kind": "event",
+        "event.module": "fortinet",
+        "event.outcome": "success",
+        "event.type": [
+            "connection"
+        ],
+        "fileset.name": "firewall",
+        "fortinet.firewall.action": "negotiate",
+        "fortinet.firewall.cookies": "125cbf9ee8349965/0000000000000000",
+        "fortinet.firewall.init": "local",
+        "fortinet.firewall.mode": "aggressive",
+        "fortinet.firewall.outintf": "port1",
+        "fortinet.firewall.result": "OK",
+        "fortinet.firewall.role": "initiator",
+        "fortinet.firewall.stage": "1",
+        "fortinet.firewall.status": "success",
+        "fortinet.firewall.subtype": "vpn",
+        "fortinet.firewall.type": "event",
+        "fortinet.firewall.vd": "root",
+        "fortinet.firewall.vpntunnel": "P1_Test",
+        "fortinet.firewall.xauthgroup": "N/A",
+        "fortinet.firewall.xauthuser": "N/A",
+        "input.type": "log",
+        "log.level": "notice",
+        "log.offset": 17123,
+        "message": "progress IPsec phase 1",
+        "network.direction": "outbound",
+        "observer.name": "testfirewall",
+        "observer.product": "Fortigate",
+        "observer.serial_number": "newrouterid",
+        "observer.type": "firewall",
+        "observer.vendor": "Fortinet",
+        "related.ip": [
+            "10.10.10.10",
+            "8.8.8.8"
+        ],
+        "rule.description": "Progress IPsec phase 1",
+        "service.type": "fortinet",
+        "source.ip": "10.10.10.10",
+        "source.port": 500,
+        "tags": [
+            "fortinet-firewall",
+            "forwarded"
+        ]
     }
 ]