o365: Support IPv4 enclosed in square brackets (elastic#18591)

The O365 Management API is generating some events that contain a `ClientIP` / `ClientIPAddress` field consisting of an IPv4 address enclosed in square brackets. This is breaking ingestion of those events as the brackets are only stripped for IPv6 addresses. > "ClientIP": "[10.11.12.13]:12345" This patch adds support for IPv4 enclosed in brackets and a few other edge cases. Fixes elastic#18587 (cherry picked from commit eaf196d)
adriansr · May 16, 2020 · 1002f13 · 1002f13
1 parent b9093e8
commit 1002f13
Show file tree

Hide file tree

Showing 4 changed files with 419 additions and 10 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix a rate limit related issue in httpjson input for Okta module. {issue}18530[18530] {pull}18534[18534]
 - Fixed ingestion of some Cisco ASA and FTD messages when a hostname was used instead of an IP for NAT fields. {issue}14034[14034] {pull}18376[18376]
 - Fix PANW module wrong mappings for bytes and packets counters. {issue}18522[18522] {pull}18525[18525]
+- Fix `o365.audit` failing to ingest events when ip address is surrounded by square brackets. {issue}18587[18587] {pull}18591[18591]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js
@@ -725,22 +725,23 @@ function AuditProcessor(tenant_names, debug) {
         'Yammer': yammerSchema(debug).Run,
     }));
 
-    builder.Add("extractClientIPv4Port", new processor.Dissect({
-        tokenizer: '%{ip}:%{port}',
+    builder.Add("extractClientIPPortBrackets", new processor.Dissect({
+        tokenizer: '[%{_ip}]:%{port}',
         field: 'client.address',
         target_prefix: 'client',
-        'when.and': [
-            {'contains.client.address': '.'},
-            {'contains.client.address': ':'},
-        ],
+        'when.contains.client.address': ']:',
     }));
-    builder.Add("extractClientIPv6Port", new processor.Dissect({
-        tokenizer: '[%{ip}]:%{port}',
+    builder.Add("extractClientIPv4Port", new processor.Dissect({
+        tokenizer: '%{_ip}:%{port}',
         field: 'client.address',
         target_prefix: 'client',
         'when.and': [
-            {'contains.client.address': '['},
+            {'not.has_fields': ['client._ip', 'client.port']},
+            {'contains.client.address': '.'},
             {'contains.client.address': ':'},
+            // Best effort to avoid parsing IPv6-mapped IPv4 as ip:port.
+            // Won't succeed if IPv6 address is not shortened.
+            {'not.contains.client.address': '::'},
         ],
     }));
 
@@ -749,11 +750,14 @@ function AuditProcessor(tenant_names, debug) {
         fields: [
             {from: "client.address", to: "client.ip", type: "ip"},
             {from: "server.address", to: "server.ip", type: "ip"},
+            {from: "client._ip",     to: "client.ip", type: "ip"},
         ],
         ignore_missing: true,
         fail_on_error: false
     }));
-
+    builder.Add("removeTempIP", function (evt) {
+        evt.Delete("client._ip");
+    });
     builder.Add("setSrcDstFields", new processor.Convert({
         fields: [
             {from: "client.ip", to: "source.ip"},

diff --git a/x-pack/filebeat/module/o365/audit/test/ip-formats.log b/x-pack/filebeat/module/o365/audit/test/ip-formats.log
@@ -0,0 +1,15 @@
+{"ClientIP":"[10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"10.11.12.13:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"::ffff:10.11.12.13","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[::ffff:10.11.12.13]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[2001:db8::abcd]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"2001:db8::abcd","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[2001:db8::abcd]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[10.11.12.13]","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"localhost","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[localhost]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"localhost:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"[cool.client.local]:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"cool.client.local","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}
+{"ClientIP":"cool.client.local:12345","RecordType":-1,"CreationTime":"2020-02-17T17:12:03","Id":"3be78a31-dbd3-4c2c-eaf9-08d7b3cc8226"}