[Filebeat] Add URI Parts Processor to multiple modules (#24699)

* Update Nginx pipelines * Update Apache, Nginx, IIS, Traefik pipelines * Update AWS S3 * Update Cisco * Update F5 * Update Fortinet * Update Imperva, Netscout, O365, Sophos, Squid, Suricata, Zscaler * additional fixes * update pipelines * unescape \ * remove urldecodes for url.original * updates after rebase * update zeek SIP * update changelog as requested by @andrewstucki * remove `url_decode` for `http.request.referrer` * update generated data
elastic · Apr 27, 2021 · f1fea95 · f1fea95
1 parent 9fb519f
commit f1fea95
Show file tree

Hide file tree

Showing 101 changed files with 5,939 additions and 1,906 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -107,6 +107,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295]
 - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201]
 - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074]
+- All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699]
 
 *Heartbeat*
 
@@ -406,6 +407,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928]
 - Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038]
 - Change `checkpoint.source_object` from Long to Keyword. {issue}25124[25124] {pull}25145[25145]
+- Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699]
 
 *Heartbeat*
 
@@ -849,6 +851,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994]
 - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041]
 - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
+- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml
@@ -4,25 +4,37 @@ processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
-- grok:
+- rename:
     field: message
+    target_field: event.original
+- grok:
+    field: event.original
     patterns:
     - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
-      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?"
       %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
       "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
     - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
-      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?"
       %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
       "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
     - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
       "-" %{NUMBER:http.response.status_code:long} -'
     - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
-      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original}
+      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig}
       HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
     ignore_missing: true
+- uri_parts:
+    field: _tmp.url_orig
+    ignore_failure: true
+- set:
+    field: url.domain
+    value: "{{destination.domain}}"
+    if: ctx.url?.domain == null && ctx.destination?.domain != null
 - remove:
-    field: message
+    field:
+      - _tmp.url_orig
+    ignore_missing: true
 - set:
     field: event.kind
     value: event
@@ -97,6 +109,31 @@ processors:
       }
       ctx.tls.version_protocol = parts[0];
 
+- script:
+    lang: painless
+    description: This script processor iterates over the whole document to remove fields with null values.
+    source: |
+      void handleMap(Map map) {
+        for (def x : map.values()) {
+          if (x instanceof Map) {
+              handleMap(x);
+          } else if (x instanceof List) {
+              handleList(x);
+          }
+        }
+        map.values().removeIf(v -> v == null);
+      }
+      void handleList(List list) {
+        for (def x : list) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+        }
+      }
+      handleMap(ctx);
+
 on_failure:
 - set:
     field: error.message

diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json
@@ -5,6 +5,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45",
         "event.outcome": "success",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -17,6 +18,7 @@
         "source.address": "::1",
         "source.ip": "::1",
         "url.original": "/",
+        "url.path": "/",
         "user.name": "-"
     },
     {
@@ -25,6 +27,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -36,7 +39,9 @@
         "service.type": "apache",
         "source.address": "::1",
         "source.ip": "::1",
+        "url.extension": "ico",
         "url.original": "/favicon.ico",
+        "url.path": "/favicon.ico",
         "user.name": "-"
     },
     {
@@ -45,6 +50,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.response.status_code": 408,
@@ -61,6 +67,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45",
         "event.outcome": "success",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -83,6 +90,7 @@
         "source.geo.region_name": "Rheinland-Pfalz",
         "source.ip": "77.179.66.156",
         "url.original": "/",
+        "url.path": "/",
         "user.name": "-"
     },
     {
@@ -91,6 +99,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -113,6 +122,7 @@
         "source.geo.region_name": "Rheinland-Pfalz",
         "source.ip": "77.179.66.156",
         "url.original": "/notfound",
+        "url.path": "/notfound",
         "user.name": "-"
     },
     {
@@ -121,6 +131,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -143,6 +154,7 @@
         "source.geo.region_name": "Rheinland-Pfalz",
         "source.ip": "77.179.66.156",
         "url.original": "/hmm",
+        "url.path": "/hmm",
         "user.name": "-"
     }
 ]
diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json
@@ -7,6 +7,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375",
         "fileset.name": "access",
         "http.request.method": "GET",
         "http.response.body.bytes": 1375,
@@ -19,7 +20,10 @@
         "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
-        "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
+        "url.extension": "php",
+        "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21",
+        "url.path": "/nagiosxi/ajaxhelper.php",
+        "url.query": "cmd=getxicoreajax&amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
     },
     {
         "@timestamp": "2019-10-16T09:53:47.000Z",
@@ -29,6 +33,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -",
         "fileset.name": "access",
         "http.request.method": "GET",
         "http.version": "1.1",
@@ -45,6 +50,9 @@
         "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
-        "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d"
+        "url.extension": "php",
+        "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d",
+        "url.path": "/appl/ajaxhelper.php",
+        "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d"
     }
 ]
diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json
@@ -6,6 +6,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -17,7 +18,9 @@
         "log.offset": 0,
         "service.type": "apache",
         "source.ip": "192.168.33.2",
+        "url.domain": "vhost1.domaine.fr",
         "url.original": "/hello",
+        "url.path": "/hello",
         "user.name": "-",
         "user_agent.device.name": "Mac",
         "user_agent.name": "Firefox",

diff --git a/filebeat/module/apache/access/test/test.log b/filebeat/module/apache/access/test/test.log
@@ -4,3 +4,4 @@
 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
 monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-"
+monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -5,6 +5,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -16,7 +17,9 @@
         "service.type": "apache",
         "source.address": "::1",
         "source.ip": "::1",
+        "url.extension": "ico",
         "url.original": "/favicon.ico",
+        "url.path": "/favicon.ico",
         "user.name": "-"
     },
     {
@@ -25,6 +28,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -38,6 +42,7 @@
         "source.address": "192.168.33.1",
         "source.ip": "192.168.33.1",
         "url.original": "/hello",
+        "url.path": "/hello",
         "user.name": "-",
         "user_agent.device.name": "Mac",
         "user_agent.name": "Firefox",
@@ -53,6 +58,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.response.status_code": 408,
@@ -69,6 +75,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -82,6 +89,7 @@
         "source.address": "172.17.0.1",
         "source.ip": "172.17.0.1",
         "url.original": "/stringpatch",
+        "url.path": "/stringpatch",
         "user.name": "-",
         "user_agent.device.name": "Other",
         "user_agent.name": "Firefox Alpha",
@@ -97,6 +105,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
         "event.outcome": "success",
         "fileset.name": "access",
         "http.request.method": "GET",
@@ -110,6 +119,7 @@
         "source.address": "monitoring-server",
         "source.domain": "monitoring-server",
         "url.original": "/status",
+        "url.path": "/status",
         "user.name": "-",
         "user_agent.device.name": "Other",
         "user_agent.name": "Firefox Alpha",
@@ -125,6 +135,7 @@
         "event.dataset": "apache.access",
         "event.kind": "event",
         "event.module": "apache",
+        "event.original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"",
         "event.outcome": "failure",
         "fileset.name": "access",
         "http.request.referrer": "-",
@@ -139,5 +150,36 @@
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2017-05-29T19:02:48.000Z",
+        "event.category": "web",
+        "event.dataset": "apache.access",
+        "event.kind": "event",
+        "event.module": "apache",
+        "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
+        "event.outcome": "success",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.body.bytes": 612,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 666,
+        "service.type": "apache",
+        "source.address": "monitoring-server",
+        "source.domain": "monitoring-server",
+        "url.extension": "mp4",
+        "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4",
+        "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Firefox Alpha",
+        "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2",
+        "user_agent.os.full": "Windows 7",
+        "user_agent.os.name": "Windows",
+        "user_agent.os.version": "7",
+        "user_agent.version": "15.0.a2"
     }
 ]