[Filebeat] Update HA Proxy log grok patterns (#25835) (#26446)

* #25827: Update HA Proxy log grok patterns

* update changelog

* add more sample data

(cherry picked from commit a300f1b)

Co-authored-by: Alex Resnick <adr8292@gmail.com>

CHANGELOG.next.asciidoc

@@ -585,7 +585,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
-
+- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
 
 *Heartbeat*
 

filebeat/module/haproxy/log/ingest/pipeline.yml

@@ -8,21 +8,21 @@ processors:
     field: message
     patterns:
     - '%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?:
-      %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD}
+      %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD}
       %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)'
-    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long}
+    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long}
       \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name}
-      %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long}
+      (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long}
       %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie}
       %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state}
       %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long}
       %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\}
       \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"'
-    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long}
+    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long}
       \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name}
       %{GREEDYDATA:haproxy.error_message}'
     - '%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]:
-      )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\]
+      )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\]
       %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name}
       %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long}
       %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long}
@@ -71,6 +71,15 @@ processors:
     ignore_failure: true
     patterns:
     - ^%{IP:source.ip}$
+- grok:
+    field: destination.address
+    patterns:
+    - ^%{IP:destination.ip}$
+    on_failure:
+      - set:
+          field: destination.domain
+          value: "{{destination.address}}"
+          ignore_empty_value: true
 - geoip:
     field: source.ip
     target_field: source.geo
@@ -121,6 +130,10 @@ processors:
     field: related.ip
     value: "{{destination.ip}}"
     if: "ctx?.destination?.ip != null"
+- append:
+    field: related.hosts
+    value: "{{destination.domain}}"
+    if: "ctx?.destination?.domain != null"
 - set:
     field: event.kind
     value: event

filebeat/module/haproxy/log/test/haproxy.log

@@ -1 +1,10 @@
 Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"
+May 22 02:22:22 server1 haproxy[5089]: -:22222 [22/May/2021:02:22:22.222] www-https~ myapp/node2 site.domain.com 0/0/0/18/18 200 200 - - ---- 222/222/2/0/0 0/0 "OPTIONS /api/v2/app/ HTTP/1.1"
+Jun 22 12:02:53 node2 haproxy[23034]: -:47625 [22/Jun/2021:12:02:53.473] www-https~ app/app-node2 app.domain.com 0/0/1/17/18 302 291 - - ---- 1/1/0/0/0 0/0 "GET / HTTP/1.1"
+Jun 22 12:03:01 node2 haproxy[23034]: -:47445 [22/Jun/2021:12:03:01.501] www-https~ app/node16 app.domain.com 0/0/1/55/56 200 3097 - - ---- 2/2/0/0/0 0/0 "GET /app/login/ HTTP/1.1"
+Jun 22 12:03:01 node2 haproxy[23034]: -:43662 [22/Jun/2021:12:03:01.427] www-https~ app/node7 app.domain.com 0/0/1/30/31 200 1235 - - ---- 1/1/0/0/0 0/0 "GET /23rfsa/ HTTP/1.1"
+Jun 22 12:02:59 node2 haproxy[23034]: -:47481 [22/Jun/2021:12:02:59.590] www-https~ app/node16 app.domain.com 0/0/3/32/35 403 142 - - ---- 1/1/0/0/0 0/0 "GET /app/event/ HTTP/1.1"
+Jun 22 12:02:57 node2 haproxy[23034]: -:47642 [22/Jun/2021:12:02:55.202] www-https~ app/app-node2 app.domain.com 1/0/1/15/2606 200 325791 - - ---- 1/1/0/0/0 0/0 "GET /static/files/3rsdfas3.js HTTP/1.1"
+Jun 22 12:03:08 node2 haproxy[23034]: -:11178 [22/Jun/2021:12:03:08.833] www-https~ app/node7 app.domain.com 0/0/1/29/30 404 448 - - ---- 3/3/0/0/0 0/0 "GET /favicon.ico HTTP/1.1"
+Jun 22 12:03:04 node2 haproxy[23034]: -:21278 [22/Jun/2021:12:03:04.060] www-https~ app/node16 app.domain.com 0/0/2/39/41 200 1235 - - ---- 3/3/0/0/0 0/0 "GET /qfe32/ HTTP/1.1"
+Jun 22 12:03:08 node3 haproxy[23034]: -:21978 [22/Jun/2021:12:03:08.339] www-https~ app/server app.domain.eu 0/0/2/45/47 404 448 - - ---- 3/3/0/0/0 0/0 "GET /dsffdssdf HTTP/1.1"