elastic#25827: Update HA Proxy log grok patterns

legoguy1000 · Jun 23, 2021 · d5bbb94 · d5bbb94
1 parent b7341ae
commit d5bbb94
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 5 deletions.
diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml
@@ -8,21 +8,21 @@ processors:
     field: message
     patterns:
     - '%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?:
-      %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD}
+      %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD}
       %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)'
-    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long}
+    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long}
       \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name}
-      %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long}
+      (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long}
       %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie}
       %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state}
       %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long}
       %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\}
       \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"'
-    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long}
+    - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long}
       \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name}
       %{GREEDYDATA:haproxy.error_message}'
     - '%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]:
-      )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\]
+      )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\]
       %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name}
       %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long}
       %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long}
@@ -71,6 +71,15 @@ processors:
     ignore_failure: true
     patterns:
     - ^%{IP:source.ip}$
+- grok:
+    field: destination.address
+    patterns:
+    - ^%{IP:destination.ip}$
+    on_failure:
+      - set:
+          field: destination.domain
+          value: "{{destination.address}}"
+          ignore_empty_value: true
 - geoip:
     field: source.ip
     target_field: source.geo
@@ -121,6 +130,10 @@ processors:
     field: related.ip
     value: "{{destination.ip}}"
     if: "ctx?.destination?.ip != null"
+- append:
+    field: related.hosts
+    value: "{{destination.domain}}"
+    if: "ctx?.destination?.domain != null"
 - set:
     field: event.kind
     value: event

diff --git a/filebeat/module/haproxy/log/test/haproxy.log b/filebeat/module/haproxy/log/test/haproxy.log
@@ -1 +1,2 @@
 Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1"
+May 22 02:22:22 server1 haproxy[5089]: -:22222 [22/May/2021:02:22:22.222] www-https~ myapp/node2 site.domain.com 0/0/0/18/18 200 200 - - ---- 222/222/2/0/0 0/0 "OPTIONS /api/v2/app/ HTTP/1.1"
diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json
@@ -59,5 +59,53 @@
         "url.extension": "js",
         "url.original": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js",
         "url.path": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js"
+    },
+    {
+        "destination.address": "site.domain.com",
+        "destination.domain": "site.domain.com",
+        "event.category": [
+            "web"
+        ],
+        "event.dataset": "haproxy.log",
+        "event.duration": 18000000,
+        "event.kind": "event",
+        "event.module": "haproxy",
+        "event.outcome": "success",
+        "event.timezone": "-02:00",
+        "fileset.name": "log",
+        "haproxy.backend_name": "myapp",
+        "haproxy.backend_queue": 0,
+        "haproxy.bytes_read": 200,
+        "haproxy.connection_wait_time_ms": 0,
+        "haproxy.connections.active": 222,
+        "haproxy.connections.backend": 2,
+        "haproxy.connections.frontend": 222,
+        "haproxy.connections.retries": 0,
+        "haproxy.connections.server": 0,
+        "haproxy.frontend_name": "www-https~",
+        "haproxy.http.request.captured_cookie": "-",
+        "haproxy.http.request.raw_request_line": "OPTIONS /api/v2/app/ HTTP/1.1",
+        "haproxy.http.request.time_wait_ms": 0,
+        "haproxy.http.request.time_wait_without_data_ms": 18,
+        "haproxy.http.response.captured_cookie": "-",
+        "haproxy.server_name": "node2",
+        "haproxy.server_queue": 0,
+        "haproxy.termination_state": "----",
+        "haproxy.total_waiting_time_ms": 0,
+        "http.request.method": "OPTIONS",
+        "http.response.bytes": 200,
+        "http.response.status_code": 200,
+        "http.version": "1.1",
+        "input.type": "log",
+        "log.offset": 260,
+        "process.name": "haproxy",
+        "process.pid": 5089,
+        "related.hosts": [
+            "site.domain.com"
+        ],
+        "service.type": "haproxy",
+        "source.port": 22222,
+        "url.original": "/api/v2/app/",
+        "url.path": "/api/v2/app/"
     }
 ]