From d5bbb949d4983a1860d067838ab38ba6ab035b7c Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 25 May 2021 02:29:14 +0000 Subject: [PATCH] #25827: Update HA Proxy log grok patterns --- .../module/haproxy/log/ingest/pipeline.yml | 23 +++++++-- filebeat/module/haproxy/log/test/haproxy.log | 1 + .../log/test/haproxy.log-expected.json | 48 +++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index f491556bd81..2813beaa155 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -8,21 +8,21 @@ processors: field: message patterns: - '%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?: - %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} + %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)' - - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} + - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} - %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} + (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"' - - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} + - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}' - '%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: - )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] + )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} @@ -71,6 +71,15 @@ processors: ignore_failure: true patterns: - ^%{IP:source.ip}$ +- grok: + field: destination.address + patterns: + - ^%{IP:destination.ip}$ + on_failure: + - set: + field: destination.domain + value: "{{destination.address}}" + ignore_empty_value: true - geoip: field: source.ip target_field: source.geo @@ -121,6 +130,10 @@ processors: field: related.ip value: "{{destination.ip}}" if: "ctx?.destination?.ip != null" +- append: + field: related.hosts + value: "{{destination.domain}}" + if: "ctx?.destination?.domain != null" - set: field: event.kind value: event diff --git a/filebeat/module/haproxy/log/test/haproxy.log b/filebeat/module/haproxy/log/test/haproxy.log index ad3550d19c9..4115c0b1af0 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log +++ b/filebeat/module/haproxy/log/test/haproxy.log @@ -1 +1,2 @@ Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1" +May 22 02:22:22 server1 haproxy[5089]: -:22222 [22/May/2021:02:22:22.222] www-https~ myapp/node2 site.domain.com 0/0/0/18/18 200 200 - - ---- 222/222/2/0/0 0/0 "OPTIONS /api/v2/app/ HTTP/1.1" diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index b8e839b8da6..484af6a2eec 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -59,5 +59,53 @@ "url.extension": "js", "url.original": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js", "url.path": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js" + }, + { + "destination.address": "site.domain.com", + "destination.domain": "site.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 18000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "myapp", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 200, + "haproxy.connection_wait_time_ms": 0, + "haproxy.connections.active": 222, + "haproxy.connections.backend": 2, + "haproxy.connections.frontend": 222, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "OPTIONS /api/v2/app/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 18, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node2", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "OPTIONS", + "http.response.bytes": 200, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 260, + "process.name": "haproxy", + "process.pid": 5089, + "related.hosts": [ + "site.domain.com" + ], + "service.type": "haproxy", + "source.port": 22222, + "url.original": "/api/v2/app/", + "url.path": "/api/v2/app/" } ] \ No newline at end of file