diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d5b2edf2e3d..f9612a4a427 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -824,7 +824,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267] - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267] - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293] - +- Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835] *Heartbeat* diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index f491556bd81..2813beaa155 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -8,21 +8,21 @@ processors: field: message patterns: - '%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?: - %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} + %{GREEDYDATA} (%{IPORHOST:source.address}|-):%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)' - - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} + - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} - %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} + (%{IPORHOST:destination.address} )?%{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?"%{GREEDYDATA:haproxy.http.request.raw_request_line}"' - - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} + - '(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}' - '%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: - )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] + )?(%{IP:source.address}|-):%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} @@ -71,6 +71,15 @@ processors: ignore_failure: true patterns: - ^%{IP:source.ip}$ +- grok: + field: destination.address + patterns: + - ^%{IP:destination.ip}$ + on_failure: + - set: + field: destination.domain + value: "{{destination.address}}" + ignore_empty_value: true - geoip: field: source.ip target_field: source.geo @@ -121,6 +130,10 @@ processors: field: related.ip value: "{{destination.ip}}" if: "ctx?.destination?.ip != null" +- append: + field: related.hosts + value: "{{destination.domain}}" + if: "ctx?.destination?.domain != null" - set: field: event.kind value: event diff --git a/filebeat/module/haproxy/log/test/haproxy.log b/filebeat/module/haproxy/log/test/haproxy.log index ad3550d19c9..1f50b581c73 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log +++ b/filebeat/module/haproxy/log/test/haproxy.log @@ -1 +1,10 @@ Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1" +May 22 02:22:22 server1 haproxy[5089]: -:22222 [22/May/2021:02:22:22.222] www-https~ myapp/node2 site.domain.com 0/0/0/18/18 200 200 - - ---- 222/222/2/0/0 0/0 "OPTIONS /api/v2/app/ HTTP/1.1" +Jun 22 12:02:53 node2 haproxy[23034]: -:47625 [22/Jun/2021:12:02:53.473] www-https~ app/app-node2 app.domain.com 0/0/1/17/18 302 291 - - ---- 1/1/0/0/0 0/0 "GET / HTTP/1.1" +Jun 22 12:03:01 node2 haproxy[23034]: -:47445 [22/Jun/2021:12:03:01.501] www-https~ app/node16 app.domain.com 0/0/1/55/56 200 3097 - - ---- 2/2/0/0/0 0/0 "GET /app/login/ HTTP/1.1" +Jun 22 12:03:01 node2 haproxy[23034]: -:43662 [22/Jun/2021:12:03:01.427] www-https~ app/node7 app.domain.com 0/0/1/30/31 200 1235 - - ---- 1/1/0/0/0 0/0 "GET /23rfsa/ HTTP/1.1" +Jun 22 12:02:59 node2 haproxy[23034]: -:47481 [22/Jun/2021:12:02:59.590] www-https~ app/node16 app.domain.com 0/0/3/32/35 403 142 - - ---- 1/1/0/0/0 0/0 "GET /app/event/ HTTP/1.1" +Jun 22 12:02:57 node2 haproxy[23034]: -:47642 [22/Jun/2021:12:02:55.202] www-https~ app/app-node2 app.domain.com 1/0/1/15/2606 200 325791 - - ---- 1/1/0/0/0 0/0 "GET /static/files/3rsdfas3.js HTTP/1.1" +Jun 22 12:03:08 node2 haproxy[23034]: -:11178 [22/Jun/2021:12:03:08.833] www-https~ app/node7 app.domain.com 0/0/1/29/30 404 448 - - ---- 3/3/0/0/0 0/0 "GET /favicon.ico HTTP/1.1" +Jun 22 12:03:04 node2 haproxy[23034]: -:21278 [22/Jun/2021:12:03:04.060] www-https~ app/node16 app.domain.com 0/0/2/39/41 200 1235 - - ---- 3/3/0/0/0 0/0 "GET /qfe32/ HTTP/1.1" +Jun 22 12:03:08 node3 haproxy[23034]: -:21978 [22/Jun/2021:12:03:08.339] www-https~ app/server app.domain.eu 0/0/2/45/47 404 448 - - ---- 3/3/0/0/0 0/0 "GET /dsffdssdf HTTP/1.1" diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index b8e839b8da6..9633ab57190 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -59,5 +59,439 @@ "url.extension": "js", "url.original": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js", "url.path": "/component---src-pages-index-js-4b15624544f97cf0bb8f.js" + }, + { + "destination.address": "site.domain.com", + "destination.domain": "site.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 18000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "myapp", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 200, + "haproxy.connection_wait_time_ms": 0, + "haproxy.connections.active": 222, + "haproxy.connections.backend": 2, + "haproxy.connections.frontend": 222, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "OPTIONS /api/v2/app/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 18, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node2", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "OPTIONS", + "http.response.bytes": 200, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 260, + "process.name": "haproxy", + "process.pid": 5089, + "related.hosts": [ + "site.domain.com" + ], + "service.type": "haproxy", + "source.port": 22222, + "url.original": "/api/v2/app/", + "url.path": "/api/v2/app/" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 18000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 291, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 17, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "app-node2", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 291, + "http.response.status_code": 302, + "http.version": "1.1", + "input.type": "log", + "log.offset": 452, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 47625, + "url.original": "/", + "url.path": "/" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 56000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 3097, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 2, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 2, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /app/login/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 55, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node16", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 3097, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 625, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 47445, + "url.original": "/app/login/", + "url.path": "/app/login/" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 31000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 1235, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /23rfsa/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 30, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node7", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 1235, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 806, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 43662, + "url.original": "/23rfsa/", + "url.path": "/23rfsa/" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 35000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 142, + "haproxy.connection_wait_time_ms": 3, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /app/event/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 32, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node16", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 142, + "http.response.status_code": 403, + "http.version": "1.1", + "input.type": "log", + "log.offset": 983, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 47481, + "url.original": "/app/event/", + "url.path": "/app/event/" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 2606000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 325791, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /static/files/3rsdfas3.js HTTP/1.1", + "haproxy.http.request.time_wait_ms": 1, + "haproxy.http.request.time_wait_without_data_ms": 15, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "app-node2", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 325791, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1163, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 47642, + "url.extension": "js", + "url.original": "/static/files/3rsdfas3.js", + "url.path": "/static/files/3rsdfas3.js" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 30000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 448, + "haproxy.connection_wait_time_ms": 1, + "haproxy.connections.active": 3, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 3, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /favicon.ico HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 29, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node7", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 448, + "http.response.status_code": 404, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1365, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 11178, + "url.extension": "ico", + "url.original": "/favicon.ico", + "url.path": "/favicon.ico" + }, + { + "destination.address": "app.domain.com", + "destination.domain": "app.domain.com", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 41000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 1235, + "haproxy.connection_wait_time_ms": 2, + "haproxy.connections.active": 3, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 3, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /qfe32/ HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 39, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "node16", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 1235, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1545, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.com" + ], + "service.type": "haproxy", + "source.port": 21278, + "url.original": "/qfe32/", + "url.path": "/qfe32/" + }, + { + "destination.address": "app.domain.eu", + "destination.domain": "app.domain.eu", + "event.category": [ + "web" + ], + "event.dataset": "haproxy.log", + "event.duration": 47000000, + "event.kind": "event", + "event.module": "haproxy", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "log", + "haproxy.backend_name": "app", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 448, + "haproxy.connection_wait_time_ms": 2, + "haproxy.connections.active": 3, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 3, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "www-https~", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /dsffdssdf HTTP/1.1", + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": 45, + "haproxy.http.response.captured_cookie": "-", + "haproxy.server_name": "server", + "haproxy.server_queue": 0, + "haproxy.termination_state": "----", + "haproxy.total_waiting_time_ms": 0, + "http.request.method": "GET", + "http.response.bytes": 448, + "http.response.status_code": 404, + "http.version": "1.1", + "input.type": "log", + "log.offset": 1722, + "process.name": "haproxy", + "process.pid": 23034, + "related.hosts": [ + "app.domain.eu" + ], + "service.type": "haproxy", + "source.port": 21978, + "url.original": "/dsffdssdf", + "url.path": "/dsffdssdf" } ] \ No newline at end of file