From baa22e9beb6d377c7b393914b3266a5923431376 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 27 Dec 2018 01:27:23 -0700 Subject: [PATCH] Cherrypick to 6.x: Support different timestamp format in postgresql module (#9650) (#9794) * Support different timestamp format in postgresql module (#9650) * Support different timestamp format in postgresql module * Update changelog (cherry picked from commit 9d60da51b93bad23683d1baa1eec1f67ac1f630a) * Rerun test with 6.x ES --- CHANGELOG.asciidoc | 1 + .../postgresql/log/ingest/pipeline.json | 2 +- .../log/test/postgresql-9.6-new-timestamp.log | 5 ++ ...gresql-9.6-new-timestamp.log-expected.json | 86 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log create mode 100644 filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a9b4b23d6b4..987589836ce 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -223,6 +223,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346] - Mark the TCP and UDP input as GA. {pull}8125[8125] - Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562] +- Support different timestamp format in postgresql module. {issue}9494[9494] {pull}9650[9650] *Heartbeat* diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 398b1d95405..aa6d8309482 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -20,7 +20,7 @@ "field": "postgresql.log.timestamp", "target_field": "@timestamp", "formats": [ - "yyyy-MM-dd HH:mm:ss.SSS" + "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ], "ignore_failure": true } diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log new file mode 100644 index 00000000000..2042fb9f0a6 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log @@ -0,0 +1,5 @@ +2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started +2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart) +2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter "max_wal_size". +2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up +2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json new file mode 100644 index 00000000000..5e24cf0952f --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json @@ -0,0 +1,86 @@ +[ + { + "@timestamp": "2017-07-31T13:36:43.000Z", + "event.dataset": "postgresql.log", + "fileset.module": "postgresql", + "fileset.name": "log", + "input.type": "log", + "message": "2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started", + "offset": 0, + "postgresql.log.core_id": "1", + "postgresql.log.level": "LOG", + "postgresql.log.message": "autovacuum launcher started", + "postgresql.log.thread_id": "835", + "postgresql.log.timestamp": "2017-07-31 13:36:43", + "postgresql.log.timezone": "EST", + "prospector.type": "log" + }, + { + "@timestamp": "2017-07-31T13:36:44.000Z", + "event.dataset": "postgresql.log", + "fileset.module": "postgresql", + "fileset.name": "log", + "input.type": "log", + "message": "2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart)", + "offset": 66, + "postgresql.log.core_id": "1", + "postgresql.log.level": "LOG", + "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", + "postgresql.log.thread_id": "832", + "postgresql.log.timestamp": "2017-07-31 13:36:44", + "postgresql.log.timezone": "EST", + "prospector.type": "log" + }, + { + "@timestamp": "2017-07-31T13:46:02.000Z", + "event.dataset": "postgresql.log", + "fileset.module": "postgresql", + "fileset.name": "log", + "input.type": "log", + "message": "2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter \"max_wal_size\".", + "offset": 164, + "postgresql.log.core_id": "2", + "postgresql.log.level": "HINT", + "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", + "postgresql.log.thread_id": "832", + "postgresql.log.timestamp": "2017-07-31 13:46:02", + "postgresql.log.timezone": "EST", + "prospector.type": "log" + }, + { + "@timestamp": "2017-07-31T13:46:23.000Z", + "event.dataset": "postgresql.log", + "fileset.module": "postgresql", + "fileset.name": "log", + "input.type": "log", + "message": "2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up", + "offset": 267, + "postgresql.log.core_id": "1", + "postgresql.log.database": "postgres", + "postgresql.log.level": "FATAL", + "postgresql.log.message": "the database system is starting up", + "postgresql.log.thread_id": "768", + "postgresql.log.timestamp": "2017-07-31 13:46:23", + "postgresql.log.timezone": "EST", + "postgresql.log.user": "postgres", + "prospector.type": "log" + }, + { + "@timestamp": "2017-07-31T13:46:55.000Z", + "event.dataset": "postgresql.log", + "fileset.module": "postgresql", + "fileset.name": "log", + "input.type": "log", + "message": "2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up", + "offset": 360, + "postgresql.log.core_id": "1", + "postgresql.log.database": "postgres", + "postgresql.log.level": "FATAL", + "postgresql.log.message": "the database system is starting up", + "postgresql.log.thread_id": "771", + "postgresql.log.timestamp": "2017-07-31 13:46:55", + "postgresql.log.timezone": "EST", + "postgresql.log.user": "postgres", + "prospector.type": "log" + } +] \ No newline at end of file