diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 8b57474c79d..8223b13973f 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -452,6 +452,16 @@ to: event.duration alias: true +## MongoDB module + +- from: mongodb.log.severity + to: log.level + alias: true + +- from: mongodb.log.message + to: message + alias: true + ## NGINX module - from: nginx.access.user_name diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 107d6ca16e1..959c13ce8b2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -6264,18 +6264,6 @@ Contains fields from MongoDB logs. -*`mongodb.log.severity`*:: -+ --- -type: keyword - -example: I - -Severity level of message - - --- - *`mongodb.log.component`*:: + -- @@ -6300,13 +6288,21 @@ Context of message -- -*`mongodb.log.message`*:: +*`mongodb.log.severity`*:: + -- -type: text +type: alias + +alias to: log.level -The message in the log line. +-- +*`mongodb.log.message`*:: ++ +-- +type: alias + +alias to: message -- diff --git a/filebeat/module/mongodb/fields.go b/filebeat/module/mongodb/fields.go index d04861b6df5..ab79107d08f 100644 --- a/filebeat/module/mongodb/fields.go +++ b/filebeat/module/mongodb/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJyUk8FOg0AQhu88xZ/e2wfgYKJtmnhAD/oCKwzbSZcZsrutxac3SzHSiCXMceD/v29CWONIXY5GxWr1kQGRo6Mcq2GzyoCKQum5jayS4yEDgEKrkyPU6tEaH1gsihTYPcGpRc2OwiYDaiZXhbzPrCGmoTErTexaymG9ntphM4G7zr4vQ+21GdN6UJoxbAx0akctf4F3oWm2KtGwhIHwv8GUxa9HoDN5jt1N+10w8DZk4OhMDlqjoRCMpZs36WKaNn2255v19dQjdZ/qqwmhUptWhSQuMdqfpEzPjENpIln1/GXSYlZu+1oUjy+7hYoS6bJIcHuNzOqwcDRSOQ6RZJHUVO2M1PuBfmJgQTxQ/6M4FtpMsPubvwMAAP//pW75AQ==" + return "eJyUkdFO6zAMhu/7FL92vz1AL4502LS7wjOE1g3WkjhK0rHy9CjtJFoIBXxp6/f3Wd7jQmMNK05L91wBiZOhGrt7Z1cBHcU2sE8srsa/CgAa6QZD6CXAqxDZaTQ5cHqAEY2eDcVDBfRMpov1lNnDKUtLVq40eqqhgwz+3ing5jpPy9AHsUvaBMq1hC2BRvRiy1fgJjTXUVxS7OKd8L1ByeLDoxXrxZFLq/WbZOA8uDbPlEGrEmkJ/KZyA9LDUoxK0ypEN2V9fuHxqWn+P55Ww/n4C42vErqiokt0+5PgcY78qMOOk3Kd4ZjI/UKqYBfpSoHTWEgrwyqu+l6ll+n3B0NXMquZZR3UfFAKAxVQpVO2SaXEZ857AAAA//9KC+uk" } diff --git a/filebeat/module/mongodb/log/_meta/fields.yml b/filebeat/module/mongodb/log/_meta/fields.yml index a45d2ca9fc7..6e05805b15c 100644 --- a/filebeat/module/mongodb/log/_meta/fields.yml +++ b/filebeat/module/mongodb/log/_meta/fields.yml @@ -3,11 +3,6 @@ description: > Contains fields from MongoDB logs. fields: - - name: severity - description: > - Severity level of message - example: I - type: keyword - name: component description: > Functional categorization of message @@ -18,7 +13,12 @@ Context of message example: initandlisten type: keyword + + - name: severity + type: alias + path: log.level + migration: true - name: message - description: > - The message in the log line. - type: text + type: alias + path: message + migration: true diff --git a/filebeat/module/mongodb/log/ingest/pipeline.json b/filebeat/module/mongodb/log/ingest/pipeline.json index 0dcf2c066a3..86db2fea118 100755 --- a/filebeat/module/mongodb/log/ingest/pipeline.json +++ b/filebeat/module/mongodb/log/ingest/pipeline.json @@ -4,20 +4,15 @@ "grok": { "field": "message", "patterns":[ - "%{TIMESTAMP_ISO8601:mongodb.log.timestamp} %{WORD:mongodb.log.severity} %{WORD:mongodb.log.component} \\s*\\[%{WORD:mongodb.log.context}\\] %{GREEDYDATA:mongodb.log.message}" + "%{TIMESTAMP_ISO8601:mongodb.log.timestamp} %{WORD:log.level} %{WORD:mongodb.log.component} \\s*\\[%{WORD:mongodb.log.context}\\] %{GREEDYDATA:message}" ], "ignore_missing": true } }, - { - "remove": { - "field": "message" - } - }, { "rename": { "field": "@timestamp", - "target_field": "read_timestamp" + "target_field": "event.created" } }, { diff --git a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json index 0d8e3c67a8f..e176e237ebb 100644 --- a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json +++ b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json @@ -6,11 +6,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 0, + "message": "git version: 009580ad490190ba33d1c6253ebd8d91808923e4", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "git version: 009580ad490190ba33d1c6253ebd8d91808923e4", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -19,11 +19,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 110, + "message": "modules: none", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "modules: none", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -32,11 +32,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 180, + "message": "OpenSSL version: OpenSSL 1.0.2l 25 May 2017", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "OpenSSL version: OpenSSL 1.0.2l 25 May 2017", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.677Z", @@ -45,11 +45,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 281, + "message": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", "mongodb.log.component": "STORAGE", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.724Z", @@ -58,11 +58,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 621, + "message": "Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", "mongodb.log.component": "FTDC", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.724Z", @@ -71,11 +71,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 774, + "message": "Starting hostname canonicalization worker", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "HostnameCanonicalizationWorker", - "mongodb.log.message": "Starting hostname canonicalization worker", - "mongodb.log.severity": "I" + "mongodb.log.context": "HostnameCanonicalizationWorker" }, { "@timestamp": "2018-02-05T12:44:56.744Z", @@ -84,11 +84,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 889, + "message": "waiting for connections on port 27017", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "waiting for connections on port 27017", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:50:55.170Z", @@ -97,11 +97,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 983, + "message": "end connection 127.0.0.1:55404 (0 connections now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "conn1", - "mongodb.log.message": "end connection 127.0.0.1:55404 (0 connections now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "conn1" }, { "@timestamp": "2018-02-05T12:50:55.487Z", @@ -110,11 +110,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1087, + "message": "connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -123,11 +123,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1211, + "message": "now exiting", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "now exiting", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -136,11 +136,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1288, + "message": "closing listening socket: 7", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "closing listening socket: 7", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -149,11 +149,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1381, + "message": "removing socket file: /run/mongodb/mongodb-27017.sock", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "removing socket file: /run/mongodb/mongodb-27017.sock", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -162,11 +162,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1500, + "message": "shutdown: going to flush diaglog...", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "shutdown: going to flush diaglog...", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -175,11 +175,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1601, + "message": "shutdown: going to close sockets...", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "shutdown: going to close sockets...", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.688Z", @@ -188,11 +188,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1702, + "message": "shutdown: removing fs lock...", "mongodb.log.component": "STORAGE", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "shutdown: removing fs lock...", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -201,11 +201,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1797, + "message": "db version v3.2.11", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "db version v3.2.11", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -214,11 +214,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1872, + "message": "build environment:", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "build environment:", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -227,11 +227,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 1947, + "message": " distarch: x86_64", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": " distarch: x86_64", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -240,11 +240,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2024, + "message": "options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:50:55.170Z", @@ -253,11 +253,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2361, + "message": "connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:50:56.180Z", @@ -266,11 +266,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2485, + "message": "end connection 127.0.0.1:55414 (0 connections now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "conn3", - "mongodb.log.message": "end connection 127.0.0.1:55414 (0 connections now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "conn3" }, { "@timestamp": "2018-02-05T13:15:42.095Z", @@ -279,11 +279,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2589, + "message": "end connection 127.0.0.1:58336 (0 connections now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "conn4", - "mongodb.log.message": "end connection 127.0.0.1:58336 (0 connections now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "conn4" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -292,11 +292,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2693, + "message": "shutdown: going to close listening sockets...", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "shutdown: going to close listening sockets...", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -305,11 +305,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2804, + "message": "WiredTigerKVEngine shutting down", "mongodb.log.component": "STORAGE", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "WiredTigerKVEngine shutting down", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.688Z", @@ -318,11 +318,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2902, + "message": "dbexit: rc: 0", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "dbexit: rc: 0", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -331,11 +331,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 2982, + "message": "MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -344,11 +344,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3123, + "message": "allocator: tcmalloc", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "allocator: tcmalloc", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:44:56.657Z", @@ -357,11 +357,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3199, + "message": " target_arch: x86_64", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": " target_arch: x86_64", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T12:50:55.487Z", @@ -370,11 +370,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3279, + "message": "end connection 127.0.0.1:55406 (0 connections now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "conn2", - "mongodb.log.message": "end connection 127.0.0.1:55406 (0 connections now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "conn2" }, { "@timestamp": "2018-02-05T12:50:56.180Z", @@ -383,11 +383,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3383, + "message": "connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T13:11:41.401Z", @@ -396,11 +396,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3507, + "message": "connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "initandlisten", - "mongodb.log.message": "connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", - "mongodb.log.severity": "I" + "mongodb.log.context": "initandlisten" }, { "@timestamp": "2018-02-05T13:49:45.605Z", @@ -409,11 +409,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3631, + "message": "got signal 15 (Terminated), will terminate after current cmd ends", "mongodb.log.component": "CONTROL", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "got signal 15 (Terminated), will terminate after current cmd ends", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.605Z", @@ -422,11 +422,11 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3762, + "message": "Shutting down full-time diagnostic data capture", "mongodb.log.component": "FTDC", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "Shutting down full-time diagnostic data capture", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" }, { "@timestamp": "2018-02-05T13:49:45.606Z", @@ -435,10 +435,10 @@ "event.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.level": "I", "log.offset": 3875, + "message": "closing listening socket: 6", "mongodb.log.component": "NETWORK", - "mongodb.log.context": "signalProcessingThread", - "mongodb.log.message": "closing listening socket: 6", - "mongodb.log.severity": "I" + "mongodb.log.context": "signalProcessingThread" } ] \ No newline at end of file