diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 20d8ed3746e..0ab846dfaf9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added netflow input type that supports NetFlow v1, v5, v6, v7, v8, v9 and IPFIX. {issue}9399[9399] - Add option to modules.yml file to indicate that a module has been moved {pull}9432[9432]. - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810] +- Support mysql 5.7.22 slowlog starting with time information. {issue}7892[7892] {pull}9647[9647] *Heartbeat* diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml index ec998f5e646..d8ffab0552c 100644 --- a/filebeat/module/mysql/slowlog/config/slowlog.yml +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -5,7 +5,7 @@ paths: {{ end }} exclude_files: ['.gz$'] multiline: - pattern: '^# User@Host: ' + pattern: '^(# User@Host: |# Time: )' negate: true match: after -exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*'] # Exclude the header +exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*', '^# Time:.*'] # Exclude the header and time diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log new file mode 100644 index 00000000000..a888e66bc9a --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log @@ -0,0 +1,30 @@ +# Time: 2018-08-07T16:27:47.169604+08:00 +# User@Host: root[root] @ [218.76.8.37] Id: 7234 +# Query_time: 15.000223 Lock_time: 0.000000 Rows_sent: 1 Rows_examined: 0 +SET timestamp=1533630467; +select sleep(15); +# Time: 2018-08-07T16:27:47.169604+08:00 +# User@Host: debian-sys-maint[debian-sys-maint] @ localhost [] +# Query_time: 0.000153 Lock_time: 0.000061 Rows_sent: 1 Rows_examined: 5 +SET timestamp=1533630467; +SELECT count(*) FROM mysql.user WHERE user='root' and password=''; +# Time: 2018-08-07T16:27:47.169604+08:00 +# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10997316 +# Query_time: 4.071491 Lock_time: 0.000212 Rows_sent: 1000 Rows_examined: 1489615 +SET timestamp=1533630467; +SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid + FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca + WHERE cus.cus_guid = mcu.cus_guid + AND cus.pro_code = 'CYB' + AND cus.cus_offline = 0 + AND mca.cus_guid = cus.cus_guid + AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE()) + AND mcu.mcu_crawlelements IS NOT NULL + GROUP BY mcu.mcu_guid + ORDER BY mcu.mcu_order ASC + LIMIT 1000; +# Time: 2018-08-07T16:27:47.169604+08:00 +# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10999834 +# Query_time: 10.346539 Lock_time: 0.000036 Rows_sent: 0 Rows_examined: 4751313 +SET timestamp=1533630467; +call load_stats(1, '2017-04-28 00:00:00'); diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json new file mode 100644 index 00000000000..3e6db0006c0 --- /dev/null +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json @@ -0,0 +1,83 @@ +[ + { + "@timestamp": "2018-08-07T08:27:47.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "slowlog", + "event.module": "mysql", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 41, + "mysql.slowlog.id": "7234", + "mysql.slowlog.ip": "218.76.8.37", + "mysql.slowlog.lock_time.sec": "0.000000", + "mysql.slowlog.query": "select sleep(15);", + "mysql.slowlog.query_time.sec": "15.000223", + "mysql.slowlog.rows_examined": "0", + "mysql.slowlog.rows_sent": "1", + "mysql.slowlog.timestamp": "1533630467", + "mysql.slowlog.user": "root" + }, + { + "@timestamp": "2018-08-07T08:27:47.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "slowlog", + "event.module": "mysql", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 254, + "mysql.slowlog.host": "localhost", + "mysql.slowlog.lock_time.sec": "0.000061", + "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", + "mysql.slowlog.query_time.sec": "0.000153", + "mysql.slowlog.rows_examined": "5", + "mysql.slowlog.rows_sent": "1", + "mysql.slowlog.timestamp": "1533630467", + "mysql.slowlog.user": "debian-sys-maint" + }, + { + "@timestamp": "2018-08-07T08:27:47.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "slowlog", + "event.module": "mysql", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 526, + "mysql.slowlog.host": "apphost", + "mysql.slowlog.id": "10997316", + "mysql.slowlog.ip": "1.1.1.1", + "mysql.slowlog.lock_time.sec": "0.000212", + "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = 'CYB'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;", + "mysql.slowlog.query_time.sec": "4.071491", + "mysql.slowlog.rows_examined": "1489615", + "mysql.slowlog.rows_sent": "1000", + "mysql.slowlog.timestamp": "1533630467", + "mysql.slowlog.user": "apphost" + }, + { + "@timestamp": "2018-08-07T08:27:47.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "slowlog", + "event.module": "mysql", + "input.type": "log", + "log.flags": [ + "multiline" + ], + "log.offset": 1438, + "mysql.slowlog.host": "apphost", + "mysql.slowlog.id": "10999834", + "mysql.slowlog.ip": "1.1.1.1", + "mysql.slowlog.lock_time.sec": "0.000036", + "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');", + "mysql.slowlog.query_time.sec": "10.346539", + "mysql.slowlog.rows_examined": "4751313", + "mysql.slowlog.rows_sent": "0", + "mysql.slowlog.timestamp": "1533630467", + "mysql.slowlog.user": "apphost" + } +] \ No newline at end of file