Fix mysql slowlog template to support mysql 5.7.22 (#9647)

* Never default to a qualifier when none of them are set. (#9148) Remove default version qualifier and rename the environment variable to set it from `BEAT_VERSION_QUALIFIER` to `VERSION_QUALIFIER` this will align with other parts of the stack. **Tested with filebeat.** ``` ❯ ./filebeat version [08:39:01] filebeat version 7.0.0 (amd64), libbeat 7.0.0 [0a0c267 built 2018-11-19 13:38:15 +0000 UTC] ``` **Without the patch** ``` ❯ ./filebeat version [08:40:07] filebeat version 7.0.0-alpha1 (amd64), libbeat 7.0.0-alpha1 [b007837 built 2018-11-19 13:39:59 +0000 UTC] ``` Fixes: #8384 * Fix mysql slowlog template to support mysql 5.7.22 * Add changelog * Remove changes introduced from rebasing that are not related to this change * Add more log entries to test log file * Add OR in multiline.pattern to remove # Time from mysql.slowlog.query * Use CHANGELOG.next.asciidoc instead
elastic · Dec 31, 2018 · b29ddfc · b29ddfc
1 parent e5afe89
commit b29ddfc
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added netflow input type that supports NetFlow v1, v5, v6, v7, v8, v9 and IPFIX. {issue}9399[9399]
 - Add option to modules.yml file to indicate that a module has been moved {pull}9432[9432].
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
+- Support mysql 5.7.22 slowlog starting with time information. {issue}7892[7892] {pull}9647[9647]
 
 *Heartbeat*
 

diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml
@@ -5,7 +5,7 @@ paths:
 {{ end }}
 exclude_files: ['.gz$']
 multiline:
-  pattern: '^# User@Host: '
+  pattern: '^(# User@Host: |# Time: )'
   negate: true
   match: after
-exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*']   # Exclude the header
+exclude_lines: ['^[\/\w\.]+, Version: .* started with:.*', '^# Time:.*']   # Exclude the header and time
diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log
@@ -0,0 +1,30 @@
+# Time: 2018-08-07T16:27:47.169604+08:00
+# User@Host: root[root] @  [218.76.8.37]  Id:  7234
+# Query_time: 15.000223  Lock_time: 0.000000 Rows_sent: 1  Rows_examined: 0
+SET timestamp=1533630467;
+select sleep(15);
+# Time: 2018-08-07T16:27:47.169604+08:00
+# User@Host: debian-sys-maint[debian-sys-maint] @ localhost []
+# Query_time: 0.000153  Lock_time: 0.000061 Rows_sent: 1  Rows_examined: 5
+SET timestamp=1533630467;
+SELECT count(*) FROM mysql.user WHERE user='root' and password='';
+# Time: 2018-08-07T16:27:47.169604+08:00
+# User@Host: apphost[apphost] @ apphost [1.1.1.1]  Id: 10997316
+# Query_time: 4.071491  Lock_time: 0.000212 Rows_sent: 1000  Rows_examined: 1489615
+SET timestamp=1533630467;
+SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid
+                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca
+                    WHERE cus.cus_guid = mcu.cus_guid
+                        AND cus.pro_code = 'CYB'
+                        AND cus.cus_offline = 0
+                        AND mca.cus_guid = cus.cus_guid
+                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())
+                        AND mcu.mcu_crawlelements IS NOT NULL
+                    GROUP BY mcu.mcu_guid
+                    ORDER BY mcu.mcu_order ASC
+                    LIMIT 1000;
+# Time: 2018-08-07T16:27:47.169604+08:00
+# User@Host: apphost[apphost] @ apphost [1.1.1.1]  Id: 10999834
+# Query_time: 10.346539  Lock_time: 0.000036 Rows_sent: 0  Rows_examined: 4751313
+SET timestamp=1533630467;
+call load_stats(1, '2017-04-28 00:00:00');
diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json
@@ -0,0 +1,83 @@
+[
+    {
+        "@timestamp": "2018-08-07T08:27:47.000Z", 
+        "ecs.version": "1.0.0-beta2", 
+        "event.dataset": "slowlog", 
+        "event.module": "mysql", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "log.offset": 41, 
+        "mysql.slowlog.id": "7234", 
+        "mysql.slowlog.ip": "218.76.8.37", 
+        "mysql.slowlog.lock_time.sec": "0.000000", 
+        "mysql.slowlog.query": "select sleep(15);", 
+        "mysql.slowlog.query_time.sec": "15.000223", 
+        "mysql.slowlog.rows_examined": "0", 
+        "mysql.slowlog.rows_sent": "1", 
+        "mysql.slowlog.timestamp": "1533630467", 
+        "mysql.slowlog.user": "root"
+    }, 
+    {
+        "@timestamp": "2018-08-07T08:27:47.000Z", 
+        "ecs.version": "1.0.0-beta2", 
+        "event.dataset": "slowlog", 
+        "event.module": "mysql", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "log.offset": 254, 
+        "mysql.slowlog.host": "localhost", 
+        "mysql.slowlog.lock_time.sec": "0.000061", 
+        "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", 
+        "mysql.slowlog.query_time.sec": "0.000153", 
+        "mysql.slowlog.rows_examined": "5", 
+        "mysql.slowlog.rows_sent": "1", 
+        "mysql.slowlog.timestamp": "1533630467", 
+        "mysql.slowlog.user": "debian-sys-maint"
+    }, 
+    {
+        "@timestamp": "2018-08-07T08:27:47.000Z", 
+        "ecs.version": "1.0.0-beta2", 
+        "event.dataset": "slowlog", 
+        "event.module": "mysql", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "log.offset": 526, 
+        "mysql.slowlog.host": "apphost", 
+        "mysql.slowlog.id": "10997316", 
+        "mysql.slowlog.ip": "1.1.1.1", 
+        "mysql.slowlog.lock_time.sec": "0.000212", 
+        "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n                    WHERE cus.cus_guid = mcu.cus_guid\n                        AND cus.pro_code = 'CYB'\n                        AND cus.cus_offline = 0\n                        AND mca.cus_guid = cus.cus_guid\n                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n                        AND mcu.mcu_crawlelements IS NOT NULL\n                    GROUP BY mcu.mcu_guid\n                    ORDER BY mcu.mcu_order ASC\n                    LIMIT 1000;", 
+        "mysql.slowlog.query_time.sec": "4.071491", 
+        "mysql.slowlog.rows_examined": "1489615", 
+        "mysql.slowlog.rows_sent": "1000", 
+        "mysql.slowlog.timestamp": "1533630467", 
+        "mysql.slowlog.user": "apphost"
+    }, 
+    {
+        "@timestamp": "2018-08-07T08:27:47.000Z", 
+        "ecs.version": "1.0.0-beta2", 
+        "event.dataset": "slowlog", 
+        "event.module": "mysql", 
+        "input.type": "log", 
+        "log.flags": [
+            "multiline"
+        ], 
+        "log.offset": 1438, 
+        "mysql.slowlog.host": "apphost", 
+        "mysql.slowlog.id": "10999834", 
+        "mysql.slowlog.ip": "1.1.1.1", 
+        "mysql.slowlog.lock_time.sec": "0.000036", 
+        "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');", 
+        "mysql.slowlog.query_time.sec": "10.346539", 
+        "mysql.slowlog.rows_examined": "4751313", 
+        "mysql.slowlog.rows_sent": "0", 
+        "mysql.slowlog.timestamp": "1533630467", 
+        "mysql.slowlog.user": "apphost"
+    }
+]