From e4f228c8a427eff0bf4b7f7f7e34bc160de81d4b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 28 Jan 2020 00:10:28 +0100 Subject: [PATCH] Cherry-pick #15845 to 7.x: SQL module now supports key/value mode (#15870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are some queries like mysql `SHOW STATUS` that display the results in a key/value format so they need to be scanned in a different way, being the key the first column of the response and the value the second one. (cherry picked from commit d8517c6aa7cacb7355e969c4c3b68ccfa31f83e3) Co-authored-by: Rodrigo Villablanca Vásquez --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/sql.asciidoc | 2 +- x-pack/metricbeat/metricbeat.reference.yml | 2 +- x-pack/metricbeat/module/sql/_meta/config.yml | 2 +- .../module/sql/query/_meta/data.json | 364 ++++++++++++++++- .../sql/query/_meta/data_mysql_tables.json | 30 ++ .../sql/query/_meta/data_mysql_variables.json | 382 ++++++++++++++++++ .../module/sql/query/_meta/data_postgres.json | 276 ++++++++++++- .../sql/query/_meta/data_postgres_tables.json | 45 +++ .../query/_meta/data_postgres_variables.json | 277 +++++++++++++ x-pack/metricbeat/module/sql/query/query.go | 92 ++++- .../sql/query/query_integration_test.go | 51 ++- x-pack/metricbeat/modules.d/sql.yml.disabled | 2 +- 13 files changed, 1479 insertions(+), 47 deletions(-) create mode 100644 x-pack/metricbeat/module/sql/query/_meta/data_mysql_tables.json create mode 100644 x-pack/metricbeat/module/sql/query/_meta/data_mysql_variables.json create mode 100644 x-pack/metricbeat/module/sql/query/_meta/data_postgres_tables.json create mode 100644 x-pack/metricbeat/module/sql/query/_meta/data_postgres_variables.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b76c1256e86..f40438ea4d2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -186,6 +186,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Enable script processor. {pull}14711[14711] - Add mixer metricset for Istio Metricbeat module {pull}15696[15696] - Add mesh metricset for Istio Metricbeat module{pull}15535[15535] +- Add `key/value` mode for SQL module. {issue}15770[15770] {pull]15845[15845] - Add support for Unix socket in Memcached metricbeat module. {issue}13685[13685] {pull}15822[15822] - Make the `system/cpu` metricset collect normalized CPU metrics by default. {issue}15618[15618] {pull}15729[15729] diff --git a/metricbeat/docs/modules/sql.asciidoc b/metricbeat/docs/modules/sql.asciidoc index c3f3d412ea7..85a532a326c 100644 --- a/metricbeat/docs/modules/sql.asciidoc +++ b/metricbeat/docs/modules/sql.asciidoc @@ -30,7 +30,7 @@ metricbeat.modules: driver: "postgres" sql_query: "select now()" - + sql_response_format: table ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index e7e28588380..9e94fc905b2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -988,7 +988,7 @@ metricbeat.modules: driver: "postgres" sql_query: "select now()" - + sql_response_format: table #--------------------------------- Stan Module --------------------------------- - module: stan diff --git a/x-pack/metricbeat/module/sql/_meta/config.yml b/x-pack/metricbeat/module/sql/_meta/config.yml index 5c6a419e77b..f980749e7e8 100644 --- a/x-pack/metricbeat/module/sql/_meta/config.yml +++ b/x-pack/metricbeat/module/sql/_meta/config.yml @@ -6,4 +6,4 @@ driver: "postgres" sql_query: "select now()" - + sql_response_format: table diff --git a/x-pack/metricbeat/module/sql/query/_meta/data.json b/x-pack/metricbeat/module/sql/query/_meta/data.json index 799c66fe7bc..485dc03aff2 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data.json @@ -10,21 +10,373 @@ "period": 10000 }, "service": { - "address": "172.22.0.3:3306", + "address": "172.21.0.2:3306", "type": "sql" }, "sql": { "driver": "mysql", "metrics": { "numeric": { - "table_rows": 6 + "aborted_clients": 0, + "aborted_connects": 0, + "binlog_cache_disk_use": 0, + "binlog_cache_use": 0, + "binlog_stmt_cache_disk_use": 0, + "binlog_stmt_cache_use": 0, + "bytes_received": 106, + "bytes_sent": 100, + "com_admin_commands": 1, + "com_alter_db": 0, + "com_alter_db_upgrade": 0, + "com_alter_event": 0, + "com_alter_function": 0, + "com_alter_instance": 0, + "com_alter_procedure": 0, + "com_alter_server": 0, + "com_alter_table": 0, + "com_alter_tablespace": 0, + "com_alter_user": 0, + "com_analyze": 0, + "com_assign_to_keycache": 0, + "com_begin": 0, + "com_binlog": 0, + "com_call_procedure": 0, + "com_change_db": 0, + "com_change_master": 0, + "com_change_repl_filter": 0, + "com_check": 0, + "com_checksum": 0, + "com_commit": 0, + "com_create_db": 0, + "com_create_event": 0, + "com_create_function": 0, + "com_create_index": 0, + "com_create_procedure": 0, + "com_create_server": 0, + "com_create_table": 0, + "com_create_trigger": 0, + "com_create_udf": 0, + "com_create_user": 0, + "com_create_view": 0, + "com_dealloc_sql": 0, + "com_delete": 0, + "com_delete_multi": 0, + "com_do": 0, + "com_drop_db": 0, + "com_drop_event": 0, + "com_drop_function": 0, + "com_drop_index": 0, + "com_drop_procedure": 0, + "com_drop_server": 0, + "com_drop_table": 0, + "com_drop_trigger": 0, + "com_drop_user": 0, + "com_drop_view": 0, + "com_empty_query": 0, + "com_execute_sql": 0, + "com_explain_other": 0, + "com_flush": 0, + "com_get_diagnostics": 0, + "com_grant": 0, + "com_group_replication_start": 0, + "com_group_replication_stop": 0, + "com_ha_close": 0, + "com_ha_open": 0, + "com_ha_read": 0, + "com_help": 0, + "com_insert": 0, + "com_insert_select": 0, + "com_install_plugin": 0, + "com_kill": 0, + "com_load": 0, + "com_lock_tables": 0, + "com_optimize": 0, + "com_preload_keys": 0, + "com_prepare_sql": 0, + "com_purge": 0, + "com_purge_before_date": 0, + "com_release_savepoint": 0, + "com_rename_table": 0, + "com_rename_user": 0, + "com_repair": 0, + "com_replace": 0, + "com_replace_select": 0, + "com_reset": 0, + "com_resignal": 0, + "com_revoke": 0, + "com_revoke_all": 0, + "com_rollback": 0, + "com_rollback_to_savepoint": 0, + "com_savepoint": 0, + "com_select": 0, + "com_set_option": 0, + "com_show_binlog_events": 0, + "com_show_binlogs": 0, + "com_show_charsets": 0, + "com_show_collations": 0, + "com_show_create_db": 0, + "com_show_create_event": 0, + "com_show_create_func": 0, + "com_show_create_proc": 0, + "com_show_create_table": 0, + "com_show_create_trigger": 0, + "com_show_create_user": 0, + "com_show_databases": 0, + "com_show_engine_logs": 0, + "com_show_engine_mutex": 0, + "com_show_engine_status": 0, + "com_show_errors": 0, + "com_show_events": 0, + "com_show_fields": 0, + "com_show_function_code": 0, + "com_show_function_status": 0, + "com_show_grants": 0, + "com_show_keys": 0, + "com_show_master_status": 0, + "com_show_open_tables": 0, + "com_show_plugins": 0, + "com_show_privileges": 0, + "com_show_procedure_code": 0, + "com_show_procedure_status": 0, + "com_show_processlist": 0, + "com_show_profile": 0, + "com_show_profiles": 0, + "com_show_relaylog_events": 0, + "com_show_slave_hosts": 0, + "com_show_slave_status": 0, + "com_show_status": 1, + "com_show_storage_engines": 0, + "com_show_table_status": 0, + "com_show_tables": 0, + "com_show_triggers": 0, + "com_show_variables": 0, + "com_show_warnings": 0, + "com_shutdown": 0, + "com_signal": 0, + "com_slave_start": 0, + "com_slave_stop": 0, + "com_stmt_close": 0, + "com_stmt_execute": 0, + "com_stmt_fetch": 0, + "com_stmt_prepare": 0, + "com_stmt_reprepare": 0, + "com_stmt_reset": 0, + "com_stmt_send_long_data": 0, + "com_truncate": 0, + "com_uninstall_plugin": 0, + "com_unlock_tables": 0, + "com_update": 0, + "com_update_multi": 0, + "com_xa_commit": 0, + "com_xa_end": 0, + "com_xa_prepare": 0, + "com_xa_recover": 0, + "com_xa_rollback": 0, + "com_xa_start": 0, + "connection_errors_accept": 0, + "connection_errors_internal": 0, + "connection_errors_max_connections": 0, + "connection_errors_peer_address": 0, + "connection_errors_select": 0, + "connection_errors_tcpwrap": 0, + "connections": 8, + "created_tmp_disk_tables": 0, + "created_tmp_files": 6, + "created_tmp_tables": 0, + "delayed_errors": 0, + "delayed_insert_threads": 0, + "delayed_writes": 0, + "flush_commands": 1, + "handler_commit": 0, + "handler_delete": 0, + "handler_discover": 0, + "handler_external_lock": 0, + "handler_mrr_init": 0, + "handler_prepare": 0, + "handler_read_first": 0, + "handler_read_key": 0, + "handler_read_last": 0, + "handler_read_next": 0, + "handler_read_prev": 0, + "handler_read_rnd": 0, + "handler_read_rnd_next": 0, + "handler_rollback": 0, + "handler_savepoint": 0, + "handler_savepoint_rollback": 0, + "handler_update": 0, + "handler_write": 0, + "innodb_available_undo_logs": 128, + "innodb_buffer_pool_bytes_data": 8011776, + "innodb_buffer_pool_bytes_dirty": 81920, + "innodb_buffer_pool_pages_data": 489, + "innodb_buffer_pool_pages_dirty": 5, + "innodb_buffer_pool_pages_flushed": 41, + "innodb_buffer_pool_pages_free": 7702, + "innodb_buffer_pool_pages_misc": 0, + "innodb_buffer_pool_pages_total": 8191, + "innodb_buffer_pool_read_ahead": 0, + "innodb_buffer_pool_read_ahead_evicted": 0, + "innodb_buffer_pool_read_ahead_rnd": 0, + "innodb_buffer_pool_read_requests": 2476, + "innodb_buffer_pool_reads": 453, + "innodb_buffer_pool_wait_free": 0, + "innodb_buffer_pool_write_requests": 1009, + "innodb_data_fsyncs": 7, + "innodb_data_pending_fsyncs": 0, + "innodb_data_pending_reads": 0, + "innodb_data_pending_writes": 0, + "innodb_data_read": 7492096, + "innodb_data_reads": 484, + "innodb_data_writes": 58, + "innodb_data_written": 706560, + "innodb_dblwr_pages_written": 2, + "innodb_dblwr_writes": 1, + "innodb_log_waits": 0, + "innodb_log_write_requests": 0, + "innodb_log_writes": 2, + "innodb_num_open_files": 23, + "innodb_os_log_fsyncs": 4, + "innodb_os_log_pending_fsyncs": 0, + "innodb_os_log_pending_writes": 0, + "innodb_os_log_written": 1024, + "innodb_page_size": 16384, + "innodb_pages_created": 37, + "innodb_pages_read": 452, + "innodb_pages_written": 41, + "innodb_row_lock_current_waits": 0, + "innodb_row_lock_time": 0, + "innodb_row_lock_time_avg": 0, + "innodb_row_lock_time_max": 0, + "innodb_row_lock_waits": 0, + "innodb_rows_deleted": 0, + "innodb_rows_inserted": 0, + "innodb_rows_read": 8, + "innodb_rows_updated": 0, + "innodb_truncated_status_writes": 0, + "key_blocks_not_flushed": 0, + "key_blocks_unused": 6693, + "key_blocks_used": 5, + "key_read_requests": 38, + "key_reads": 5, + "key_write_requests": 0, + "key_writes": 0, + "last_query_cost": 0, + "last_query_partial_plans": 0, + "locked_connects": 0, + "max_execution_time_exceeded": 0, + "max_execution_time_set": 0, + "max_execution_time_set_failed": 0, + "max_used_connections": 1, + "not_flushed_delayed_rows": 0, + "ongoing_anonymous_transaction_count": 0, + "open_files": 31, + "open_streams": 0, + "open_table_definitions": 219, + "open_tables": 352, + "opened_files": 384, + "opened_table_definitions": 0, + "opened_tables": 0, + "performance_schema_accounts_lost": 0, + "performance_schema_cond_classes_lost": 0, + "performance_schema_cond_instances_lost": 0, + "performance_schema_digest_lost": 0, + "performance_schema_file_classes_lost": 0, + "performance_schema_file_handles_lost": 0, + "performance_schema_file_instances_lost": 0, + "performance_schema_hosts_lost": 0, + "performance_schema_index_stat_lost": 0, + "performance_schema_locker_lost": 0, + "performance_schema_memory_classes_lost": 0, + "performance_schema_metadata_lock_lost": 0, + "performance_schema_mutex_classes_lost": 0, + "performance_schema_mutex_instances_lost": 0, + "performance_schema_nested_statement_lost": 0, + "performance_schema_prepared_statements_lost": 0, + "performance_schema_program_lost": 0, + "performance_schema_rwlock_classes_lost": 0, + "performance_schema_rwlock_instances_lost": 0, + "performance_schema_session_connect_attrs_lost": 0, + "performance_schema_socket_classes_lost": 0, + "performance_schema_socket_instances_lost": 0, + "performance_schema_stage_classes_lost": 0, + "performance_schema_statement_classes_lost": 0, + "performance_schema_table_handles_lost": 0, + "performance_schema_table_instances_lost": 0, + "performance_schema_table_lock_stat_lost": 0, + "performance_schema_thread_classes_lost": 0, + "performance_schema_thread_instances_lost": 0, + "performance_schema_users_lost": 0, + "prepared_stmt_count": 0, + "qcache_free_blocks": 1, + "qcache_free_memory": 1031832, + "qcache_hits": 0, + "qcache_inserts": 0, + "qcache_lowmem_prunes": 0, + "qcache_not_cached": 4, + "qcache_queries_in_cache": 0, + "qcache_total_blocks": 1, + "queries": 18, + "questions": 1, + "select_full_join": 0, + "select_full_range_join": 0, + "select_range": 0, + "select_range_check": 0, + "select_scan": 0, + "slave_open_temp_tables": 0, + "slow_launch_threads": 0, + "slow_queries": 0, + "sort_merge_passes": 0, + "sort_range": 0, + "sort_rows": 0, + "sort_scan": 0, + "ssl_accept_renegotiates": 0, + "ssl_accepts": 0, + "ssl_callback_cache_hits": 0, + "ssl_client_connects": 0, + "ssl_connect_renegotiates": 0, + "ssl_ctx_verify_depth": 0, + "ssl_ctx_verify_mode": 0, + "ssl_default_timeout": 0, + "ssl_finished_accepts": 0, + "ssl_finished_connects": 0, + "ssl_session_cache_hits": 0, + "ssl_session_cache_misses": 0, + "ssl_session_cache_overflows": 0, + "ssl_session_cache_size": 0, + "ssl_session_cache_timeouts": 0, + "ssl_sessions_reused": 0, + "ssl_used_session_cache_entries": 0, + "ssl_verify_depth": 0, + "ssl_verify_mode": 0, + "table_locks_immediate": 118, + "table_locks_waited": 0, + "table_open_cache_hits": 0, + "table_open_cache_misses": 0, + "table_open_cache_overflows": 0, + "tc_log_max_pages_used": 0, + "tc_log_page_size": 0, + "tc_log_page_waits": 0, + "threads_cached": 0, + "threads_connected": 1, + "threads_created": 1, + "threads_running": 1, + "uptime": 4, + "uptime_since_flush_status": 4 }, "string": { - "engine": "InnoDB", - "table_name": "sys_config", - "table_schema": "sys" + "compression": "OFF", + "innodb_buffer_pool_dump_status": "Dumping of buffer pool not started", + "innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 200126 13:05:12", + "innodb_buffer_pool_resize_status": "", + "max_used_connections_time": "2020-01-26 13:05:13", + "ssl_cipher": "", + "ssl_cipher_list": "", + "ssl_server_not_after": "", + "ssl_server_not_before": "", + "ssl_session_cache_mode": "NONE", + "ssl_version": "" } }, - "query": "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows \u003e 0;" + "query": "show status;" } } \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_mysql_tables.json b/x-pack/metricbeat/module/sql/query/_meta/data_mysql_tables.json new file mode 100644 index 00000000000..50cb2d6eae9 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/_meta/data_mysql_tables.json @@ -0,0 +1,30 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.21.0.2:3306", + "type": "sql" + }, + "sql": { + "driver": "mysql", + "metrics": { + "numeric": { + "table_rows": 1 + }, + "string": { + "engine": "MyISAM", + "table_name": "db", + "table_schema": "mysql" + } + }, + "query": "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows \u003e 0;" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_mysql_variables.json b/x-pack/metricbeat/module/sql/query/_meta/data_mysql_variables.json new file mode 100644 index 00000000000..2ab14d9f16d --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/_meta/data_mysql_variables.json @@ -0,0 +1,382 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.21.0.2:3306", + "type": "sql" + }, + "sql": { + "driver": "mysql", + "metrics": { + "numeric": { + "aborted_clients": 0, + "aborted_connects": 0, + "binlog_cache_disk_use": 0, + "binlog_cache_use": 0, + "binlog_stmt_cache_disk_use": 0, + "binlog_stmt_cache_use": 0, + "bytes_received": 106, + "bytes_sent": 100, + "com_admin_commands": 1, + "com_alter_db": 0, + "com_alter_db_upgrade": 0, + "com_alter_event": 0, + "com_alter_function": 0, + "com_alter_instance": 0, + "com_alter_procedure": 0, + "com_alter_server": 0, + "com_alter_table": 0, + "com_alter_tablespace": 0, + "com_alter_user": 0, + "com_analyze": 0, + "com_assign_to_keycache": 0, + "com_begin": 0, + "com_binlog": 0, + "com_call_procedure": 0, + "com_change_db": 0, + "com_change_master": 0, + "com_change_repl_filter": 0, + "com_check": 0, + "com_checksum": 0, + "com_commit": 0, + "com_create_db": 0, + "com_create_event": 0, + "com_create_function": 0, + "com_create_index": 0, + "com_create_procedure": 0, + "com_create_server": 0, + "com_create_table": 0, + "com_create_trigger": 0, + "com_create_udf": 0, + "com_create_user": 0, + "com_create_view": 0, + "com_dealloc_sql": 0, + "com_delete": 0, + "com_delete_multi": 0, + "com_do": 0, + "com_drop_db": 0, + "com_drop_event": 0, + "com_drop_function": 0, + "com_drop_index": 0, + "com_drop_procedure": 0, + "com_drop_server": 0, + "com_drop_table": 0, + "com_drop_trigger": 0, + "com_drop_user": 0, + "com_drop_view": 0, + "com_empty_query": 0, + "com_execute_sql": 0, + "com_explain_other": 0, + "com_flush": 0, + "com_get_diagnostics": 0, + "com_grant": 0, + "com_group_replication_start": 0, + "com_group_replication_stop": 0, + "com_ha_close": 0, + "com_ha_open": 0, + "com_ha_read": 0, + "com_help": 0, + "com_insert": 0, + "com_insert_select": 0, + "com_install_plugin": 0, + "com_kill": 0, + "com_load": 0, + "com_lock_tables": 0, + "com_optimize": 0, + "com_preload_keys": 0, + "com_prepare_sql": 0, + "com_purge": 0, + "com_purge_before_date": 0, + "com_release_savepoint": 0, + "com_rename_table": 0, + "com_rename_user": 0, + "com_repair": 0, + "com_replace": 0, + "com_replace_select": 0, + "com_reset": 0, + "com_resignal": 0, + "com_revoke": 0, + "com_revoke_all": 0, + "com_rollback": 0, + "com_rollback_to_savepoint": 0, + "com_savepoint": 0, + "com_select": 0, + "com_set_option": 0, + "com_show_binlog_events": 0, + "com_show_binlogs": 0, + "com_show_charsets": 0, + "com_show_collations": 0, + "com_show_create_db": 0, + "com_show_create_event": 0, + "com_show_create_func": 0, + "com_show_create_proc": 0, + "com_show_create_table": 0, + "com_show_create_trigger": 0, + "com_show_create_user": 0, + "com_show_databases": 0, + "com_show_engine_logs": 0, + "com_show_engine_mutex": 0, + "com_show_engine_status": 0, + "com_show_errors": 0, + "com_show_events": 0, + "com_show_fields": 0, + "com_show_function_code": 0, + "com_show_function_status": 0, + "com_show_grants": 0, + "com_show_keys": 0, + "com_show_master_status": 0, + "com_show_open_tables": 0, + "com_show_plugins": 0, + "com_show_privileges": 0, + "com_show_procedure_code": 0, + "com_show_procedure_status": 0, + "com_show_processlist": 0, + "com_show_profile": 0, + "com_show_profiles": 0, + "com_show_relaylog_events": 0, + "com_show_slave_hosts": 0, + "com_show_slave_status": 0, + "com_show_status": 1, + "com_show_storage_engines": 0, + "com_show_table_status": 0, + "com_show_tables": 0, + "com_show_triggers": 0, + "com_show_variables": 0, + "com_show_warnings": 0, + "com_shutdown": 0, + "com_signal": 0, + "com_slave_start": 0, + "com_slave_stop": 0, + "com_stmt_close": 0, + "com_stmt_execute": 0, + "com_stmt_fetch": 0, + "com_stmt_prepare": 0, + "com_stmt_reprepare": 0, + "com_stmt_reset": 0, + "com_stmt_send_long_data": 0, + "com_truncate": 0, + "com_uninstall_plugin": 0, + "com_unlock_tables": 0, + "com_update": 0, + "com_update_multi": 0, + "com_xa_commit": 0, + "com_xa_end": 0, + "com_xa_prepare": 0, + "com_xa_recover": 0, + "com_xa_rollback": 0, + "com_xa_start": 0, + "connection_errors_accept": 0, + "connection_errors_internal": 0, + "connection_errors_max_connections": 0, + "connection_errors_peer_address": 0, + "connection_errors_select": 0, + "connection_errors_tcpwrap": 0, + "connections": 29, + "created_tmp_disk_tables": 0, + "created_tmp_files": 6, + "created_tmp_tables": 0, + "delayed_errors": 0, + "delayed_insert_threads": 0, + "delayed_writes": 0, + "flush_commands": 1, + "handler_commit": 0, + "handler_delete": 0, + "handler_discover": 0, + "handler_external_lock": 0, + "handler_mrr_init": 0, + "handler_prepare": 0, + "handler_read_first": 0, + "handler_read_key": 0, + "handler_read_last": 0, + "handler_read_next": 0, + "handler_read_prev": 0, + "handler_read_rnd": 0, + "handler_read_rnd_next": 0, + "handler_rollback": 0, + "handler_savepoint": 0, + "handler_savepoint_rollback": 0, + "handler_update": 0, + "handler_write": 0, + "innodb_available_undo_logs": 128, + "innodb_buffer_pool_bytes_data": 5652480, + "innodb_buffer_pool_bytes_dirty": 81920, + "innodb_buffer_pool_pages_data": 345, + "innodb_buffer_pool_pages_dirty": 5, + "innodb_buffer_pool_pages_flushed": 44, + "innodb_buffer_pool_pages_free": 7846, + "innodb_buffer_pool_pages_misc": 0, + "innodb_buffer_pool_pages_total": 8191, + "innodb_buffer_pool_read_ahead": 0, + "innodb_buffer_pool_read_ahead_evicted": 0, + "innodb_buffer_pool_read_ahead_rnd": 0, + "innodb_buffer_pool_read_requests": 2785, + "innodb_buffer_pool_reads": 309, + "innodb_buffer_pool_wait_free": 0, + "innodb_buffer_pool_write_requests": 1351, + "innodb_data_fsyncs": 7, + "innodb_data_pending_fsyncs": 0, + "innodb_data_pending_reads": 0, + "innodb_data_pending_writes": 0, + "innodb_data_read": 5132800, + "innodb_data_reads": 340, + "innodb_data_writes": 61, + "innodb_data_written": 755712, + "innodb_dblwr_pages_written": 2, + "innodb_dblwr_writes": 1, + "innodb_log_waits": 0, + "innodb_log_write_requests": 0, + "innodb_log_writes": 2, + "innodb_num_open_files": 23, + "innodb_os_log_fsyncs": 4, + "innodb_os_log_pending_fsyncs": 0, + "innodb_os_log_pending_writes": 0, + "innodb_os_log_written": 1024, + "innodb_page_size": 16384, + "innodb_pages_created": 37, + "innodb_pages_read": 308, + "innodb_pages_written": 44, + "innodb_row_lock_current_waits": 0, + "innodb_row_lock_time": 0, + "innodb_row_lock_time_avg": 0, + "innodb_row_lock_time_max": 0, + "innodb_row_lock_waits": 0, + "innodb_rows_deleted": 0, + "innodb_rows_inserted": 0, + "innodb_rows_read": 8, + "innodb_rows_updated": 0, + "innodb_truncated_status_writes": 0, + "key_blocks_not_flushed": 0, + "key_blocks_unused": 6693, + "key_blocks_used": 5, + "key_read_requests": 54, + "key_reads": 5, + "key_write_requests": 0, + "key_writes": 0, + "last_query_cost": 0, + "last_query_partial_plans": 0, + "locked_connects": 0, + "max_execution_time_exceeded": 0, + "max_execution_time_set": 0, + "max_execution_time_set_failed": 0, + "max_used_connections": 1, + "not_flushed_delayed_rows": 0, + "ongoing_anonymous_transaction_count": 0, + "open_files": 38, + "open_streams": 0, + "open_table_definitions": 219, + "open_tables": 485, + "opened_files": 399, + "opened_table_definitions": 0, + "opened_tables": 0, + "performance_schema_accounts_lost": 0, + "performance_schema_cond_classes_lost": 0, + "performance_schema_cond_instances_lost": 0, + "performance_schema_digest_lost": 0, + "performance_schema_file_classes_lost": 0, + "performance_schema_file_handles_lost": 0, + "performance_schema_file_instances_lost": 0, + "performance_schema_hosts_lost": 0, + "performance_schema_index_stat_lost": 0, + "performance_schema_locker_lost": 0, + "performance_schema_memory_classes_lost": 0, + "performance_schema_metadata_lock_lost": 0, + "performance_schema_mutex_classes_lost": 0, + "performance_schema_mutex_instances_lost": 0, + "performance_schema_nested_statement_lost": 0, + "performance_schema_prepared_statements_lost": 0, + "performance_schema_program_lost": 0, + "performance_schema_rwlock_classes_lost": 0, + "performance_schema_rwlock_instances_lost": 0, + "performance_schema_session_connect_attrs_lost": 0, + "performance_schema_socket_classes_lost": 0, + "performance_schema_socket_instances_lost": 0, + "performance_schema_stage_classes_lost": 0, + "performance_schema_statement_classes_lost": 0, + "performance_schema_table_handles_lost": 0, + "performance_schema_table_instances_lost": 0, + "performance_schema_table_lock_stat_lost": 0, + "performance_schema_thread_classes_lost": 0, + "performance_schema_thread_instances_lost": 0, + "performance_schema_users_lost": 0, + "prepared_stmt_count": 0, + "qcache_free_blocks": 1, + "qcache_free_memory": 1031832, + "qcache_hits": 0, + "qcache_inserts": 0, + "qcache_lowmem_prunes": 0, + "qcache_not_cached": 24, + "qcache_queries_in_cache": 0, + "qcache_total_blocks": 1, + "queries": 81, + "questions": 1, + "select_full_join": 0, + "select_full_range_join": 0, + "select_range": 0, + "select_range_check": 0, + "select_scan": 0, + "slave_open_temp_tables": 0, + "slow_launch_threads": 0, + "slow_queries": 0, + "sort_merge_passes": 0, + "sort_range": 0, + "sort_rows": 0, + "sort_scan": 0, + "ssl_accept_renegotiates": 0, + "ssl_accepts": 0, + "ssl_callback_cache_hits": 0, + "ssl_client_connects": 0, + "ssl_connect_renegotiates": 0, + "ssl_ctx_verify_depth": 0, + "ssl_ctx_verify_mode": 0, + "ssl_default_timeout": 0, + "ssl_finished_accepts": 0, + "ssl_finished_connects": 0, + "ssl_session_cache_hits": 0, + "ssl_session_cache_misses": 0, + "ssl_session_cache_overflows": 0, + "ssl_session_cache_size": 0, + "ssl_session_cache_timeouts": 0, + "ssl_sessions_reused": 0, + "ssl_used_session_cache_entries": 0, + "ssl_verify_depth": 0, + "ssl_verify_mode": 0, + "table_locks_immediate": 146, + "table_locks_waited": 0, + "table_open_cache_hits": 0, + "table_open_cache_misses": 0, + "table_open_cache_overflows": 0, + "tc_log_max_pages_used": 0, + "tc_log_page_size": 0, + "tc_log_page_waits": 0, + "threads_cached": 0, + "threads_connected": 1, + "threads_created": 1, + "threads_running": 1, + "uptime": 30, + "uptime_since_flush_status": 30 + }, + "string": { + "compression": "OFF", + "innodb_buffer_pool_dump_status": "Dumping of buffer pool not started", + "innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 200127 12:35:03", + "innodb_buffer_pool_resize_status": "", + "max_used_connections_time": "2020-01-27 12:35:04", + "ssl_cipher": "", + "ssl_cipher_list": "", + "ssl_server_not_after": "", + "ssl_server_not_before": "", + "ssl_session_cache_mode": "NONE", + "ssl_version": "" + } + }, + "query": "show status;" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json index 273c434c382..eea20f9bfb2 100644 --- a/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres.json @@ -10,36 +10,268 @@ "period": 10000 }, "service": { - "address": "192.168.0.3:5432", + "address": "172.21.0.3:5432", "type": "sql" }, "sql": { "driver": "postgres", "metrics": { "numeric": { - "blk_read_time": 0, - "blk_write_time": 0, - "blks_hit": 948, - "blks_read": 96, - "conflicts": 0, - "datid": 12379, - "deadlocks": 0, - "numbackends": 1, - "temp_bytes": 0, - "temp_files": 0, - "tup_deleted": 0, - "tup_fetched": 616, - "tup_inserted": 0, - "tup_returned": 717, - "tup_updated": 0, - "xact_commit": 3, - "xact_rollback": 0 + "archive_timeout": 0, + "authentication_timeout": 60, + "autovacuum_analyze_scale_factor": 0.1, + "autovacuum_analyze_threshold": 50, + "autovacuum_freeze_max_age": 200000000, + "autovacuum_max_workers": 3, + "autovacuum_multixact_freeze_max_age": 400000000, + "autovacuum_naptime": 60, + "autovacuum_vacuum_cost_delay": 20, + "autovacuum_vacuum_cost_limit": -1, + "autovacuum_vacuum_scale_factor": 0.2, + "autovacuum_vacuum_threshold": 50, + "autovacuum_work_mem": -1, + "bgwriter_delay": 200, + "bgwriter_lru_maxpages": 100, + "bgwriter_lru_multiplier": 2, + "block_size": 8192, + "checkpoint_completion_target": 0.5, + "checkpoint_timeout": 300, + "checkpoint_warning": 30, + "commit_delay": 0, + "commit_siblings": 5, + "cpu_index_tuple_cost": 0.005, + "cpu_operator_cost": 0.0025, + "cpu_tuple_cost": 0.01, + "cursor_tuple_fraction": 0.1, + "deadlock_timeout": 1000, + "default_statistics_target": 100, + "effective_cache_size": 524288, + "effective_io_concurrency": 1, + "extra_float_digits": 2, + "from_collapse_limit": 8, + "geqo_effort": 5, + "geqo_generations": 0, + "geqo_pool_size": 0, + "geqo_seed": 0, + "geqo_selection_bias": 2, + "geqo_threshold": 12, + "gin_fuzzy_search_limit": 0, + "gin_pending_list_limit": 4096, + "join_collapse_limit": 8, + "lock_timeout": 0, + "log_autovacuum_min_duration": -1, + "log_file_mode": 600, + "log_min_duration_statement": -1, + "log_rotation_age": 1440, + "log_rotation_size": 10240, + "log_temp_files": -1, + "maintenance_work_mem": 65536, + "max_connections": 100, + "max_files_per_process": 1000, + "max_function_args": 100, + "max_identifier_length": 63, + "max_index_keys": 32, + "max_locks_per_transaction": 64, + "max_pred_locks_per_transaction": 64, + "max_prepared_transactions": 0, + "max_replication_slots": 0, + "max_stack_depth": 2048, + "max_standby_archive_delay": 30000, + "max_standby_streaming_delay": 30000, + "max_wal_senders": 0, + "max_wal_size": 64, + "max_worker_processes": 8, + "min_wal_size": 5, + "pg_stat_statements.max": 10000, + "port": 5432, + "post_auth_delay": 0, + "pre_auth_delay": 0, + "random_page_cost": 4, + "segment_size": 131072, + "seq_page_cost": 1, + "server_version_num": 90503, + "shared_buffers": 16384, + "statement_timeout": 0, + "superuser_reserved_connections": 3, + "tcp_keepalives_count": 9, + "tcp_keepalives_idle": 7200, + "tcp_keepalives_interval": 75, + "temp_buffers": 1024, + "temp_file_limit": -1, + "track_activity_query_size": 1024, + "unix_socket_permissions": 777, + "vacuum_cost_delay": 0, + "vacuum_cost_limit": 200, + "vacuum_cost_page_dirty": 20, + "vacuum_cost_page_hit": 1, + "vacuum_cost_page_miss": 10, + "vacuum_defer_cleanup_age": 0, + "vacuum_freeze_min_age": 50000000, + "vacuum_freeze_table_age": 150000000, + "vacuum_multixact_freeze_min_age": 5000000, + "vacuum_multixact_freeze_table_age": 150000000, + "wal_block_size": 8192, + "wal_buffers": 512, + "wal_keep_segments": 0, + "wal_receiver_status_interval": 10, + "wal_receiver_timeout": 60000, + "wal_retrieve_retry_interval": 5000, + "wal_segment_size": 2048, + "wal_sender_timeout": 60000, + "wal_writer_delay": 200, + "work_mem": 4096 }, "string": { - "datname": "postgres", - "stats_reset": "2020-01-25T18:21:44.869405Z" + "allow_system_table_mods": "off", + "application_name": "", + "archive_command": "(disabled)", + "archive_mode": "off", + "array_nulls": "on", + "autovacuum": "on", + "backslash_quote": "safe_encoding", + "bonjour": "off", + "bonjour_name": "", + "bytea_output": "hex", + "check_function_bodies": "on", + "client_encoding": "UTF8", + "client_min_messages": "notice", + "cluster_name": "", + "config_file": "/var/lib/postgresql/data/postgresql.conf", + "constraint_exclusion": "partition", + "data_checksums": "off", + "data_directory": "/var/lib/postgresql/data", + "datestyle": "ISO, MDY", + "db_user_namespace": "off", + "debug_assertions": "off", + "debug_pretty_print": "on", + "debug_print_parse": "off", + "debug_print_plan": "off", + "debug_print_rewritten": "off", + "default_tablespace": "", + "default_text_search_config": "pg_catalog.english", + "default_transaction_deferrable": "off", + "default_transaction_isolation": "read committed", + "default_transaction_read_only": "off", + "default_with_oids": "off", + "dynamic_library_path": "$libdir", + "dynamic_shared_memory_type": "posix", + "enable_bitmapscan": "on", + "enable_hashagg": "on", + "enable_hashjoin": "on", + "enable_indexonlyscan": "on", + "enable_indexscan": "on", + "enable_material": "on", + "enable_mergejoin": "on", + "enable_nestloop": "on", + "enable_seqscan": "on", + "enable_sort": "on", + "enable_tidscan": "on", + "escape_string_warning": "on", + "event_source": "PostgreSQL", + "exit_on_error": "off", + "external_pid_file": "", + "fsync": "on", + "full_page_writes": "on", + "geqo": "on", + "hba_file": "/var/lib/postgresql/data/pg_hba.conf", + "hot_standby": "off", + "hot_standby_feedback": "off", + "huge_pages": "try", + "ident_file": "/var/lib/postgresql/data/pg_ident.conf", + "ignore_checksum_failure": "off", + "ignore_system_indexes": "off", + "integer_datetimes": "on", + "intervalstyle": "postgres", + "krb_caseins_users": "off", + "krb_server_keyfile": "FILE:/etc/postgresql-common/krb5.keytab", + "lc_collate": "en_US.utf8", + "lc_ctype": "en_US.utf8", + "lc_messages": "en_US.utf8", + "lc_monetary": "en_US.utf8", + "lc_numeric": "en_US.utf8", + "lc_time": "en_US.utf8", + "listen_addresses": "*", + "lo_compat_privileges": "off", + "local_preload_libraries": "", + "log_checkpoints": "off", + "log_connections": "off", + "log_destination": "stderr", + "log_directory": "pg_log", + "log_disconnections": "off", + "log_duration": "off", + "log_error_verbosity": "default", + "log_executor_stats": "off", + "log_filename": "postgresql-%Y-%m-%d_%H%M%S.log", + "log_hostname": "off", + "log_line_prefix": "", + "log_lock_waits": "off", + "log_min_error_statement": "error", + "log_min_messages": "warning", + "log_parser_stats": "off", + "log_planner_stats": "off", + "log_replication_commands": "off", + "log_statement": "none", + "log_statement_stats": "off", + "log_timezone": "UTC", + "log_truncate_on_rotation": "off", + "logging_collector": "off", + "operator_precedence_warning": "off", + "password_encryption": "on", + "pg_stat_statements.save": "on", + "pg_stat_statements.track": "all", + "pg_stat_statements.track_utility": "on", + "quote_all_identifiers": "off", + "restart_after_crash": "on", + "row_security": "on", + "search_path": "\"$user\", public", + "server_encoding": "UTF8", + "server_version": "9.5.3", + "session_preload_libraries": "", + "session_replication_role": "origin", + "shared_preload_libraries": "pg_stat_statements", + "sql_inheritance": "on", + "ssl": "off", + "ssl_ca_file": "", + "ssl_cert_file": "server.crt", + "ssl_ciphers": "HIGH:MEDIUM:+3DES:!aNULL", + "ssl_crl_file": "", + "ssl_ecdh_curve": "prime256v1", + "ssl_key_file": "server.key", + "ssl_prefer_server_ciphers": "on", + "standard_conforming_strings": "on", + "stats_temp_directory": "pg_stat_tmp", + "synchronize_seqscans": "on", + "synchronous_commit": "on", + "synchronous_standby_names": "", + "syslog_facility": "local0", + "syslog_ident": "postgres", + "temp_tablespaces": "", + "timezone": "UTC", + "timezone_abbreviations": "Default", + "trace_notify": "off", + "trace_recovery_messages": "log", + "trace_sort": "off", + "track_activities": "on", + "track_commit_timestamp": "off", + "track_counts": "on", + "track_functions": "none", + "track_io_timing": "off", + "transaction_deferrable": "off", + "transaction_isolation": "read committed", + "transaction_read_only": "off", + "transform_null_equals": "off", + "unix_socket_directories": "/var/run/postgresql", + "unix_socket_group": "", + "update_process_title": "on", + "wal_compression": "off", + "wal_level": "minimal", + "wal_log_hints": "off", + "wal_sync_method": "fdatasync", + "xmlbinary": "base64", + "xmloption": "content", + "zero_damaged_pages": "off" } }, - "query": "select * from pg_stat_database where datname='postgres'" + "query": "select name, setting from pg_settings" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres_tables.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres_tables.json new file mode 100644 index 00000000000..6fa50540bd4 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres_tables.json @@ -0,0 +1,45 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.21.0.3:5432", + "type": "sql" + }, + "sql": { + "driver": "postgres", + "metrics": { + "numeric": { + "blk_read_time": 0, + "blk_write_time": 0, + "blks_hit": 105648, + "blks_read": 240, + "conflicts": 0, + "datid": 12379, + "deadlocks": 0, + "numbackends": 1, + "temp_bytes": 0, + "temp_files": 0, + "tup_deleted": 0, + "tup_fetched": 71845, + "tup_inserted": 0, + "tup_returned": 150847, + "tup_updated": 0, + "xact_commit": 1157, + "xact_rollback": 0 + }, + "string": { + "datname": "postgres", + "stats_reset": "2020-01-27T02:45:53.51247Z" + } + }, + "query": "select * from pg_stat_database where datname='postgres'" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/_meta/data_postgres_variables.json b/x-pack/metricbeat/module/sql/query/_meta/data_postgres_variables.json new file mode 100644 index 00000000000..76fe62ff546 --- /dev/null +++ b/x-pack/metricbeat/module/sql/query/_meta/data_postgres_variables.json @@ -0,0 +1,277 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "sql.query", + "duration": 115000, + "module": "sql" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "service": { + "address": "172.21.0.3:5432", + "type": "sql" + }, + "sql": { + "driver": "postgres", + "metrics": { + "numeric": { + "archive_timeout": 0, + "authentication_timeout": 60, + "autovacuum_analyze_scale_factor": 0.1, + "autovacuum_analyze_threshold": 50, + "autovacuum_freeze_max_age": 200000000, + "autovacuum_max_workers": 3, + "autovacuum_multixact_freeze_max_age": 400000000, + "autovacuum_naptime": 60, + "autovacuum_vacuum_cost_delay": 20, + "autovacuum_vacuum_cost_limit": -1, + "autovacuum_vacuum_scale_factor": 0.2, + "autovacuum_vacuum_threshold": 50, + "autovacuum_work_mem": -1, + "bgwriter_delay": 200, + "bgwriter_lru_maxpages": 100, + "bgwriter_lru_multiplier": 2, + "block_size": 8192, + "checkpoint_completion_target": 0.5, + "checkpoint_timeout": 300, + "checkpoint_warning": 30, + "commit_delay": 0, + "commit_siblings": 5, + "cpu_index_tuple_cost": 0.005, + "cpu_operator_cost": 0.0025, + "cpu_tuple_cost": 0.01, + "cursor_tuple_fraction": 0.1, + "deadlock_timeout": 1000, + "default_statistics_target": 100, + "effective_cache_size": 524288, + "effective_io_concurrency": 1, + "extra_float_digits": 2, + "from_collapse_limit": 8, + "geqo_effort": 5, + "geqo_generations": 0, + "geqo_pool_size": 0, + "geqo_seed": 0, + "geqo_selection_bias": 2, + "geqo_threshold": 12, + "gin_fuzzy_search_limit": 0, + "gin_pending_list_limit": 4096, + "join_collapse_limit": 8, + "lock_timeout": 0, + "log_autovacuum_min_duration": -1, + "log_file_mode": 600, + "log_min_duration_statement": -1, + "log_rotation_age": 1440, + "log_rotation_size": 10240, + "log_temp_files": -1, + "maintenance_work_mem": 65536, + "max_connections": 100, + "max_files_per_process": 1000, + "max_function_args": 100, + "max_identifier_length": 63, + "max_index_keys": 32, + "max_locks_per_transaction": 64, + "max_pred_locks_per_transaction": 64, + "max_prepared_transactions": 0, + "max_replication_slots": 0, + "max_stack_depth": 2048, + "max_standby_archive_delay": 30000, + "max_standby_streaming_delay": 30000, + "max_wal_senders": 0, + "max_wal_size": 64, + "max_worker_processes": 8, + "min_wal_size": 5, + "pg_stat_statements.max": 10000, + "port": 5432, + "post_auth_delay": 0, + "pre_auth_delay": 0, + "random_page_cost": 4, + "segment_size": 131072, + "seq_page_cost": 1, + "server_version_num": 90503, + "shared_buffers": 16384, + "statement_timeout": 0, + "superuser_reserved_connections": 3, + "tcp_keepalives_count": 9, + "tcp_keepalives_idle": 7200, + "tcp_keepalives_interval": 75, + "temp_buffers": 1024, + "temp_file_limit": -1, + "track_activity_query_size": 1024, + "unix_socket_permissions": 777, + "vacuum_cost_delay": 0, + "vacuum_cost_limit": 200, + "vacuum_cost_page_dirty": 20, + "vacuum_cost_page_hit": 1, + "vacuum_cost_page_miss": 10, + "vacuum_defer_cleanup_age": 0, + "vacuum_freeze_min_age": 50000000, + "vacuum_freeze_table_age": 150000000, + "vacuum_multixact_freeze_min_age": 5000000, + "vacuum_multixact_freeze_table_age": 150000000, + "wal_block_size": 8192, + "wal_buffers": 512, + "wal_keep_segments": 0, + "wal_receiver_status_interval": 10, + "wal_receiver_timeout": 60000, + "wal_retrieve_retry_interval": 5000, + "wal_segment_size": 2048, + "wal_sender_timeout": 60000, + "wal_writer_delay": 200, + "work_mem": 4096 + }, + "string": { + "allow_system_table_mods": "off", + "application_name": "", + "archive_command": "(disabled)", + "archive_mode": "off", + "array_nulls": "on", + "autovacuum": "on", + "backslash_quote": "safe_encoding", + "bonjour": "off", + "bonjour_name": "", + "bytea_output": "hex", + "check_function_bodies": "on", + "client_encoding": "UTF8", + "client_min_messages": "notice", + "cluster_name": "", + "config_file": "/var/lib/postgresql/data/postgresql.conf", + "constraint_exclusion": "partition", + "data_checksums": "off", + "data_directory": "/var/lib/postgresql/data", + "datestyle": "ISO, MDY", + "db_user_namespace": "off", + "debug_assertions": "off", + "debug_pretty_print": "on", + "debug_print_parse": "off", + "debug_print_plan": "off", + "debug_print_rewritten": "off", + "default_tablespace": "", + "default_text_search_config": "pg_catalog.english", + "default_transaction_deferrable": "off", + "default_transaction_isolation": "read committed", + "default_transaction_read_only": "off", + "default_with_oids": "off", + "dynamic_library_path": "$libdir", + "dynamic_shared_memory_type": "posix", + "enable_bitmapscan": "on", + "enable_hashagg": "on", + "enable_hashjoin": "on", + "enable_indexonlyscan": "on", + "enable_indexscan": "on", + "enable_material": "on", + "enable_mergejoin": "on", + "enable_nestloop": "on", + "enable_seqscan": "on", + "enable_sort": "on", + "enable_tidscan": "on", + "escape_string_warning": "on", + "event_source": "PostgreSQL", + "exit_on_error": "off", + "external_pid_file": "", + "fsync": "on", + "full_page_writes": "on", + "geqo": "on", + "hba_file": "/var/lib/postgresql/data/pg_hba.conf", + "hot_standby": "off", + "hot_standby_feedback": "off", + "huge_pages": "try", + "ident_file": "/var/lib/postgresql/data/pg_ident.conf", + "ignore_checksum_failure": "off", + "ignore_system_indexes": "off", + "integer_datetimes": "on", + "intervalstyle": "postgres", + "krb_caseins_users": "off", + "krb_server_keyfile": "FILE:/etc/postgresql-common/krb5.keytab", + "lc_collate": "en_US.utf8", + "lc_ctype": "en_US.utf8", + "lc_messages": "en_US.utf8", + "lc_monetary": "en_US.utf8", + "lc_numeric": "en_US.utf8", + "lc_time": "en_US.utf8", + "listen_addresses": "*", + "lo_compat_privileges": "off", + "local_preload_libraries": "", + "log_checkpoints": "off", + "log_connections": "off", + "log_destination": "stderr", + "log_directory": "pg_log", + "log_disconnections": "off", + "log_duration": "off", + "log_error_verbosity": "default", + "log_executor_stats": "off", + "log_filename": "postgresql-%Y-%m-%d_%H%M%S.log", + "log_hostname": "off", + "log_line_prefix": "", + "log_lock_waits": "off", + "log_min_error_statement": "error", + "log_min_messages": "warning", + "log_parser_stats": "off", + "log_planner_stats": "off", + "log_replication_commands": "off", + "log_statement": "none", + "log_statement_stats": "off", + "log_timezone": "UTC", + "log_truncate_on_rotation": "off", + "logging_collector": "off", + "operator_precedence_warning": "off", + "password_encryption": "on", + "pg_stat_statements.save": "on", + "pg_stat_statements.track": "all", + "pg_stat_statements.track_utility": "on", + "quote_all_identifiers": "off", + "restart_after_crash": "on", + "row_security": "on", + "search_path": "\"$user\", public", + "server_encoding": "UTF8", + "server_version": "9.5.3", + "session_preload_libraries": "", + "session_replication_role": "origin", + "shared_preload_libraries": "pg_stat_statements", + "sql_inheritance": "on", + "ssl": "off", + "ssl_ca_file": "", + "ssl_cert_file": "server.crt", + "ssl_ciphers": "HIGH:MEDIUM:+3DES:!aNULL", + "ssl_crl_file": "", + "ssl_ecdh_curve": "prime256v1", + "ssl_key_file": "server.key", + "ssl_prefer_server_ciphers": "on", + "standard_conforming_strings": "on", + "stats_temp_directory": "pg_stat_tmp", + "synchronize_seqscans": "on", + "synchronous_commit": "on", + "synchronous_standby_names": "", + "syslog_facility": "local0", + "syslog_ident": "postgres", + "temp_tablespaces": "", + "timezone": "UTC", + "timezone_abbreviations": "Default", + "trace_notify": "off", + "trace_recovery_messages": "log", + "trace_sort": "off", + "track_activities": "on", + "track_commit_timestamp": "off", + "track_counts": "on", + "track_functions": "none", + "track_io_timing": "off", + "transaction_deferrable": "off", + "transaction_isolation": "read committed", + "transaction_read_only": "off", + "transform_null_equals": "off", + "unix_socket_directories": "/var/run/postgresql", + "unix_socket_group": "", + "update_process_title": "on", + "wal_compression": "off", + "wal_level": "minimal", + "wal_log_hints": "off", + "wal_sync_method": "fdatasync", + "xmlbinary": "base64", + "xmloption": "content", + "zero_damaged_pages": "off" + } + }, + "query": "select name, setting from pg_settings" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/sql/query/query.go b/x-pack/metricbeat/module/sql/query/query.go index 07c9c9350b2..7a0c66de9da 100644 --- a/x-pack/metricbeat/module/sql/query/query.go +++ b/x-pack/metricbeat/module/sql/query/query.go @@ -18,6 +18,12 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) +// represents the response format of the query +const ( + tableResponseFormat = "table" + variableResponseFormat = "variables" +) + // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of // the MetricSet for each host defined in the module's configuration. After the @@ -34,8 +40,9 @@ func init() { // interface methods except for Fetch. type MetricSet struct { mb.BaseMetricSet - Driver string - Query string + Driver string + Query string + ResponseFormat string } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -44,24 +51,32 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Beta("The sql query metricset is beta.") config := struct { - Driver string `config:"driver" validate:"nonzero,required"` - Query string `config:"sql_query" validate:"nonzero,required"` - }{} + Driver string `config:"driver" validate:"nonzero,required"` + Query string `config:"sql_query" validate:"nonzero,required"` + ResponseFormat string `config:"sql_response_format"` + }{ResponseFormat: tableResponseFormat} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } + if config.ResponseFormat != variableResponseFormat && config.ResponseFormat != tableResponseFormat { + return nil, fmt.Errorf("invalid sql_response_format value: %s", config.ResponseFormat) + } + return &MetricSet{ - BaseMetricSet: base, - Driver: config.Driver, - Query: config.Query, + BaseMetricSet: base, + Driver: config.Driver, + Query: config.Query, + ResponseFormat: config.ResponseFormat, }, nil } // Fetch methods implements the data gathering and data conversion to the right // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). +// It calls m.fetchTableMode() or m.fetchVariableMode() depending on the response +// format of the query. func (m *MetricSet) Fetch(report mb.ReporterV2) error { db, err := sqlx.Open(m.Driver, m.HostData().URI) if err != nil { @@ -80,6 +95,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } defer rows.Close() + if m.ResponseFormat == tableResponseFormat { + return m.fetchTableMode(rows, report) + } + + return m.fetchVariableMode(rows, report) +} + +// fetchTableMode scan the rows and publishes the event for querys that return the response in a table format. +func (m *MetricSet) fetchTableMode(rows *sqlx.Rows, report mb.ReporterV2) error { + // Extracted from // https://stackoverflow.com/questions/23507531/is-golangs-sql-package-incapable-of-ad-hoc-exploratory-queries/23507765#23507765 cols, err := rows.Columns() @@ -131,10 +156,59 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { }) } - if rows.Err() != nil { + if err = rows.Err(); err != nil { + m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) + } + + return nil +} + +// fetchVariableMode scan the rows and publishes the event for querys that return the response in a key/value format. +func (m *MetricSet) fetchVariableMode(rows *sqlx.Rows, report mb.ReporterV2) error { + data := common.MapStr{} + for rows.Next() { + var key string + var val interface{} + err := rows.Scan(&key, &val) + if err != nil { + m.Logger().Debug(errors.Wrap(err, "error trying to scan rows")) + continue + } + + key = strings.ToLower(key) + data[key] = val + } + + if err := rows.Err(); err != nil { m.Logger().Debug(errors.Wrap(err, "error trying to read rows")) } + numericMetrics := common.MapStr{} + stringMetrics := common.MapStr{} + + for key, value := range data { + value := getValue(&value) + num, err := strconv.ParseFloat(value, 64) + if err == nil { + numericMetrics[key] = num + } else { + stringMetrics[key] = value + } + } + + report.Event(mb.Event{ + RootFields: common.MapStr{ + "sql": common.MapStr{ + "driver": m.Driver, + "query": m.Query, + "metrics": common.MapStr{ + "numeric": numericMetrics, + "string": stringMetrics, + }, + }, + }, + }) + return nil } diff --git a/x-pack/metricbeat/module/sql/query/query_integration_test.go b/x-pack/metricbeat/module/sql/query/query_integration_test.go index 6e23be8ce9a..9d75c59943c 100644 --- a/x-pack/metricbeat/module/sql/query/query_integration_test.go +++ b/x-pack/metricbeat/module/sql/query/query_integration_test.go @@ -27,9 +27,10 @@ import ( ) type testFetchConfig struct { - Driver string - Query string - Host string + Driver string + Query string + Host string + ResponseFormat string Assertion func(t *testing.T, event beat.Event) } @@ -48,7 +49,23 @@ func TestMySQL(t *testing.T) { }) t.Run("data", func(t *testing.T) { - testData(t, config, "") + testData(t, config, "./_meta/data_mysql_tables.json") + }) + + config = testFetchConfig{ + Driver: "mysql", + Query: "show status;", + Host: mysql.GetMySQLEnvDSN(service.Host()), + ResponseFormat: variableResponseFormat, + Assertion: assertFieldNotContains("service.address", ":test@"), + } + + t.Run("fetch", func(t *testing.T) { + testFetch(t, config) + }) + + t.Run("data", func(t *testing.T) { + testData(t, config, "./_meta/data_mysql_variables.json") }) } @@ -83,7 +100,23 @@ func TestPostgreSQL(t *testing.T) { }) t.Run("data", func(t *testing.T) { - testData(t, config, "./_meta/data_postgres.json") + testData(t, config, "./_meta/data_postgres_tables.json") + }) + + config = testFetchConfig{ + Driver: "postgres", + Query: "select name, setting from pg_settings", + Host: fmt.Sprintf("postgres://%s:%s@%s:%s/?sslmode=disable", user, password, host, port), + ResponseFormat: variableResponseFormat, + Assertion: assertFieldNotContains("service.address", ":"+password+"@"), + } + + t.Run("fetch with URL", func(t *testing.T) { + testFetch(t, config) + }) + + t.Run("data", func(t *testing.T) { + testData(t, config, "./_meta/data_postgres_variables.json") }) } @@ -107,13 +140,19 @@ func testData(t *testing.T, cfg testFetchConfig, postfix string) { } func getConfig(cfg testFetchConfig) map[string]interface{} { - return map[string]interface{}{ + values := map[string]interface{}{ "module": "sql", "metricsets": []string{"query"}, "hosts": []string{cfg.Host}, "driver": cfg.Driver, "sql_query": cfg.Query, } + + if cfg.ResponseFormat != "" { + values["sql_response_format"] = cfg.ResponseFormat + } + + return values } func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event) { diff --git a/x-pack/metricbeat/modules.d/sql.yml.disabled b/x-pack/metricbeat/modules.d/sql.yml.disabled index 4fd521c69ee..0c2597f31c6 100644 --- a/x-pack/metricbeat/modules.d/sql.yml.disabled +++ b/x-pack/metricbeat/modules.d/sql.yml.disabled @@ -9,4 +9,4 @@ driver: "postgres" sql_query: "select now()" - + sql_response_format: table