From 9375a2a3e5265a19c927615c9a10df76054e15c5 Mon Sep 17 00:00:00 2001 From: Yoshinori Matsunobu Date: Tue, 22 Dec 2015 17:03:41 -0800 Subject: [PATCH] MyRocks: Flush memtables on clean shutdown (#127) Summary: On clean shutdown, memtables were not flushed. This caused data loss, if WAL was disabled. This diff flushes all memtables before calling CancelAllBackgroundWork(). Test Plan: new test case rocksdb.insert_optimized_config Reviewers: spetrunia, jkedgar, hermanlee4 Reviewed By: hermanlee4 Subscribers: sdong, rven, webscalesql-eng Differential Revision: https://reviews.facebook.net/D52257 --- .../rocksdb/r/insert_optimized_config.result | 16 ++++++++ .../t/insert_optimized_config-master.opt | 6 +++ .../rocksdb/t/insert_optimized_config.test | 41 +++++++++++++++++++ storage/rocksdb/ha_rocksdb.cc | 17 ++++++-- 4 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 mysql-test/suite/rocksdb/r/insert_optimized_config.result create mode 100644 mysql-test/suite/rocksdb/t/insert_optimized_config-master.opt create mode 100644 mysql-test/suite/rocksdb/t/insert_optimized_config.test diff --git a/mysql-test/suite/rocksdb/r/insert_optimized_config.result b/mysql-test/suite/rocksdb/r/insert_optimized_config.result new file mode 100644 index 000000000000..ded480578546 --- /dev/null +++ b/mysql-test/suite/rocksdb/r/insert_optimized_config.result @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS t1; +create table t1( +id bigint not null primary key, +i1 bigint, #unique +i2 bigint, #repeating +c1 varchar(20), #unique +c2 varchar(20), #repeating +index t1_2(i1) +) engine=rocksdb; +select count(*), sum(id), sum(i1), sum(i2) from t1; +count(*) sum(id) sum(i1) sum(i2) +50000 1250025000 1250025000 124980000 +select count(*), sum(id), sum(i1), sum(i2) from t1; +count(*) sum(id) sum(i1) sum(i2) +50000 1250025000 1250025000 124980000 +drop table t1; diff --git a/mysql-test/suite/rocksdb/t/insert_optimized_config-master.opt b/mysql-test/suite/rocksdb/t/insert_optimized_config-master.opt new file mode 100644 index 000000000000..2d5e33057f6a --- /dev/null +++ b/mysql-test/suite/rocksdb/t/insert_optimized_config-master.opt @@ -0,0 +1,6 @@ +--rocksdb_write_disable_wal=1 +--rocksdb_default_cf_options=write_buffer_size=16k;target_file_size_base=16k;level0_file_num_compaction_trigger=4;level0_slowdown_writes_trigger=256;level0_stop_writes_trigger=256;max_write_buffer_number=16;compression_per_level=kNoCompression;memtable=vector:1024 +--rocksdb_override_cf_options=__system__={memtable=skip_list:16} +--rocksdb_compaction_sequential_deletes=0 +--rocksdb_compaction_sequential_deletes_window=0 + diff --git a/mysql-test/suite/rocksdb/t/insert_optimized_config.test b/mysql-test/suite/rocksdb/t/insert_optimized_config.test new file mode 100644 index 000000000000..3e1cf7375e80 --- /dev/null +++ b/mysql-test/suite/rocksdb/t/insert_optimized_config.test @@ -0,0 +1,41 @@ +--source include/have_rocksdb.inc + +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings + +create table t1( + id bigint not null primary key, + i1 bigint, #unique + i2 bigint, #repeating + c1 varchar(20), #unique + c2 varchar(20), #repeating + index t1_2(i1) +) engine=rocksdb; + +--disable_query_log +set rocksdb_bulk_load=1; +let $i=0; +while ($i<50000) +{ + inc $i; + eval insert t1(id, i1, i2, c1, c2) values($i, $i, $i div 10, $i, $i div 10); +} +set rocksdb_bulk_load=0; +--enable_query_log + +select count(*), sum(id), sum(i1), sum(i2) from t1; + +# reload without load optimized config +let $restart_file= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; +--exec echo "wait" > $restart_file +--shutdown_server 10 +--source include/wait_until_disconnected.inc +-- exec echo "restart:--rocksdb_write_disable_wal=0 --rocksdb_default_cf_options=write_buffer_size=64k;target_file_size_base=64k;max_bytes_for_level_base=1m;compression_per_level=kNoCompression;" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +-- enable_reconnect +-- source include/wait_until_connected_again.inc + +select count(*), sum(id), sum(i1), sum(i2) from t1; + +drop table t1; + diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 899ff2a487e9..5a15ef366506 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -151,6 +151,15 @@ background_thread_control bg_control; // collation check requirement. std::vector collation_exception_list; +static void +rocksdb_flush_all_memtables() +{ + Column_family_manager& cf_manager = rocksdb_get_cf_manager(); + for (auto cf_handle : cf_manager.get_all_cf()) { + rdb->Flush(rocksdb::FlushOptions(), cf_handle); + } +} + static void rocksdb_compact_column_family(THD* thd, struct st_mysql_sys_var* var, @@ -235,10 +244,7 @@ rocksdb_force_flush_memtable_now(THD* thd, const void* save) { sql_print_information("RocksDB: Manual memtable flush\n"); - Column_family_manager& cf_manager = rocksdb_get_cf_manager(); - for (auto cf_handle : cf_manager.get_all_cf()) { - rdb->Flush(rocksdb::FlushOptions(), cf_handle); - } + rocksdb_flush_all_memtables(); } static void @@ -2418,6 +2424,9 @@ static int rocksdb_done_func(void *p) // signal the drop index thread to stop signal_drop_index_thread(true); + // Flush all memtables for not lose data, even if WAL is disabled. + rocksdb_flush_all_memtables(); + // Stop all rocksdb background work CancelAllBackgroundWork(rdb->GetBaseDB(), true);