diff --git a/.gitmodules b/.gitmodules index f56537a00..ef0e03496 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,7 @@ [submodule "librocksdb-sys/rocksdb"] path = librocksdb-sys/rocksdb url = https://github.com/haizhi-tech/rocksdb.git - branch = v7.9.2_statistics_cf + branch = v8.1.1-hz [submodule "librocksdb-sys/lz4"] path = librocksdb-sys/lz4 url = https://github.com/lz4/lz4.git diff --git a/Cargo.toml b/Cargo.toml index 1af148685..892573160 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,4 +45,7 @@ trybuild = "1.0" tempfile = "3.1" pretty_assertions = "1.0" bincode = "1.3" -serde = { version = "1", features = [ "derive" ] } \ No newline at end of file +serde = { version = "1", features = [ "derive" ] } + +[lib] +doctest = false \ No newline at end of file diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index f1626a0aa..937597dd1 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit f1626a0aa11dc5aff74707f44f0f9d460e0396ba +Subproject commit 937597dd1c921936dbadff31661678c96ce5bb23 diff --git a/src/checkpoint.rs b/src/checkpoint.rs index 246ac7e47..0a5e61a58 100644 --- a/src/checkpoint.rs +++ b/src/checkpoint.rs @@ -78,6 +78,12 @@ pub struct RocksdbLevelMetaData { file_creation_time: u64, file_checksum: String, file_checksum_func_name: String, + #[serde(default)] + epoch_number: u64, + #[serde(default)] + hex_smallest: String, + #[serde(default)] + hex_largest: String, } impl ExportImportFilesMetaData { @@ -123,6 +129,8 @@ impl ExportImportFilesMetaData { let hex_largestkey = CString::new(file.hex_largestkey).unwrap(); let file_checksum = CString::new(file.file_checksum).unwrap(); let file_checksum_func_name = CString::new(file.file_checksum_func_name).unwrap(); + let hex_smallest = CString::new(file.hex_smallest).unwrap(); + let hex_largest = CString::new(file.hex_largest).unwrap(); files.push(ffi_try!(ffi::rocksdb_new_live_file_metadata( column_family_name.as_ptr(), file.level, @@ -147,6 +155,9 @@ impl ExportImportFilesMetaData { file.file_creation_time, file_checksum.as_ptr(), file_checksum_func_name.as_ptr(), + file.epoch_number, + hex_smallest.as_ptr(), + hex_largest.as_ptr(), ))); } diff --git a/tests/fail/checkpoint_outlive_db.rs b/tests/fail/checkpoint_outlive_db.rs index d8400e008..def7c0e7c 100644 --- a/tests/fail/checkpoint_outlive_db.rs +++ b/tests/fail/checkpoint_outlive_db.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{DB, checkpoint::Checkpoint}; fn main() { diff --git a/tests/fail/checkpoint_outlive_db.stderr b/tests/fail/checkpoint_outlive_db.stderr index a8ff0cc64..ed81c98fe 100644 --- a/tests/fail/checkpoint_outlive_db.stderr +++ b/tests/fail/checkpoint_outlive_db.stderr @@ -1,10 +1,11 @@ error[E0597]: `db` does not live long enough - --> tests/fail/checkpoint_outlive_db.rs:6:25 + --> tests/fail/checkpoint_outlive_db.rs:7:25 | -4 | let _checkpoint = { +5 | let _checkpoint = { | ----------- borrow later stored here -5 | let db = DB::open_default("foo").unwrap(); -6 | Checkpoint::new(&db) +6 | let db = DB::open_default("foo").unwrap(); + | -- binding `db` declared here +7 | Checkpoint::new(&db) | ^^^ borrowed value does not live long enough -7 | }; +8 | }; | - `db` dropped here while still borrowed diff --git a/tests/fail/iterator_outlive_db.rs b/tests/fail/iterator_outlive_db.rs index 49463f141..fa672da58 100644 --- a/tests/fail/iterator_outlive_db.rs +++ b/tests/fail/iterator_outlive_db.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{IteratorMode, DB}; fn main() { diff --git a/tests/fail/iterator_outlive_db.stderr b/tests/fail/iterator_outlive_db.stderr index 98b5c0b56..e93a13e04 100644 --- a/tests/fail/iterator_outlive_db.stderr +++ b/tests/fail/iterator_outlive_db.stderr @@ -1,10 +1,11 @@ error[E0597]: `db` does not live long enough - --> tests/fail/iterator_outlive_db.rs:6:9 + --> tests/fail/iterator_outlive_db.rs:7:9 | -4 | let _iter = { +5 | let _iter = { | ----- borrow later stored here -5 | let db = DB::open_default("foo").unwrap(); -6 | db.iterator(IteratorMode::Start) - | ^^ borrowed value does not live long enough -7 | }; +6 | let db = DB::open_default("foo").unwrap(); + | -- binding `db` declared here +7 | db.iterator(IteratorMode::Start) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ borrowed value does not live long enough +8 | }; | - `db` dropped here while still borrowed diff --git a/tests/fail/open_with_multiple_refs_as_single_threaded.rs b/tests/fail/open_with_multiple_refs_as_single_threaded.rs index 8bf3524c0..511ae1147 100644 --- a/tests/fail/open_with_multiple_refs_as_single_threaded.rs +++ b/tests/fail/open_with_multiple_refs_as_single_threaded.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{SingleThreaded, DBWithThreadMode, Options}; fn main() { diff --git a/tests/fail/open_with_multiple_refs_as_single_threaded.stderr b/tests/fail/open_with_multiple_refs_as_single_threaded.stderr index f34244685..85bbdbc65 100644 --- a/tests/fail/open_with_multiple_refs_as_single_threaded.stderr +++ b/tests/fail/open_with_multiple_refs_as_single_threaded.stderr @@ -1,17 +1,21 @@ error[E0596]: cannot borrow `*db_ref1` as mutable, as it is behind a `&` reference - --> tests/fail/open_with_multiple_refs_as_single_threaded.rs:8:5 + --> tests/fail/open_with_multiple_refs_as_single_threaded.rs:9:5 + | +9 | db_ref1.create_cf("cf1", &opts).unwrap(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `db_ref1` is a `&` reference, so the data it refers to cannot be borrowed as mutable + | +help: consider changing this to be a mutable reference | -5 | let db_ref1 = &db; - | --- help: consider changing this to be a mutable reference: `&mut db` -... -8 | db_ref1.create_cf("cf1", &opts).unwrap(); - | ^^^^^^^ `db_ref1` is a `&` reference, so the data it refers to cannot be borrowed as mutable +6 | let db_ref1 = &mut db; + | ~~~~~~~ error[E0596]: cannot borrow `*db_ref2` as mutable, as it is behind a `&` reference - --> tests/fail/open_with_multiple_refs_as_single_threaded.rs:9:5 - | -6 | let db_ref2 = &db; - | --- help: consider changing this to be a mutable reference: `&mut db` -... -9 | db_ref2.create_cf("cf2", &opts).unwrap(); - | ^^^^^^^ `db_ref2` is a `&` reference, so the data it refers to cannot be borrowed as mutable + --> tests/fail/open_with_multiple_refs_as_single_threaded.rs:10:5 + | +10 | db_ref2.create_cf("cf2", &opts).unwrap(); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ `db_ref2` is a `&` reference, so the data it refers to cannot be borrowed as mutable + | +help: consider changing this to be a mutable reference + | +7 | let db_ref2 = &mut db; + | ~~~~~~~ diff --git a/tests/fail/snapshot_outlive_db.rs b/tests/fail/snapshot_outlive_db.rs index 09141d90e..5ced7f449 100644 --- a/tests/fail/snapshot_outlive_db.rs +++ b/tests/fail/snapshot_outlive_db.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::DB; fn main() { diff --git a/tests/fail/snapshot_outlive_db.stderr b/tests/fail/snapshot_outlive_db.stderr index ac44bab42..10c0649dc 100644 --- a/tests/fail/snapshot_outlive_db.stderr +++ b/tests/fail/snapshot_outlive_db.stderr @@ -1,10 +1,11 @@ error[E0597]: `db` does not live long enough - --> tests/fail/snapshot_outlive_db.rs:6:9 + --> tests/fail/snapshot_outlive_db.rs:7:9 | -4 | let _snapshot = { +5 | let _snapshot = { | --------- borrow later stored here -5 | let db = DB::open_default("foo").unwrap(); -6 | db.snapshot() - | ^^ borrowed value does not live long enough -7 | }; +6 | let db = DB::open_default("foo").unwrap(); + | -- binding `db` declared here +7 | db.snapshot() + | ^^^^^^^^^^^^^ borrowed value does not live long enough +8 | }; | - `db` dropped here while still borrowed diff --git a/tests/fail/snapshot_outlive_transaction.rs b/tests/fail/snapshot_outlive_transaction.rs index 5ec23466d..20ee7859b 100644 --- a/tests/fail/snapshot_outlive_transaction.rs +++ b/tests/fail/snapshot_outlive_transaction.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{TransactionDB, SingleThreaded}; fn main() { diff --git a/tests/fail/snapshot_outlive_transaction.stderr b/tests/fail/snapshot_outlive_transaction.stderr index 653c1a8ec..8f1f3d7c4 100644 --- a/tests/fail/snapshot_outlive_transaction.stderr +++ b/tests/fail/snapshot_outlive_transaction.stderr @@ -1,10 +1,11 @@ error[E0597]: `txn` does not live long enough - --> tests/fail/snapshot_outlive_transaction.rs:7:9 + --> tests/fail/snapshot_outlive_transaction.rs:8:9 | -5 | let _snapshot = { +6 | let _snapshot = { | --------- borrow later stored here -6 | let txn = db.transaction(); -7 | txn.snapshot() +7 | let txn = db.transaction(); + | --- binding `txn` declared here +8 | txn.snapshot() | ^^^^^^^^^^^^^^ borrowed value does not live long enough -8 | }; +9 | }; | - `txn` dropped here while still borrowed diff --git a/tests/fail/snapshot_outlive_transaction_db.rs b/tests/fail/snapshot_outlive_transaction_db.rs index 8c3b949ba..a02d62f5f 100644 --- a/tests/fail/snapshot_outlive_transaction_db.rs +++ b/tests/fail/snapshot_outlive_transaction_db.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{TransactionDB, SingleThreaded}; fn main() { diff --git a/tests/fail/snapshot_outlive_transaction_db.stderr b/tests/fail/snapshot_outlive_transaction_db.stderr index d07341f1a..97b91d654 100644 --- a/tests/fail/snapshot_outlive_transaction_db.stderr +++ b/tests/fail/snapshot_outlive_transaction_db.stderr @@ -1,10 +1,11 @@ error[E0597]: `db` does not live long enough - --> tests/fail/snapshot_outlive_transaction_db.rs:6:9 + --> tests/fail/snapshot_outlive_transaction_db.rs:7:9 | -4 | let _snapshot = { +5 | let _snapshot = { | --------- borrow later stored here -5 | let db = TransactionDB::::open_default("foo").unwrap(); -6 | db.snapshot() +6 | let db = TransactionDB::::open_default("foo").unwrap(); + | -- binding `db` declared here +7 | db.snapshot() | ^^^^^^^^^^^^^ borrowed value does not live long enough -7 | }; +8 | }; | - `db` dropped here while still borrowed diff --git a/tests/fail/transaction_outlive_transaction_db.rs b/tests/fail/transaction_outlive_transaction_db.rs index 75d43398e..7cafc2fdb 100644 --- a/tests/fail/transaction_outlive_transaction_db.rs +++ b/tests/fail/transaction_outlive_transaction_db.rs @@ -1,3 +1,4 @@ +use haizhi_rocksdb as rocksdb; use rocksdb::{TransactionDB, SingleThreaded}; fn main() { diff --git a/tests/fail/transaction_outlive_transaction_db.stderr b/tests/fail/transaction_outlive_transaction_db.stderr index 45237a3e9..e998039e2 100644 --- a/tests/fail/transaction_outlive_transaction_db.stderr +++ b/tests/fail/transaction_outlive_transaction_db.stderr @@ -1,10 +1,11 @@ error[E0597]: `db` does not live long enough - --> tests/fail/transaction_outlive_transaction_db.rs:6:9 + --> tests/fail/transaction_outlive_transaction_db.rs:7:9 | -4 | let _txn = { +5 | let _txn = { | ---- borrow later stored here -5 | let db = TransactionDB::::open_default("foo").unwrap(); -6 | db.transaction() +6 | let db = TransactionDB::::open_default("foo").unwrap(); + | -- binding `db` declared here +7 | db.transaction() | ^^^^^^^^^^^^^^^^ borrowed value does not live long enough -7 | }; +8 | }; | - `db` dropped here while still borrowed diff --git a/tests/test_approximate.rs b/tests/test_approximate.rs index ffecf2c0c..c455eb0f7 100644 --- a/tests/test_approximate.rs +++ b/tests/test_approximate.rs @@ -1,6 +1,7 @@ -use haizhi_rocksdb as rocksdb; use std::time::Duration; +use haizhi_rocksdb as rocksdb; + use rocksdb::Ranges; use rocksdb::{ColumnFamilyDescriptor, Options, DB}; #[test] diff --git a/tests/test_backup.rs b/tests/test_backup.rs index 31f3faf30..42303b84f 100644 --- a/tests/test_backup.rs +++ b/tests/test_backup.rs @@ -16,7 +16,9 @@ mod util; use pretty_assertions::assert_eq; -use haizhi_rocksdb::{ +use haizhi_rocksdb as rocksdb; + +use rocksdb::{ backup::{BackupEngine, BackupEngineOptions, RestoreOptions}, Env, DB, }; diff --git a/tests/test_checkpoint.rs b/tests/test_checkpoint.rs index 14a5ef3f8..da551c76c 100644 --- a/tests/test_checkpoint.rs +++ b/tests/test_checkpoint.rs @@ -14,9 +14,10 @@ mod util; -use haizhi_rocksdb as rocksdb; use pretty_assertions::assert_eq; +use haizhi_rocksdb as rocksdb; + use rocksdb::{ checkpoint::{Checkpoint, ExportImportFilesMetaData}, Options, DB, diff --git a/tests/test_column_family.rs b/tests/test_column_family.rs index a250c6e68..197299918 100644 --- a/tests/test_column_family.rs +++ b/tests/test_column_family.rs @@ -14,9 +14,10 @@ mod util; -use haizhi_rocksdb as rocksdb; use pretty_assertions::assert_eq; +use haizhi_rocksdb as rocksdb; + use rocksdb::checkpoint::{Checkpoint, ExportImportFilesMetaData}; use rocksdb::{ColumnFamilyDescriptor, MergeOperands, Options, DB, DEFAULT_COLUMN_FAMILY_NAME}; use rocksdb::{TransactionDB, TransactionDBOptions}; @@ -503,10 +504,11 @@ fn test_no_leaked_column_family() { #[test] fn test_create_cf_with_import() { - const PATH_PREFIX: &str = "_rust_rocksdb_create_cf_with_import_"; - + const PATH_PREFIX: &str = "/tmp/_rust_rocksdb_create_cf_with_import_"; + let _ = std::fs::remove_dir_all(PATH_PREFIX); // Create DB with some data - let origin_db_path = DBPath::new(&format!("{}db1", PATH_PREFIX)); + let origin_db_string_path = format!("{}/db1", PATH_PREFIX); + let origin_db_path = Path::new(&origin_db_string_path); let mut opts = Options::default(); opts.create_if_missing(true); @@ -524,12 +526,17 @@ fn test_create_cf_with_import() { assert!(origin_db.put_cf(&cf1, b"1", b"1").is_ok()); let cf2 = origin_db.cf_handle("cf2").unwrap(); assert!(origin_db.put_cf(&cf2, b"2", b"2").is_ok()); + // add some keys and call delete range + assert!(origin_db.put_cf(&cf1, b"a1", b"a1").is_ok()); + assert!(origin_db.put_cf(&cf1, b"a2", b"a2").is_ok()); + assert!(origin_db.delete_range_cf(&cf1, b"a0", b"a5").is_ok()); let checkpoint = Checkpoint::new(&origin_db); assert!(checkpoint.is_ok()); let checkpoint = checkpoint.unwrap(); - let export_path = DBPath::new(&format!("{}db1_backup", PATH_PREFIX)); + let export_path = format!("{}/db1_backup", PATH_PREFIX); + let export_path = Path::new(&export_path); let result = checkpoint.export_column_family(cf1, &export_path); assert!(result.is_ok()); drop(checkpoint); @@ -537,9 +544,11 @@ fn test_create_cf_with_import() { let origin_metadata = result.unwrap(); let metadata_path = Path::new("/tmp/db1_metadata.json"); origin_metadata.save(metadata_path).unwrap(); - let recover_metadata = ExportImportFilesMetaData::load(metadata_path).unwrap(); + let recover_metadata = + ExportImportFilesMetaData::load(metadata_path, origin_db_string_path.clone()).unwrap(); // new db from export path - let recover_db_path = DBPath::new(&format!("{}db1_recover", PATH_PREFIX)); + let recover_db_path = format!("{}/db1_recover", PATH_PREFIX); + let recover_db_path = Path::new(&recover_db_path); let mut recover_db = DB::open(&opts, &recover_db_path).unwrap(); assert!(recover_db.cf_handle("cf1").is_none()); assert!(recover_db.cf_handle("cf2").is_none()); @@ -553,23 +562,6 @@ fn test_create_cf_with_import() { ); assert!(recover_db.cf_handle("cf2").is_none()); assert!(recover_db.get_cf(&cf1, b"2").unwrap().is_none()); - // then we will test origin db - assert!(origin_db.put_cf(&cf1, b"11", b"11").is_ok()); - assert!(origin_db.drop_cf("cf1").is_ok()); - // cf1 and its data are none - assert!(origin_db.cf_handle("cf1").is_none()); - // import cf1 - assert!(origin_db - .create_cf_with_import("cf1", &opts, &recover_metadata) - .is_ok()); - let cf1 = origin_db.cf_handle("cf1").unwrap(); - assert_eq!( - origin_db.get_cf(&cf1, vec![1]).unwrap().unwrap(), - b"illegal1" - ); - assert_eq!(origin_db.get_cf(&cf1, b"1").unwrap().unwrap(), b"1"); - assert!(origin_db.get_cf(&cf1, b"11").unwrap().is_none()); - assert!(origin_db.put_cf(&cf1, b"11", b"11").is_ok()); // import cf3 assert!(origin_db .create_cf_with_import("cf3", &opts, &recover_metadata) @@ -581,4 +573,6 @@ fn test_create_cf_with_import() { ); assert_eq!(origin_db.get_cf(&cf3, b"1").unwrap().unwrap(), b"1"); assert!(origin_db.get_cf(&cf3, b"11").unwrap().is_none()); + assert!(origin_db.get_cf(&cf3, b"a1").unwrap().is_none()); + let _ = std::fs::remove_dir_all(PATH_PREFIX); } diff --git a/tests/test_comparator.rs b/tests/test_comparator.rs index 81910a245..4dcfd0c63 100644 --- a/tests/test_comparator.rs +++ b/tests/test_comparator.rs @@ -1,3 +1,5 @@ +use haizhi_rocksdb as rocksdb; + use rocksdb::{Options, DB}; use std::cmp::Ordering; use std::iter::FromIterator; diff --git a/tests/test_compationfilter.rs b/tests/test_compationfilter.rs index d16fbee83..191877fda 100644 --- a/tests/test_compationfilter.rs +++ b/tests/test_compationfilter.rs @@ -13,8 +13,8 @@ // limitations under the License. mod util; - use haizhi_rocksdb as rocksdb; + use pretty_assertions::assert_eq; use rocksdb::{CompactionDecision, Options, DB}; diff --git a/tests/test_db.rs b/tests/test_db.rs index 2f5820a94..6d2fd4d40 100644 --- a/tests/test_db.rs +++ b/tests/test_db.rs @@ -14,10 +14,11 @@ mod util; +use haizhi_rocksdb as rocksdb; + use std::convert::TryInto; use std::{mem, sync::Arc, thread, time::Duration}; -use haizhi_rocksdb as rocksdb; use pretty_assertions::assert_eq; use rocksdb::{ @@ -851,7 +852,7 @@ fn get_with_cache_and_bulkload_test() { { // set block based table and cache - let cache = Cache::new_lru_cache(512 << 10); + let cache = Cache::new_lru_cache(512 << 10).unwrap(); assert_eq!(cache.get_usage(), 0); let mut block_based_opts = BlockBasedOptions::default(); block_based_opts.set_block_cache(&cache); @@ -986,7 +987,7 @@ fn get_with_cache_and_bulkload_and_blobs_test() { { // set block based table and cache - let cache = Cache::new_lru_cache(512 << 10); + let cache = Cache::new_lru_cache(512 << 10).unwrap(); assert_eq!(cache.get_usage(), 0); let mut block_based_opts = BlockBasedOptions::default(); block_based_opts.set_block_cache(&cache); diff --git a/tests/test_iterator.rs b/tests/test_iterator.rs index a74f9464d..10d63f7a1 100644 --- a/tests/test_iterator.rs +++ b/tests/test_iterator.rs @@ -15,6 +15,7 @@ mod util; use haizhi_rocksdb as rocksdb; + use pretty_assertions::assert_eq; use rocksdb::{Direction, IteratorMode, MemtableFactory, Options, DB}; diff --git a/tests/test_rocksdb_options.rs b/tests/test_rocksdb_options.rs index 77dbcbbc0..360f3da8f 100644 --- a/tests/test_rocksdb_options.rs +++ b/tests/test_rocksdb_options.rs @@ -35,7 +35,7 @@ fn test_load_latest() { &n, Env::new().unwrap(), true, - Cache::new_lru_cache(1024 * 8), + Cache::new_lru_cache(1024 * 8).unwrap(), ) .unwrap(); assert!(cfs.iter().any(|cf| cf.name() == "default")); diff --git a/tests/test_write_batch.rs b/tests/test_write_batch.rs index 47bf32a91..379268d6f 100644 --- a/tests/test_write_batch.rs +++ b/tests/test_write_batch.rs @@ -16,7 +16,6 @@ use std::collections::HashMap; use haizhi_rocksdb as rocksdb; use pretty_assertions::assert_eq; - use rocksdb::{WriteBatch, WriteBatchIterator}; #[test] diff --git a/tests/util/mod.rs b/tests/util/mod.rs index 4fbd54e5e..add27c737 100644 --- a/tests/util/mod.rs +++ b/tests/util/mod.rs @@ -1,9 +1,9 @@ #![allow(dead_code)] +use haizhi_rocksdb as rocksdb; +use rocksdb::{Error, Options, DB}; use std::path::{Path, PathBuf}; -use haizhi_rocksdb::{Error, Options, DB}; - /// Temporary database path which calls DB::Destroy when DBPath is dropped. pub struct DBPath { dir: tempfile::TempDir, // kept for cleaning up during drop