From 1475967c5e47bc6fe8c66481028bad277b513934 Mon Sep 17 00:00:00 2001 From: zhangsoledad <787953403@qq.com> Date: Thu, 21 Sep 2023 10:39:31 +0800 Subject: [PATCH 1/3] chore: Update RocksDB to 8.5.3 --- .github/workflows/rust.yml | 6 +++--- Cargo.toml | 2 +- librocksdb-sys/Cargo.toml | 2 +- librocksdb-sys/build_version.cc | 15 +++++++++++--- librocksdb-sys/rocksdb | 2 +- librocksdb-sys/rocksdb_lib_sources.txt | 2 ++ librocksdb-sys/snappy | 2 +- librocksdb-sys/tests/ffi.rs | 2 +- rust-toolchain | 2 +- src/checkpoint.rs | 2 +- src/db_options.rs | 28 +++++++++++++++++++++++++- src/lib.rs | 6 +++--- 12 files changed, 54 insertions(+), 17 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0d88b976e..1dc7a5db1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,7 +13,7 @@ jobs: - name: Install rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.67.1 + toolchain: 1.71.1 components: rustfmt profile: minimal override: true @@ -34,7 +34,7 @@ jobs: - name: Install rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.67.1 + toolchain: 1.71.1 components: clippy profile: minimal override: true @@ -98,7 +98,7 @@ jobs: - name: Install rust uses: actions-rs/toolchain@v1 with: - toolchain: 1.67.1 + toolchain: 1.71.1 target: ${{ matrix.target }} profile: minimal override: true diff --git a/Cargo.toml b/Cargo.toml index 0b1a45f98..22582df9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ march-native = ["librocksdb-sys/march-native"] [dependencies] libc = "0.2" -librocksdb-sys = { package = "ckb-librocksdb-sys", path = "librocksdb-sys", version = "=8.1.1" } +librocksdb-sys = { package = "ckb-librocksdb-sys", path = "librocksdb-sys", version = "=8.5.3" } tempfile = "3" [dev-dependencies] diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index 2eb569504..be35d0e48 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ckb-librocksdb-sys" -version = "8.1.1" +version = "8.5.3" edition = "2018" authors = ["Karl Hobley ", "Arkadiy Paronyan ", "Nervos Core Dev "] license = "MIT/Apache-2.0/BSD-3-Clause" diff --git a/librocksdb-sys/build_version.cc b/librocksdb-sys/build_version.cc index a437d4e94..b97b8b94d 100644 --- a/librocksdb-sys/build_version.cc +++ b/librocksdb-sys/build_version.cc @@ -8,9 +8,18 @@ // The build script may replace these values with real values based // on whether or not GIT is available and the platform settings -static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:6a436150417120a3f9732d65a2a5c2b8d19b60fc"; -static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v8.1.1"; -static const std::string rocksdb_build_date = "rocksdb_build_date:2023-04-06 09:38:52"; +static const std::string rocksdb_build_git_sha = "rocksdb_build_git_sha:f32521662acf3352397d438b732144c7813bbbec"; +static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:v8.5.3"; +#define HAS_GIT_CHANGES 0 +#if HAS_GIT_CHANGES == 0 +// If HAS_GIT_CHANGES is 0, the GIT date is used. +// Use the time the branch/tag was last modified +static const std::string rocksdb_build_date = "rocksdb_build_date:2023-09-01 13:58:39"; +#else +// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications. +// Use the time the build was created. +static const std::string rocksdb_build_date = "rocksdb_build_date:2023-09-21 10:21:22"; +#endif extern "C" { diff --git a/librocksdb-sys/rocksdb b/librocksdb-sys/rocksdb index 6a4361504..f32521662 160000 --- a/librocksdb-sys/rocksdb +++ b/librocksdb-sys/rocksdb @@ -1 +1 @@ -Subproject commit 6a436150417120a3f9732d65a2a5c2b8d19b60fc +Subproject commit f32521662acf3352397d438b732144c7813bbbec diff --git a/librocksdb-sys/rocksdb_lib_sources.txt b/librocksdb-sys/rocksdb_lib_sources.txt index 0383ca736..9b439d323 100644 --- a/librocksdb-sys/rocksdb_lib_sources.txt +++ b/librocksdb-sys/rocksdb_lib_sources.txt @@ -249,6 +249,8 @@ util/stderr_logger.cc util/string_util.cc util/thread_local.cc util/threadpool_imp.cc +util/udt_util.cc +util/write_batch_util.cc util/xxhash.cc utilities/agg_merge/agg_merge.cc utilities/backup/backup_engine.cc diff --git a/librocksdb-sys/snappy b/librocksdb-sys/snappy index 2b63814b1..dc05e0264 160000 --- a/librocksdb-sys/snappy +++ b/librocksdb-sys/snappy @@ -1 +1 @@ -Subproject commit 2b63814b15a2aaae54b7943f0cd935892fae628f +Subproject commit dc05e026488865bc69313a68bcc03ef2e4ea8e83 diff --git a/librocksdb-sys/tests/ffi.rs b/librocksdb-sys/tests/ffi.rs index e7c3db2f2..e27164aca 100644 --- a/librocksdb-sys/tests/ffi.rs +++ b/librocksdb-sys/tests/ffi.rs @@ -1117,7 +1117,7 @@ fn ffi() { rocksdb_slicetransform_create_fixed_prefix(3), ); rocksdb_options_set_hash_skip_list_rep(options, 5000, 4, 4); - rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16); + rocksdb_options_set_plain_table_factory(options, 4, 10, 0.75, 16, 0, 0, 0, 0); rocksdb_options_set_allow_concurrent_memtable_write(options, 0); db = rocksdb_open(options, dbname, &mut err); diff --git a/rust-toolchain b/rust-toolchain index 737e2ba50..68bc7ff2a 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.67.1 +1.71.1 diff --git a/src/checkpoint.rs b/src/checkpoint.rs index 831486937..bc9016d43 100644 --- a/src/checkpoint.rs +++ b/src/checkpoint.rs @@ -15,7 +15,7 @@ use crate::ffi; use crate::ops::*; -///! Implementation of bindings to RocksDB Checkpoint[1] API +/// Implementation of bindings to RocksDB Checkpoint[1] API /// /// [1]: https://github.com/facebook/rocksdb/wiki/Checkpoints use crate::{Error, DB}; diff --git a/src/db_options.rs b/src/db_options.rs index 0b4154ba7..2bde83aee 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -2466,7 +2466,7 @@ impl Options { /// # Examples /// /// ``` - /// use ckb_rocksdb::{Options, PlainTableFactoryOptions}; + /// use ckb_rocksdb::{KeyEncodingType, Options, PlainTableFactoryOptions}; /// /// let mut opts = Options::default(); /// let factory_opts = PlainTableFactoryOptions { @@ -2474,6 +2474,10 @@ impl Options { /// bloom_bits_per_key: 20, /// hash_table_ratio: 0.75, /// index_sparseness: 16, + /// huge_page_tlb_size: 0, + /// encoding_type: KeyEncodingType::Plain, + /// full_scan_mode: false, + /// store_index_in_file: false, /// }; /// /// opts.set_plain_table_factory(&factory_opts); @@ -2486,6 +2490,10 @@ impl Options { options.bloom_bits_per_key, options.hash_table_ratio, options.index_sparseness, + options.huge_page_tlb_size, + options.encoding_type as c_char, + c_uchar::from(options.full_scan_mode), + c_uchar::from(options.store_index_in_file), ); } } @@ -3447,6 +3455,16 @@ pub enum MemtableFactory { }, } +/// Used in [`PlainTableFactoryOptions`]. +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)] +pub enum KeyEncodingType { + /// Always write full keys. + #[default] + Plain = 0, + /// Find opportunities to write the same prefix for multiple rows. + Prefix = 1, +} + /// Used with DBOptions::set_plain_table_factory. /// See https://github.com/facebook/rocksdb/wiki/PlainTable-Format. /// @@ -3455,11 +3473,19 @@ pub enum MemtableFactory { /// bloom_bits_per_key: 10 /// hash_table_ratio: 0.75 /// index_sparseness: 16 +/// huge_page_tlb_size: 0 +/// encoding_type: KeyEncodingType::Plain +/// full_scan_mode: false +/// store_index_in_file: false pub struct PlainTableFactoryOptions { pub user_key_length: u32, pub bloom_bits_per_key: i32, pub hash_table_ratio: f64, pub index_sparseness: usize, + pub huge_page_tlb_size: usize, + pub encoding_type: KeyEncodingType, + pub full_scan_mode: bool, + pub store_index_in_file: bool, } #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/src/lib.rs b/src/lib.rs index 2d8b21bbb..e9e2fe49d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,9 +103,9 @@ pub use crate::db_iterator::{DBIterator, DBRawIterator, Direction, IteratorMode} pub use crate::db_options::{ BlockBasedIndexType, BlockBasedOptions, BottommostLevelCompaction, Cache, CompactOptions, CuckooTableOptions, DBCompactionStyle, DBCompressionType, DBPath, DBRecoveryMode, - DataBlockIndexType, Env, FifoCompactOptions, FlushOptions, IngestExternalFileOptions, LogLevel, - MemtableFactory, Options, PlainTableFactoryOptions, ReadOptions, UniversalCompactOptions, - UniversalCompactionStopStyle, WriteOptions, + DataBlockIndexType, Env, FifoCompactOptions, FlushOptions, IngestExternalFileOptions, + KeyEncodingType, LogLevel, MemtableFactory, Options, PlainTableFactoryOptions, ReadOptions, + UniversalCompactOptions, UniversalCompactionStopStyle, WriteOptions, }; pub use crate::db_pinnable_slice::DBPinnableSlice; pub use crate::db_vector::DBVector; From d97b9a7c79d8f7a11b3beb3d97859459318c3f32 Mon Sep 17 00:00:00 2001 From: zhangsoledad <787953403@qq.com> Date: Thu, 21 Sep 2023 11:13:00 +0800 Subject: [PATCH 2/3] chore: Remove temporary boxed keys in batched_multi_get --- src/ops/multi_get.rs | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/ops/multi_get.rs b/src/ops/multi_get.rs index 8b004652b..10d77d972 100644 --- a/src/ops/multi_get.rs +++ b/src/ops/multi_get.rs @@ -191,7 +191,7 @@ where } pub trait BatchedMultiGetCF { - fn batched_multi_get_cf_full( + fn batched_multi_get_cf_full<'a, K, I>( &self, cf: &ColumnFamily, keys: I, @@ -199,23 +199,23 @@ pub trait BatchedMultiGetCF { readopts: Option<&R>, ) -> Vec, Error>> where - K: AsRef<[u8]>, - I: IntoIterator; + K: AsRef<[u8]> + 'a + ?Sized, + I: IntoIterator; - fn batched_multi_get_cf( + fn batched_multi_get_cf<'a, K, I>( &self, cf: &ColumnFamily, keys: I, sorted_input: bool, ) -> Vec, Error>> where - K: AsRef<[u8]>, - I: IntoIterator, + K: AsRef<[u8]> + 'a + ?Sized, + I: IntoIterator, { self.batched_multi_get_cf_full(cf, keys, sorted_input, None) } - fn batched_multi_get_cf_opt( + fn batched_multi_get_cf_opt<'a, K, I>( &self, cf: &ColumnFamily, keys: I, @@ -223,8 +223,8 @@ pub trait BatchedMultiGetCF { readopts: &R, ) -> Vec, Error>> where - K: AsRef<[u8]>, - I: IntoIterator, + K: AsRef<[u8]> + 'a + ?Sized, + I: IntoIterator, { self.batched_multi_get_cf_full(cf, keys, sorted_input, Some(readopts)) } @@ -234,7 +234,7 @@ impl BatchedMultiGetCF for T where T: Handle + super::Read, { - fn batched_multi_get_cf_full( + fn batched_multi_get_cf_full<'a, K, I>( &self, cf: &ColumnFamily, keys: I, @@ -242,8 +242,8 @@ where readopts: Option<&ReadOptions>, ) -> Vec, Error>> where - K: AsRef<[u8]>, - I: IntoIterator, + K: AsRef<[u8]> + 'a + ?Sized, + I: IntoIterator, { let mut default_readopts = None; let ro_handle = match ReadOptions::input_or_default(readopts, &mut default_readopts) { @@ -258,11 +258,13 @@ where } }; - let (keys, keys_sizes): (Vec>, Vec<_>) = keys + let (ptr_keys, keys_sizes): (Vec<_>, Vec<_>) = keys .into_iter() - .map(|k| (Box::from(k.as_ref()), k.as_ref().len())) + .map(|k| { + let k = k.as_ref(); + (k.as_ptr() as *const c_char, k.len()) + }) .unzip(); - let ptr_keys: Vec<_> = keys.iter().map(|k| k.as_ptr() as *const c_char).collect(); let mut pinned_values = vec![ptr::null_mut(); ptr_keys.len()]; let mut errors = vec![ptr::null_mut(); ptr_keys.len()]; From fa82a38598dc5777408ffbbf1abae988ee8c2c8b Mon Sep 17 00:00:00 2001 From: zhangsoledad <787953403@qq.com> Date: Thu, 21 Sep 2023 11:24:42 +0800 Subject: [PATCH 3/3] chore: bump 0.21 --- Cargo.toml | 4 ++-- librocksdb-sys/Cargo.toml | 8 ++++---- src/db_options.rs | 2 +- src/db_vector.rs | 2 +- src/ffi_util.rs | 2 +- src/ops/property.rs | 2 +- tests/fail/checkpoint_outlive_db.stderr | 3 ++- tests/fail/iterator/db.stderr | 3 ++- tests/fail/iterator/optimistic_transaction_db.stderr | 3 ++- 9 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 22582df9d..255e194b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ckb-rocksdb" description = "Rust wrapper for Facebook's RocksDB embeddable database" -version = "0.20.0" +version = "0.21.0" edition = "2018" authors = ["Tyler Neely ", "David Greenberg ", "Nervos Core Dev "] license = "Apache-2.0" @@ -36,7 +36,7 @@ tempfile = "3" [dev-dependencies] trybuild = "1.0" -criterion = "0.4" +criterion = "0.5" [[bench]] name = "transaction" diff --git a/librocksdb-sys/Cargo.toml b/librocksdb-sys/Cargo.toml index be35d0e48..3054c4e31 100644 --- a/librocksdb-sys/Cargo.toml +++ b/librocksdb-sys/Cargo.toml @@ -40,7 +40,7 @@ bzip2 = [] [dependencies] libc = "0.2" -pkg-config = "0.3.25" +pkg-config = "0.3" [dev-dependencies] const-cstr = "0.3" @@ -48,7 +48,7 @@ uuid = { version = "1.0", features = ["v4"] } [build-dependencies] cc = { version = "1.0", features = ["parallel"] } -bindgen = "0.63.0" -glob = "0.3.0" +bindgen = "0.68.1" +glob = "0.3.1" pkg-config = "0.3" -rust-ini = "0.18" +rust-ini = "0.19" diff --git a/src/db_options.rs b/src/db_options.rs index 2bde83aee..d03ddacfc 100644 --- a/src/db_options.rs +++ b/src/db_options.rs @@ -2577,7 +2577,7 @@ impl Options { // Must have valid UTF-8 format. let s = CStr::from_ptr(value).to_str().unwrap().to_owned(); - libc::free(value as *mut c_void); + ffi::rocksdb_free(value as *mut c_void); Some(s) } } diff --git a/src/db_vector.rs b/src/db_vector.rs index 3651d9b35..9a6d07510 100644 --- a/src/db_vector.rs +++ b/src/db_vector.rs @@ -46,7 +46,7 @@ impl AsRef<[u8]> for DBVector { impl Drop for DBVector { fn drop(&mut self) { unsafe { - libc::free(self.base as *mut c_void); + ffi::rocksdb_free(self.base as *mut c_void); } } } diff --git a/src/ffi_util.rs b/src/ffi_util.rs index f1d80bbe5..58afff40a 100644 --- a/src/ffi_util.rs +++ b/src/ffi_util.rs @@ -25,7 +25,7 @@ pub fn error_message(ptr: *const c_char) -> String { let cstr = unsafe { CStr::from_ptr(ptr as *const _) }; let s = String::from_utf8_lossy(cstr.to_bytes()).into_owned(); unsafe { - libc::free(ptr as *mut c_void); + ffi::rocksdb_free(ptr as *mut c_void); } s } diff --git a/src/ops/property.rs b/src/ops/property.rs index b716b2087..4b31e2b9d 100644 --- a/src/ops/property.rs +++ b/src/ops/property.rs @@ -118,7 +118,7 @@ fn property_value_impl( ))), }; unsafe { - libc::free(value as *mut c_void); + ffi::rocksdb_free(value as *mut c_void); } result } diff --git a/tests/fail/checkpoint_outlive_db.stderr b/tests/fail/checkpoint_outlive_db.stderr index d4231a3e0..c9b12dbc5 100644 --- a/tests/fail/checkpoint_outlive_db.stderr +++ b/tests/fail/checkpoint_outlive_db.stderr @@ -1,9 +1,10 @@ error[E0597]: `db` does not live long enough - --> $DIR/checkpoint_outlive_db.rs:9:25 + --> tests/fail/checkpoint_outlive_db.rs:9:25 | 7 | let _checkpoint = { | ----------- borrow later stored here 8 | let db = DB::open_default("foo").unwrap(); + | -- binding `db` declared here 9 | Checkpoint::new(&db) | ^^^ borrowed value does not live long enough 10 | }; diff --git a/tests/fail/iterator/db.stderr b/tests/fail/iterator/db.stderr index 5b302ebb5..43d381d4a 100644 --- a/tests/fail/iterator/db.stderr +++ b/tests/fail/iterator/db.stderr @@ -1,9 +1,10 @@ error[E0597]: `db` does not live long enough - --> $DIR/db.rs:9:9 + --> tests/fail/iterator/db.rs:9:9 | 7 | let _iter = { | ----- borrow later stored here 8 | let db = DB::open_default("foo").unwrap(); + | -- binding `db` declared here 9 | db.iterator(IteratorMode::Start) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ borrowed value does not live long enough 10 | }; diff --git a/tests/fail/iterator/optimistic_transaction_db.stderr b/tests/fail/iterator/optimistic_transaction_db.stderr index 272316911..b1b068790 100644 --- a/tests/fail/iterator/optimistic_transaction_db.stderr +++ b/tests/fail/iterator/optimistic_transaction_db.stderr @@ -1,9 +1,10 @@ error[E0597]: `db` does not live long enough - --> $DIR/optimistic_transaction_db.rs:9:9 + --> tests/fail/iterator/optimistic_transaction_db.rs:9:9 | 7 | let _iter = { | ----- borrow later stored here 8 | let db = OptimisticTransactionDB::open_default("foo").unwrap(); + | -- binding `db` declared here 9 | db.iterator(IteratorMode::Start) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ borrowed value does not live long enough 10 | };