Merge remote-tracking branch 'tikv/master' into merge-master-beyond-8.4

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tikv"
-version = "8.4.0-alpha"
+version = "8.5.0-alpha"
 authors = ["The TiKV Authors"]
 description = "A distributed transactional key-value database powered by Rust and Raft"
 license = "Apache-2.0"
@@ -35,7 +35,6 @@ testexport = [
   "engine_traits/testexport",
   "engine_rocks/testexport",
   "engine_panic/testexport",
-  "hybrid_engine/testexport",
   "encryption/testexport",
 ]
 test-engine-kv-rocksdb = ["engine_test/test-engine-kv-rocksdb"]

components/backup-stream/src/observer.rs

@@ -296,7 +296,7 @@ mod tests {
         // Test region out of range won't be added to observe list.
         let r = fake_region(43, b"0010", b"0042");
         let mut ctx = ObserverContext::new(&r);
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert!(task.is_err(), "it is {:?}", task);
         assert!(!subs.is_observing(43));
@@ -311,7 +311,7 @@ mod tests {
         // Test give up subscripting when become follower.
         let r = fake_region(42, b"0008", b"0009");
         let mut ctx = ObserverContext::new(&r);
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert_matches!(
             task,
@@ -333,7 +333,7 @@ mod tests {
             RegionChangeEvent::Update(RegionChangeReason::Split),
             StateRole::Leader,
         );
-        o.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        o.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         let task = rx.recv_timeout(Duration::from_millis(20));
         assert!(task.is_err(), "it is {:?}", task);
     }

components/cdc/src/observer.rs

@@ -263,7 +263,7 @@ mod tests {
         region.mut_peers().push(new_peer(3, 3));
 
         let mut ctx = ObserverContext::new(&region);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         let oid = ObserveId::new();
@@ -329,7 +329,7 @@ mod tests {
         };
 
         // No event if it changes to leader.
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Leader));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Leader));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         // unsubscribed fail if observer id is different.
@@ -338,13 +338,13 @@ mod tests {
         // No event if it is unsubscribed.
         let oid_ = observer.unsubscribe_region(1, oid).unwrap();
         assert_eq!(oid_, oid);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
 
         // No event if it is unsubscribed.
         region.set_id(999);
         let mut ctx = ObserverContext::new(&region);
-        observer.on_role_change(&mut ctx, &RoleChange::new(StateRole::Follower));
+        observer.on_role_change(&mut ctx, &RoleChange::new_for_test(StateRole::Follower));
         rx.recv_timeout(Duration::from_millis(10)).unwrap_err();
     }
 }

components/cdc/tests/failpoints/test_register.rs

@@ -97,7 +97,7 @@ fn test_region_ready_after_deregister_impl<F: KvFormat>() {
         .obs
         .get(&leader.get_store_id())
         .unwrap()
-        .on_role_change(&mut context, &RoleChange::new(StateRole::Follower));
+        .on_role_change(&mut context, &RoleChange::new_for_test(StateRole::Follower));
 
     // Then CDC should not panic
     fail::remove(fp);

components/crossbeam-skiplist/Cargo.toml

@@ -30,6 +30,9 @@ alloc = ["crossbeam-epoch/alloc"]
 
 [dependencies]
 crossbeam-epoch = { workspace = true }
+# Add the official version of `crossbeam-skiplist` to dependency tree, so that
+# our CI can keep track of upstream critical bug-fixes and security updates.
+crossbeam-skiplist-offical = { version = "0.1.3", package = "crossbeam-skiplist" }
 crossbeam-utils = { workspace = true }
 
 # Rename test and example binary names to pass TiKV jemalloc check.

components/crossbeam-skiplist/src/lib.rs

@@ -257,3 +257,9 @@ pub mod set;
 #[cfg(feature = "std")]
 #[doc(inline)]
 pub use crate::{map::SkipMap, set::SkipSet};
+
+// Prevent cargo machete warnings.
+mod seal {
+    pub trait Sealed {}
+    impl Sealed for crossbeam_skiplist_offical::SkipList<(), ()> {}
+}

components/engine_panic/Cargo.toml

@@ -10,9 +10,9 @@ license = "Apache-2.0"
 testexport = []
 
 [dependencies]
-encryption = { workspace = true }
 engine_traits = { workspace = true }
 kvproto = { workspace = true }
+encryption = { workspace = true }
 raft = { workspace = true }
 tracker = { workspace = true }
 txn_types = { workspace = true }

components/engine_panic/src/engine.rs

@@ -2,7 +2,7 @@
 
 use engine_traits::{
     IterMetricsCollector, IterOptions, Iterable, Iterator, KvEngine, MetricsExt, Peekable,
-    ReadOptions, Result, SnapshotContext, SyncMutable, WriteOptions,
+    ReadOptions, Result, SyncMutable, WriteOptions,
 };
 
 use crate::{db_vector::PanicDbVector, snapshot::PanicSnapshot, write_batch::PanicWriteBatch};

components/engine_rocks/Cargo.toml

@@ -61,6 +61,6 @@ package = "rocksdb"
 features = ["encryption"]
 
 [dev-dependencies]
-proptest = "1.0.0"
 rand = "0.8"
 toml = "0.5"
+proptest = "1.0.0"

components/engine_traits/Cargo.toml

@@ -11,7 +11,6 @@ testexport = []
 
 [dependencies]
 collections = { workspace = true }
-encryption = { workspace = true }
 error_code = { workspace = true }
 fail = "0.5"
 file_system = { workspace = true }
@@ -20,6 +19,7 @@ kvproto = { workspace = true }
 log_wrappers = { workspace = true }
 protobuf = "2"
 raft = { workspace = true }
+encryption = { workspace = true }
 serde = "1.0"
 slog = { workspace = true }
 slog-global = { workspace = true }

components/engine_traits/src/engine.rs

@@ -78,16 +78,3 @@ pub trait KvEngine:
     #[cfg(feature = "testexport")]
     fn inner_refcount(&self) -> usize;
 }
-
-#[derive(Debug, Clone)]
-pub struct SnapshotContext {
-    pub region: Option<CacheRegion>,
-    pub read_ts: u64,
-}
-
-impl SnapshotContext {
-    pub fn set_region(&mut self, region: CacheRegion) {
-        assert!(self.region.is_none());
-        self.region = Some(region);
-    }
-}

components/engine_traits/src/region_cache_engine.rs

@@ -57,6 +57,7 @@ pub enum EvictReason {
     ApplySnapshot,
     Flashback,
     Manual,
+    PeerDestroy,
 }
 
 /// RegionCacheEngine works as a region cache caching some regions (in Memory or
@@ -92,13 +93,13 @@ pub trait RegionCacheEngine:
 }
 
 pub trait RegionCacheEngineExt {
-    // TODO(SpadeA): try to find a better way to reduce coupling degree of range
-    // cache engine and kv engine
+    // TODO(SpadeA): try to find a better way to reduce coupling degree of
+    // region cache engine and kv engine
     fn on_region_event(&self, event: RegionEvent);
 
-    fn region_cached(&self, range: &Region) -> bool;
+    fn region_cached(&self, region: &Region) -> bool;
 
-    fn load_region(&self, range: &Region);
+    fn load_region(&self, region: &Region);
 }
 
 /// A service that should run in the background to retrieve and apply cache

components/engine_traits/src/snapshot.rs

@@ -13,10 +13,9 @@ where
     Self:
         'static + Peekable + Iterable + CfNamesExt + SnapshotMiscExt + Send + Sync + Sized + Debug,
 {
-    /// Whether the snapshot acquired hit the cached region in the region cache
-    /// engine. It always returns false if the region cahce engine is not
-    /// enabled.
-    fn region_cache_engine_hit(&self) -> bool {
+    /// Whether the snapshot acquired hit the in memory engine. It always
+    /// returns false if the in memory engine is disabled.
+    fn in_memory_engine_hit(&self) -> bool {
         false
     }
 }

components/engine_traits/src/write_batch.rs

@@ -1,7 +1,5 @@
 // Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.
 
-use kvproto::metapb;
-
 use crate::{errors::Result, options::WriteOptions};
 
 /// Engines that can create write batches
@@ -138,8 +136,4 @@ pub trait WriteBatch: Mutable {
     fn merge(&mut self, src: Self) -> Result<()>
     where
         Self: Sized;
-
-    /// It declares that the following consecutive write will be within this
-    /// region.
-    fn prepare_for_region(&mut self, _: &metapb::Region) {}
 }

components/hybrid_engine/Cargo.toml

@@ -5,37 +5,28 @@ edition = "2021"
 publish = false
 license = "Apache-2.0"
 
-[features]
-testexport = ["raftstore/testexport"]
-failpoints = ["fail/failpoints"]
-
-[[test]]
-name = "failpoints"
-path = "tests/failpoints/mod.rs"
-required-features = ["failpoints"]
-
 [dependencies]
-crossbeam = { workspace = true }
-engine_rocks = { workspace = true }
 engine_traits = { workspace = true }
-fail = "0.5"
-in_memory_engine = { workspace = true }
-keys = { workspace = true }
-kvproto = { workspace = true }
-lazy_static = "1.4.0"
+txn_types = { workspace = true }
+tikv_util = { workspace = true }
+engine_rocks = { workspace = true }
 online_config = { workspace = true }
+in_memory_engine = { workspace = true }
+slog = { workspace = true }
+slog-global = { workspace = true }
+tempfile = "3.0"
 prometheus = { version = "0.13", default-features = false, features = [
     "nightly",
 ] }
 prometheus-static-metric = "0.5"
-raft = { workspace = true }
+lazy_static = "1.4.0"
+crossbeam = { workspace = true }
 raftstore = { workspace = true }
-slog = { workspace = true }
-slog-global = { workspace = true }
-tempfile = "3.0"
-tikv_util = { workspace = true }
-txn_types = { workspace = true }
+raft = { workspace = true }
+kvproto = { workspace = true }
+keys = { workspace = true }
 
 [dev-dependencies]
 tempfile = "3.0"
 test_util = { workspace = true }
+fail = { version = "0.5", features = ["failpoints"] }