Re-reorganization the common crates.

Cargo.toml

@@ -5,18 +5,15 @@ members = [
     "common/base",
     "common/building",
     "common/cache",
-    "common/containers",
     "common/contexts",
     "common/datablocks",
     "common/datavalues",
     "common/exception",
     "common/grpc",
     "common/functions",
-    "common/infallible",
     "common/io",
     "common/macros",
     "common/management",
-    "common/mem-allocator",
     "common/planners",
     "common/proto-conv",
     "common/protos",
@@ -25,7 +22,6 @@ members = [
     "common/meta/raft-store",
     "common/meta/sled-store",
     "common/meta/types",
-    "common/range-map",
     "common/streams",
     "common/codegen",
     "common/tracing",

common/ast/tests/it/udfs/udf_parser.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use common_ast::udfs::*;
-use common_base::tokio;
+use common_base::base::tokio;
 use common_exception::Result;
 use pretty_assertions::assert_eq;
 use sqlparser::ast::Expr;

common/ast/tests/it/udfs/udf_transformer.rs

@@ -14,7 +14,7 @@
 
 use async_trait::async_trait;
 use common_ast::udfs::*;
-use common_base::tokio;
+use common_base::base::tokio;
 use common_exception::ErrorCode;
 use common_exception::Result;
 use pretty_assertions::assert_eq;

common/base/Cargo.toml

@@ -12,6 +12,13 @@ test = false
 
 [features]
 tracing = ["tokio/tracing"]
+memory-profiling = [
+    "tikv-jemalloc-ctl",
+    "libc",
+    "tikv-jemalloc-sys/stats",
+    "tikv-jemalloc-sys/profiling",
+    "tikv-jemalloc-sys/unprefixed_malloc_on_supported_platforms",
+]
 
 [dependencies] # In alphabetical order
 # Workspace dependencies
@@ -25,6 +32,8 @@ async-trait = "0.1.53"
 ctrlc = { version = "3.2.1", features = ["termination"] }
 futures = "0.3.21"
 hyper = "0.14.18"
+libc = { version = "0.2", optional = true }
+parking_lot = "0.12.0"
 poem = { version = "=1.3.16", features = ["rustls"] }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { version = "1.0.79", default-features = false, features = ["raw_value"] }
@@ -35,6 +44,12 @@ pprof = { git = "https://github.com/tikv/pprof-rs", rev = "666d9e2", features =
     "protobuf-codec",
     "protobuf",
 ] }
+tikv-jemalloc-ctl = { version = "0.4", optional = true }
+tikv-jemalloc-sys = { version = "0.4.3" }
 tokio = { version = "1.17.0", features = ["full"] }
 toml = { version = "0.5.8", default-features = false }
 uuid = { version = "0.8.2", features = ["serde", "v4"] }
+
+[dev-dependencies]
+anyhow = "1.0.56"
+common-macros = { path = "../macros" }

common/base/src/http_shutdown_handlers.rs → common/base/src/base/http_shutdown_handlers.rs

@@ -26,11 +26,10 @@ use poem::listener::Listener;
 use poem::listener::RustlsConfig;
 use poem::listener::TcpListener;
 use poem::Endpoint;
+use tokio::sync::broadcast;
 use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
 
-use crate::tokio::sync::broadcast;
-
 pub struct HttpShutdownHandler {
     service_name: String,
     join_handle: Option<JoinHandle<std::io::Result<()>>>,

common/base/src/base/mod.rs

@@ -0,0 +1,40 @@
+mod format;
+mod http_shutdown_handlers;
+mod net;
+mod profiling;
+mod progress;
+mod runtime;
+mod runtime_tracker;
+mod shutdown_signal;
+mod stop_handle;
+mod stoppable;
+mod string_func;
+mod thread;
+mod uniq_id;
+
+pub use format::Format;
+pub use http_shutdown_handlers::HttpShutdownHandler;
+pub use net::get_free_tcp_port;
+pub use net::get_free_udp_port;
+pub use profiling::Profiling;
+pub use progress::Progress;
+pub use progress::ProgressValues;
+pub use runtime::Dropper;
+pub use runtime::Runtime;
+pub use runtime::TrySpawn;
+pub use runtime_tracker::RuntimeTracker;
+pub use runtime_tracker::ThreadTracker;
+pub use shutdown_signal::signal_stream;
+pub use shutdown_signal::DummySignalStream;
+pub use shutdown_signal::SignalStream;
+pub use shutdown_signal::SignalType;
+pub use stop_handle::StopHandle;
+pub use stoppable::Stoppable;
+pub use string_func::escape_for_key;
+pub use string_func::mask_string;
+pub use string_func::unescape_for_key;
+pub use thread::Thread;
+pub use tokio;
+pub use uniq_id::GlobalSequence;
+pub use uniq_id::GlobalUniqName;
+pub use uuid;

common/base/src/runtime.rs → common/base/src/base/runtime.rs

@@ -22,7 +22,7 @@ use tokio::runtime::Handle;
 use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
 
-use crate::runtime_tracker::RuntimeTracker;
+use super::runtime_tracker::RuntimeTracker;
 
 /// Methods to spawn tasks.
 pub trait TrySpawn {

common/base/src/stop_handle.rs → common/base/src/base/stop_handle.rs

@@ -21,7 +21,7 @@ use common_tracing::tracing;
 use futures::Future;
 use tokio::sync::broadcast;
 
-use crate::Stoppable;
+use super::Stoppable;
 
 /// Handle a group of `Stoppable` tasks.
 /// When a user press ctrl-c, it calls the `stop()` method on every task to close them.

common/base/src/thread.rs → common/base/src/base/thread.rs

@@ -15,7 +15,7 @@
 use std::thread::Builder;
 use std::thread::JoinHandle;
 
-use crate::runtime_tracker::ThreadTracker;
+use super::runtime_tracker::ThreadTracker;
 
 pub struct Thread;
 

common/containers/src/pool.rs → common/base/src/containers/pool.rs

@@ -21,10 +21,11 @@ use std::sync::Mutex;
 use std::time::Duration;
 
 use async_trait::async_trait;
-use common_base::tokio;
-use common_base::tokio::time::sleep;
 use common_tracing::tracing;
 
+use crate::base::tokio;
+use crate::base::tokio::time::sleep;
+
 pub type PoolItem<T> = Arc<tokio::sync::Mutex<Option<T>>>;
 
 /// To build or check an item.

common/infallible/src/lib.rs → common/base/src/infallible/mod.rs

@@ -27,7 +27,7 @@ pub use rwlock_upgrade_read::RwLockUpgradableReadGuard;
 #[macro_export]
 macro_rules! exit_scope {
     ($x:block) => {
-        use common_infallible::ExitGuard;
+        use common_base::infallible::ExitGuard;
         let _exit_guard = ExitGuard::create(move || $x);
     };
 }

common/infallible/src/rwlock.rs → common/base/src/infallible/rwlock.rs

@@ -16,7 +16,7 @@ use parking_lot::RwLock as ParkingRwLock;
 use parking_lot::RwLockReadGuard;
 use parking_lot::RwLockWriteGuard;
 
-use crate::RwLockUpgradableReadGuard;
+use super::RwLockUpgradableReadGuard;
 
 /// A simple wrapper around the lock() function of a std::sync::RwLock
 /// The only difference is that you don't need to call unwrap() on it.

common/base/src/lib.rs

@@ -14,43 +14,8 @@
 
 #![feature(thread_local)]
 
-mod format;
-mod http_shutdown_handlers;
-mod net;
-mod profiling;
-mod progress;
-mod runtime;
-mod runtime_tracker;
-mod shutdown_signal;
-mod stop_handle;
-mod stoppable;
-mod string_func;
-mod thread;
-mod uniq_id;
-
-pub use format::Format;
-pub use http_shutdown_handlers::HttpShutdownHandler;
-pub use net::get_free_tcp_port;
-pub use net::get_free_udp_port;
-pub use profiling::Profiling;
-pub use progress::Progress;
-pub use progress::ProgressValues;
-pub use runtime::Dropper;
-pub use runtime::Runtime;
-pub use runtime::TrySpawn;
-pub use runtime_tracker::RuntimeTracker;
-pub use runtime_tracker::ThreadTracker;
-pub use shutdown_signal::signal_stream;
-pub use shutdown_signal::DummySignalStream;
-pub use shutdown_signal::SignalStream;
-pub use shutdown_signal::SignalType;
-pub use stop_handle::StopHandle;
-pub use stoppable::Stoppable;
-pub use string_func::escape_for_key;
-pub use string_func::mask_string;
-pub use string_func::unescape_for_key;
-pub use thread::Thread;
-pub use tokio;
-pub use uniq_id::GlobalSequence;
-pub use uniq_id::GlobalUniqName;
-pub use uuid;
+pub mod base;
+pub mod containers;
+pub mod infallible;
+pub mod mem_allocator;
+pub mod rangemap;

common/mem-allocator/src/allocators.rs → common/base/src/mem_allocator/allocators.rs

@@ -12,9 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::malloc_size::MallocSizeOf;
-use crate::malloc_size::MallocSizeOfOps;
-use crate::malloc_size::MallocUnconditionalSizeOf;
+use super::malloc_size::MallocSizeOf;
+use super::malloc_size::MallocSizeOfOps;
+use super::malloc_size::MallocUnconditionalSizeOf;
 
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -32,10 +32,10 @@ mod platform {
     use std::os::raw::c_int;
     use std::os::raw::c_void;
 
-    use common_base::ThreadTracker;
     use tikv_jemalloc_sys as ffi;
 
-    use crate::malloc_size::VoidPtrToSizeFn;
+    use crate::base::ThreadTracker;
+    use crate::mem_allocator::malloc_size::VoidPtrToSizeFn;
 
     /// Get the size of a heap block.
     pub unsafe extern "C" fn usable_size(ptr: *const c_void) -> usize {

common/mem-allocator/src/malloc_size.rs → common/base/src/mem_allocator/malloc_size.rs

@@ -524,7 +524,7 @@ impl<T: MallocSizeOf> MallocSizeOf for parking_lot::Mutex<T> {
     }
 }
 
-impl<T: MallocSizeOf> MallocSizeOf for common_infallible::Mutex<T> {
+impl<T: MallocSizeOf> MallocSizeOf for crate::infallible::Mutex<T> {
     fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
         self.lock().size_of(ops)
     }
@@ -542,7 +542,7 @@ impl<T: MallocSizeOf> MallocSizeOf for parking_lot::RwLock<T> {
     }
 }
 
-impl<T: MallocSizeOf> MallocSizeOf for common_infallible::RwLock<T> {
+impl<T: MallocSizeOf> MallocSizeOf for crate::infallible::RwLock<T> {
     fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
         self.read().size_of(ops)
     }
@@ -581,9 +581,9 @@ impl<T> MallocUnconditionalShallowSizeOf for Arc<T> {
 macro_rules! malloc_size_of_is_0(
 	($($ty:ty),+) => (
 		$(
-			impl $crate::MallocSizeOf for $ty {
+			impl $crate::mem_allocator::MallocSizeOf for $ty {
 				#[inline(always)]
-				fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize {
+				fn size_of(&self, _: &mut $crate::mem_allocator::MallocSizeOfOps) -> usize {
 					0
 				}
 				#[inline(always)]
@@ -593,9 +593,9 @@ macro_rules! malloc_size_of_is_0(
 	);
 	(any: $($ty:ident<$($gen:ident),+>),+) => (
 		$(
-			impl<$($gen),+> $crate::MallocSizeOf for $ty<$($gen),+> {
+			impl<$($gen),+> $crate::mem_allocator::MallocSizeOf for $ty<$($gen),+> {
 				#[inline(always)]
-				fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize {
+				fn size_of(&self, _: &mut $crate::mem_allocator::MallocSizeOfOps) -> usize {
 					0
 				}
 				#[inline(always)]
@@ -605,9 +605,9 @@ macro_rules! malloc_size_of_is_0(
 	);
 	($($ty:ident<$($gen:ident),+>),+) => (
 		$(
-			impl<$($gen: $crate::MallocSizeOf),+> $crate::MallocSizeOf for $ty<$($gen),+> {
+			impl<$($gen: $crate::mem_allocator::MallocSizeOf),+> $crate::mem_allocator::MallocSizeOf for $ty<$($gen),+> {
 				#[inline(always)]
-				fn size_of(&self, _: &mut $crate::MallocSizeOfOps) -> usize {
+				fn size_of(&self, _: &mut $crate::mem_allocator::MallocSizeOfOps) -> usize {
 					0
 				}
 				#[inline(always)]

common/mem-allocator/src/sizeof.rs → common/base/src/mem_allocator/sizeof.rs

@@ -23,10 +23,10 @@ use std::mem::size_of;
 use std::mem::size_of_val;
 use std::sync::Arc;
 
-use crate::malloc_size::MallocShallowSizeOf;
-use crate::malloc_size::MallocSizeOf;
-use crate::malloc_size::MallocSizeOfOps;
-use crate::malloc_size::MallocUnconditionalShallowSizeOf;
+use super::malloc_size::MallocShallowSizeOf;
+use super::malloc_size::MallocSizeOf;
+use super::malloc_size::MallocSizeOfOps;
+use super::malloc_size::MallocUnconditionalShallowSizeOf;
 
 impl<T: ?Sized> MallocShallowSizeOf for Box<T> {
     fn shallow_size_of(&self, _ops: &mut MallocSizeOfOps) -> usize {

common/range-map/src/lib.rs → common/base/src/rangemap/mod.rs

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-mod range_key;
+mod range_map_key;
 mod range_map;
 
-pub use range_key::RangeKey;
+pub use range_map_key::RangeMapKey;
 pub use range_map::RangeMap;

common/range-map/src/range_map.rs → common/base/src/rangemap/range_map.rs

@@ -17,11 +17,11 @@ use core::ops::Bound;
 use core::ops::Range;
 use std::collections::BTreeMap;
 
-use super::range_key::RangeKey;
+use super::range_map_key::RangeMapKey;
 
 #[derive(Clone, Debug, Default)]
 pub struct RangeMap<T, K, V> {
-    pub(crate) map: BTreeMap<RangeKey<T, K>, V>,
+    pub(crate) map: BTreeMap<RangeMapKey<T, K>, V>,
 }
 
 impl<T, K, V> RangeMap<T, K, V>
@@ -38,7 +38,7 @@ where
     pub fn insert(&mut self, range: Range<T>, key: K, val: V) {
         assert!(range.start <= range.end);
 
-        let range_key: RangeKey<T, K> = RangeKey::new(range, key);
+        let range_key: RangeMapKey<T, K> = RangeMapKey::new(range, key);
 
         self.map.insert(range_key, val);
     }
@@ -49,9 +49,9 @@ where
     // 2. `get_by_point(2)` return [1,5],[2,4],[2,6]
     // 3. `get_by_point(5)` return [2,4],[2,6]
     // Use the default key when construct `RangeKey::key` for search.
-    pub fn get_by_point(&self, point: &T) -> Vec<(&RangeKey<T, K>, &V)> {
+    pub fn get_by_point(&self, point: &T) -> Vec<(&RangeMapKey<T, K>, &V)> {
         let key = point.clone();
-        let range_key = RangeKey::new(key.clone()..key.clone(), K::default());
+        let range_key = RangeMapKey::new(key.clone()..key.clone(), K::default());
 
         self.map
             .range((Bound::Included(range_key), Bound::Unbounded))
@@ -60,10 +60,10 @@ where
     }
 
     pub fn remove(&mut self, range: Range<T>, k: K) {
-        self.map.remove(&RangeKey::new(range, k));
+        self.map.remove(&RangeMapKey::new(range, k));
     }
 
-    pub fn remove_by_key(&mut self, key: &RangeKey<T, K>) {
+    pub fn remove_by_key(&mut self, key: &RangeMapKey<T, K>) {
         self.map.remove(key);
     }
 }