feat(rust, python): streaming inner joins. (pola-rs#5400)

zundertj · Jan 7, 2023 · dc63579 · dc63579
1 parent 0a9e171
commit dc63579
Show file tree

Hide file tree

Showing 35 changed files with 1,734 additions and 922 deletions.
diff --git a/polars/Cargo.toml b/polars/Cargo.toml
@@ -129,6 +129,7 @@ algo = ["polars-algo"]
 cse = ["polars-lazy/cse"]
 propagate_nans = ["polars-lazy/propagate_nans"]
 coalesce = ["polars-lazy/coalesce"]
+streaming = ["polars-lazy/streaming"]
 
 test = [
   "lazy",

diff --git a/polars/polars-core/src/datatypes/aliases.rs b/polars/polars-core/src/datatypes/aliases.rs
@@ -1,20 +1,13 @@
 use super::*;
 
-#[cfg(feature = "private")]
-pub type PlHashMap<K, V> = hashbrown::HashMap<K, V, RandomState>;
-#[cfg(feature = "private")]
-pub type PlHashSet<V> = hashbrown::HashSet<V, RandomState>;
-#[cfg(feature = "private")]
-pub type PlIndexMap<K, V> = indexmap::IndexMap<K, V, RandomState>;
-#[cfg(feature = "private")]
-pub type PlIndexSet<K> = indexmap::IndexSet<K, RandomState>;
-
 #[cfg(not(feature = "bigidx"))]
 pub type IdxCa = UInt32Chunked;
 #[cfg(feature = "bigidx")]
 pub type IdxCa = UInt64Chunked;
 pub use polars_arrow::index::{IdxArr, IdxSize};
 
+use crate::vector_hasher::IdBuildHasher;
+
 #[cfg(not(feature = "bigidx"))]
 pub const IDX_DTYPE: DataType = DataType::UInt32;
 #[cfg(feature = "bigidx")]
@@ -27,6 +20,19 @@ pub type IdxType = UInt64Type;
 
 pub const NULL_DTYPE: DataType = DataType::Int32;
 
+#[cfg(feature = "private")]
+pub type PlHashMap<K, V> = hashbrown::HashMap<K, V, RandomState>;
+#[cfg(feature = "private")]
+
+/// This hashmap has the uses an IdHasher
+pub type PlIdHashMap<K, V> = hashbrown::HashMap<K, V, IdBuildHasher>;
+#[cfg(feature = "private")]
+pub type PlHashSet<V> = hashbrown::HashSet<V, RandomState>;
+#[cfg(feature = "private")]
+pub type PlIndexMap<K, V> = indexmap::IndexMap<K, V, RandomState>;
+#[cfg(feature = "private")]
+pub type PlIndexSet<K> = indexmap::IndexSet<K, RandomState>;
+
 pub trait InitHashMaps {
     type HashMap;
 
@@ -81,3 +87,14 @@ impl<K, V> InitHashMaps for PlIndexMap<K, V> {
         Self::with_capacity_and_hasher(capacity, Default::default())
     }
 }
+impl<K, V> InitHashMaps for PlIdHashMap<K, V> {
+    type HashMap = Self;
+
+    fn new() -> Self::HashMap {
+        Self::with_capacity_and_hasher(0, Default::default())
+    }
+
+    fn with_capacity(capacity: usize) -> Self {
+        Self::with_capacity_and_hasher(capacity, Default::default())
+    }
+}