Primary caching 19 (final): de-statification (#4856)

- Quick sanity pass over all the intermediary locks and refcounts to
make sure we don't hold anything for longer than we need.
- Get rid of all static globals and let the caches live with their
associated stores in `EntityDb`.
- `CacheKey` no longer requires a `StoreId`.

---

- Fixes #4815 

---

Part of the primary caching series of PR (index search, joins,
deserialization):
- #4592
- #4593
- #4659
- #4680 
- #4681
- #4698
- #4711
- #4712
- #4721 
- #4726 
- #4773
- #4784
- #4785
- #4793
- #4800
- #4851
- #4852
- #4853
- #4856

crates/re_entity_db/Cargo.toml

@@ -20,16 +20,18 @@ all-features = true
 default = []
 
 ## Enable (de)serialization using serde.
-serde = ["dep:serde", "dep:rmp-serde", "re_log_types/serde"]
+serde = ["dep:serde", "dep:rmp-serde", "re_log_types/serde", "re_query/serde"]
 
 
 [dependencies]
 re_data_store.workspace = true
 re_format.workspace = true
 re_int_histogram.workspace = true
+re_log.workspace = true
 re_log_encoding = { workspace = true, optional = true }
 re_log_types.workspace = true
-re_log.workspace = true
+re_query.workspace = true
+re_query_cache.workspace = true
 re_smart_channel.workspace = true
 re_tracing.workspace = true
 re_types_core.workspace = true

crates/re_entity_db/src/entity_db.rs

@@ -110,11 +110,20 @@ pub struct EntityDb {
     /// Stores all components for all entities for all timelines.
     data_store: DataStore,
 
+    /// Query caches for the data in [`Self::data_store`].
+    query_caches: re_query_cache::Caches,
+
     stats: IngestionStatistics,
 }
 
 impl EntityDb {
     pub fn new(store_id: StoreId) -> Self {
+        let data_store = re_data_store::DataStore::new(
+            store_id.clone(),
+            InstanceKey::name(),
+            DataStoreConfig::default(),
+        );
+        let query_caches = re_query_cache::Caches::new(&data_store);
         Self {
             store_id: store_id.clone(),
             data_source: None,
@@ -123,11 +132,8 @@ impl EntityDb {
             entity_path_from_hash: Default::default(),
             times_per_timeline: Default::default(),
             tree: crate::EntityTree::root(),
-            data_store: re_data_store::DataStore::new(
-                store_id.clone(),
-                InstanceKey::name(),
-                DataStoreConfig::default(),
-            ),
+            data_store,
+            query_caches,
             stats: IngestionStatistics::new(store_id),
         }
     }
@@ -175,6 +181,11 @@ impl EntityDb {
         self.store_info().map(|ri| &ri.application_id)
     }
 
+    #[inline]
+    pub fn query_caches(&self) -> &re_query_cache::Caches {
+        &self.query_caches
+    }
+
     #[inline]
     pub fn store(&self) -> &DataStore {
         &self.data_store
@@ -315,6 +326,7 @@ impl EntityDb {
         // and/or pending clears.
         let original_store_events = &[store_event];
         self.times_per_timeline.on_events(original_store_events);
+        self.query_caches.on_events(original_store_events);
         let clear_cascade = self.tree.on_store_additions(original_store_events);
 
         // Second-pass: update the [`DataStore`] by applying the [`ClearCascade`].
@@ -323,6 +335,7 @@ impl EntityDb {
         // notified of, again!
         let new_store_events = self.on_clear_cascade(clear_cascade);
         self.times_per_timeline.on_events(&new_store_events);
+        self.query_caches.on_events(&new_store_events);
         let clear_cascade = self.tree.on_store_additions(&new_store_events);
 
         // Clears don't affect `Clear` components themselves, therefore we cannot have recursive
@@ -476,10 +489,12 @@ impl EntityDb {
             times_per_timeline,
             tree,
             data_store: _,
+            query_caches,
             stats: _,
         } = self;
 
         times_per_timeline.on_events(store_events);
+        query_caches.on_events(store_events);
 
         let store_events = store_events.iter().collect_vec();
         let compacted = CompactedStoreEvents::new(&store_events);

crates/re_entity_db/src/entity_properties.rs

@@ -1,7 +1,7 @@
+use std::fmt::Formatter;
+
 #[cfg(feature = "serde")]
 use re_log_types::EntityPath;
-use re_log_types::TimeInt;
-use std::fmt::Formatter;
 
 #[cfg(feature = "serde")]
 use crate::EditableAutoValue;
@@ -95,7 +95,7 @@ impl FromIterator<(EntityPath, EntityProperties)> for EntityPropertyMap {
 #[cfg_attr(feature = "serde", serde(default))]
 pub struct EntityProperties {
     pub visible: bool,
-    pub visible_history: ExtraQueryHistory,
+    pub visible_history: re_query::ExtraQueryHistory,
     pub interactive: bool,
 
     /// What kind of color mapping should be applied (none, map, texture, transfer..)?
@@ -142,7 +142,7 @@ impl Default for EntityProperties {
     fn default() -> Self {
         Self {
             visible: true,
-            visible_history: ExtraQueryHistory::default(),
+            visible_history: re_query::ExtraQueryHistory::default(),
             interactive: true,
             color_mapper: EditableAutoValue::default(),
             pinhole_image_plane_distance: EditableAutoValue::default(),
@@ -269,105 +269,6 @@ impl EntityProperties {
 
 // ----------------------------------------------------------------------------
 
-/// One of the boundaries of the visible history.
-///
-/// For [`VisibleHistoryBoundary::RelativeToTimeCursor`] and [`VisibleHistoryBoundary::Absolute`],
-/// the value are either nanos or frames, depending on the type of timeline.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
-pub enum VisibleHistoryBoundary {
-    /// Boundary is a value relative to the time cursor
-    RelativeToTimeCursor(i64),
-
-    /// Boundary is an absolute value
-    Absolute(i64),
-
-    /// The boundary extends to infinity.
-    Infinite,
-}
-
-impl VisibleHistoryBoundary {
-    /// Value when the boundary is set to the current time cursor.
-    pub const AT_CURSOR: Self = Self::RelativeToTimeCursor(0);
-}
-
-impl Default for VisibleHistoryBoundary {
-    fn default() -> Self {
-        Self::AT_CURSOR
-    }
-}
-
-/// Visible history bounds.
-#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
-pub struct VisibleHistory {
-    /// Low time boundary.
-    pub from: VisibleHistoryBoundary,
-
-    /// High time boundary.
-    pub to: VisibleHistoryBoundary,
-}
-
-impl VisibleHistory {
-    /// Value with the visible history feature is disabled.
-    pub const OFF: Self = Self {
-        from: VisibleHistoryBoundary::AT_CURSOR,
-        to: VisibleHistoryBoundary::AT_CURSOR,
-    };
-
-    pub const ALL: Self = Self {
-        from: VisibleHistoryBoundary::Infinite,
-        to: VisibleHistoryBoundary::Infinite,
-    };
-
-    pub fn from(&self, cursor: TimeInt) -> TimeInt {
-        match self.from {
-            VisibleHistoryBoundary::Absolute(value) => TimeInt::from(value),
-            VisibleHistoryBoundary::RelativeToTimeCursor(value) => cursor + TimeInt::from(value),
-            VisibleHistoryBoundary::Infinite => TimeInt::MIN,
-        }
-    }
-
-    pub fn to(&self, cursor: TimeInt) -> TimeInt {
-        match self.to {
-            VisibleHistoryBoundary::Absolute(value) => TimeInt::from(value),
-            VisibleHistoryBoundary::RelativeToTimeCursor(value) => cursor + TimeInt::from(value),
-            VisibleHistoryBoundary::Infinite => TimeInt::MAX,
-        }
-    }
-}
-
-/// When showing an entity in the history view, add this much history to it.
-#[derive(Clone, Copy, Default, Debug, PartialEq, Eq)]
-#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
-#[cfg_attr(feature = "serde", serde(default))]
-pub struct ExtraQueryHistory {
-    /// Is the feature enabled?
-    pub enabled: bool,
-
-    /// Visible history settings for time timelines
-    pub nanos: VisibleHistory,
-
-    /// Visible history settings for frame timelines
-    pub sequences: VisibleHistory,
-}
-
-impl ExtraQueryHistory {
-    /// Multiply/and these together.
-    #[allow(dead_code)]
-    fn with_child(&self, child: &Self) -> Self {
-        if child.enabled {
-            *child
-        } else if self.enabled {
-            *self
-        } else {
-            Self::default()
-        }
-    }
-}
-
-// ----------------------------------------------------------------------------
-
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 pub enum Colormap {

crates/re_entity_db/src/lib.rs

@@ -30,6 +30,8 @@ pub(crate) use self::entity_tree::{ClearCascade, CompactedStoreEvents};
 use re_log_types::DataTableError;
 pub use re_log_types::{EntityPath, EntityPathPart, TimeInt, Timeline};
 
+pub use re_query::{ExtraQueryHistory, VisibleHistory, VisibleHistoryBoundary};
+
 #[cfg(feature = "serde")]
 pub use blueprint::components::EntityPropertiesComponent;
 #[cfg(feature = "serde")]

crates/re_query/Cargo.toml

@@ -19,10 +19,13 @@ all-features = true
 [features]
 default = []
 
+## Enable (de)serialization using serde.
+serde = ["dep:serde", "dep:rmp-serde"]
+
+
 [dependencies]
 # Rerun dependencies:
 re_data_store.workspace = true
-re_entity_db.workspace = true
 re_format = { workspace = true, features = ["arrow"] }
 re_log_types.workspace = true
 re_types_core.workspace = true
@@ -34,6 +37,8 @@ arrow2.workspace = true
 backtrace.workspace = true
 document-features.workspace = true
 itertools = { workspace = true }
+rmp-serde = { workspace = true, optional = true }
+serde = { workspace = true, features = ["derive", "rc"], optional = true }
 smallvec.workspace = true
 thiserror.workspace = true
 

crates/re_query/src/lib.rs

@@ -8,7 +8,9 @@ mod util;
 pub use self::archetype_view::{ArchetypeView, ComponentWithInstances};
 pub use self::query::{get_component_with_instances, query_archetype};
 pub use self::range::range_archetype;
-pub use self::util::query_archetype_with_history;
+pub use self::util::{
+    query_archetype_with_history, ExtraQueryHistory, VisibleHistory, VisibleHistoryBoundary,
+};
 
 // Used for doc-tests
 #[doc(hidden)]