end-to-end arrow-based text entries

crates/re_arrow_store/src/store.rs

@@ -10,8 +10,8 @@ use nohash_hasher::IntMap;
 use parking_lot::RwLock;
 use re_format::{arrow, format_bytes, format_number};
 use re_log_types::{
-    ComponentName, ObjPath as EntityPath, ObjPathHash as EntityPathHash, TimeInt, TimeRange,
-    Timeline,
+    ComponentName, MsgId, ObjPath as EntityPath, ObjPathHash as EntityPathHash, TimeInt, TimePoint,
+    TimeRange, Timeline,
 };
 
 // --- Indices & offsets ---
@@ -200,6 +200,11 @@ pub struct DataStore {
     /// so that they can be properly deduplicated.
     pub(crate) cluster_comp_cache: IntMap<usize, RowIndex>,
 
+    /// Maps `MsgId`s to some metadata (just timepoints at the moment).
+    ///
+    /// `BTreeMap` because of garbage collection.
+    pub(crate) messages: BTreeMap<MsgId, TimePoint>,
+
     /// Maps an entity to its index, for a specific timeline.
     ///
     /// An index maps specific points in time to rows in component tables.
@@ -223,6 +228,7 @@ impl DataStore {
             cluster_key,
             config,
             cluster_comp_cache: Default::default(),
+            messages: Default::default(),
             indices: Default::default(),
             components: Default::default(),
             insert_id: 0,
@@ -328,6 +334,7 @@ impl std::fmt::Display for DataStore {
             cluster_key,
             config,
             cluster_comp_cache: _,
+            messages: _,
             indices,
             components,
             insert_id: _,

crates/re_arrow_store/src/store_write.rs

@@ -9,7 +9,7 @@ use parking_lot::RwLock;
 use re_log::{debug, trace};
 use re_log_types::{
     msg_bundle::{wrap_in_listarray, ComponentBundle, MsgBundle},
-    ComponentName, ObjPath as EntityPath, TimeInt, TimePoint, TimeRange, Timeline,
+    ComponentName, MsgId, ObjPath as EntityPath, TimeInt, TimePoint, TimeRange, Timeline,
 };
 
 use crate::{
@@ -70,7 +70,7 @@ impl DataStore {
         self.insert_id += 1;
 
         let MsgBundle {
-            msg_id: _,
+            msg_id,
             obj_path: ent_path,
             time_point,
             components,
@@ -152,6 +152,8 @@ impl DataStore {
             index.insert(&self.config, *time, &row_indices)?;
         }
 
+        self.messages.insert(*msg_id, time_point.clone());
+
         Ok(())
     }
 
@@ -313,6 +315,10 @@ impl DataStore {
             }
         }
     }
+
+    pub fn get_msg_metadata(&self, msg_id: &MsgId) -> Option<&TimePoint> {
+        self.messages.get(msg_id)
+    }
 }
 
 // --- Indices ---

crates/re_arrow_store/tests/data_store.rs

@@ -398,7 +398,7 @@ fn joint_df(cluster_key: ComponentName, bundles: &[(ComponentName, &MsgBundle)])
                 Series::try_from((
                     cluster_key.as_str(),
                     wrap_in_listarray(
-                        UInt64Array::from_vec((0..bundle.len() as u64).collect()).to_boxed(),
+                        UInt64Array::from_vec((0..bundle.row_len(0) as u64).collect()).to_boxed(),
                     )
                     .to_boxed(),
                 ))

crates/re_data_store/src/log_db.rs

@@ -2,7 +2,7 @@ use itertools::Itertools as _;
 use nohash_hasher::IntMap;
 
 use re_log_types::{
-    field_types::Instance,
+    field_types::{Instance, TextEntry},
     msg_bundle::{Component as _, MsgBundle},
     objects, ArrowMsg, BatchIndex, BeginRecordingMsg, DataMsg, DataPath, DataVec, LogMsg,
     LoggedData, MsgId, ObjPath, ObjPathHash, ObjTypePath, ObjectType, PathOp, PathOpMsg,
@@ -128,11 +128,24 @@ impl ObjDb {
     fn try_add_arrow_data_msg(&mut self, msg: &ArrowMsg) -> Result<(), Error> {
         let msg_bundle = MsgBundle::try_from(msg).map_err(Error::MsgBundleError)?;
 
-        // TODO(jleibs): Hack in a type so the UI treats these objects as visible
-        // This can go away once we determine object categories directly from the arrow table
-        self.types
-            .entry(msg_bundle.obj_path.obj_type_path().clone())
-            .or_insert(ObjectType::ArrowObject);
+        // Determine the kind of object we're looking at based on the components that have been
+        // uploaded _first_.
+        //
+        // TODO(cmc): That's an extension of the hack below, and will disappear at the same time
+        // and for the same reasons.
+        {
+            let obj_type = if msg_bundle.find_component(&TextEntry::name()).is_some() {
+                ObjectType::TextEntry
+            } else {
+                // TODO(jleibs): Hack in a type so the UI treats these objects as visible
+                // This can go away once we determine object categories directly from the arrow
+                // table
+                ObjectType::ArrowObject
+            };
+            self.types
+                .entry(msg_bundle.obj_path.obj_type_path().clone())
+                .or_insert(obj_type);
+        }
 
         self.register_obj_path(&msg_bundle.obj_path);
 

crates/re_log_types/src/field_types/mod.rs

@@ -19,6 +19,7 @@ mod quaternion;
 mod radius;
 mod rect;
 mod size;
+mod text_entry;
 
 pub use class_id::ClassId;
 pub use color::ColorRGBA;
@@ -31,10 +32,11 @@ pub use quaternion::Quaternion;
 pub use radius::Radius;
 pub use rect::Rect2D;
 pub use size::Size3D;
+pub use text_entry::TextEntry;
 
 lazy_static! {
     //TODO(john) actully use a run-time type registry
-    static ref FIELDS: [Field; 11] = [
+    static ref FIELDS: [Field; 12] = [
         <ColorRGBA as Component>::field(),
         <Instance as Component>::field(),
         <KeypointId as Component>::field(),
@@ -46,6 +48,7 @@ lazy_static! {
         <Radius as Component>::field(),
         <Rect2D as Component>::field(),
         <Size3D as Component>::field(),
+        <TextEntry as Component>::field(),
     ];
 }
 

crates/re_log_types/src/field_types/text_entry.rs

@@ -0,0 +1,48 @@
+use arrow2_convert::ArrowField;
+
+use crate::msg_bundle::Component;
+
+/// A text entry component, comprised of a text body and its log level.
+///
+/// ```
+/// use re_log_types::field_types::TextEntry;
+/// use arrow2_convert::field::ArrowField;
+/// use arrow2::datatypes::{DataType, Field};
+///
+/// assert_eq!(
+///     TextEntry::data_type(),
+///     DataType::Struct(vec![
+///         Field::new("body", DataType::Utf8, false),
+///         Field::new("level", DataType::Utf8, true),
+///     ])
+/// );
+/// ```
+#[derive(Clone, Debug, ArrowField, PartialEq, Eq)]
+pub struct TextEntry {
+    pub body: String,
+    pub level: Option<String>,
+}
+
+impl TextEntry {
+    #[inline]
+    pub fn new(body: impl Into<String>, level: Option<impl Into<String>>) -> Self {
+        Self {
+            body: body.into(),
+            level: level.map(Into::into),
+        }
+    }
+
+    #[inline]
+    pub fn from_body(body: impl Into<String>) -> Self {
+        Self {
+            body: body.into(),
+            level: None,
+        }
+    }
+}
+
+impl Component for TextEntry {
+    fn name() -> crate::ComponentName {
+        "rerun.text_entry".into()
+    }
+}

crates/re_log_types/src/msg_bundle.rs

@@ -214,7 +214,7 @@ impl MsgBundle {
         //
         // TODO(#440): support splats & remove this hack.
         this.components
-            .push(vec![msg_id; this.len()].try_into().unwrap());
+            .push(vec![msg_id; this.row_len(0)].try_into().unwrap());
 
         this
     }
@@ -242,16 +242,33 @@ impl MsgBundle {
         Ok(())
     }
 
-    /// Returns the length of the bundle, i.e. its _number of rows_.
-    pub fn len(&self) -> usize {
+    /// Returns the length of a specific row within the bundle, i.e. the row's _number of
+    /// instances_.
+    ///
+    /// Panics if `row_nr` is out of bounds.
+    pub fn row_len(&self, row_nr: usize) -> usize {
+        // TODO(#440): won't be able to pick any component randomly once we support splats!
         self.components.first().map_or(0, |bundle| {
             let offsets = bundle
                 .value
                 .as_any()
                 .downcast_ref::<ListArray<i32>>()
                 .unwrap()
                 .offsets();
-            (offsets[1] - offsets[0]) as usize
+            (offsets[row_nr + 1] - offsets[row_nr]) as usize
+        })
+    }
+
+    /// Returns the length of the bundle, i.e. its _number of rows_.
+    pub fn len(&self) -> usize {
+        // TODO(#440): won't be able to pick any component randomly once we support splats!
+        self.components.first().map_or(0, |bundle| {
+            bundle
+                .value
+                .as_any()
+                .downcast_ref::<ListArray<i32>>()
+                .unwrap()
+                .len()
         })
     }
 

crates/re_viewer/src/ui/view_text/scene.rs

@@ -1,13 +1,19 @@
+use re_arrow_store::TimeRange;
 use re_data_store::{query::visit_type_data_2, FieldName, ObjPath, TimeQuery};
-use re_log_types::{IndexHash, MsgId, ObjectType};
+use re_log_types::{
+    field_types::{self, Instance},
+    msg_bundle::Component,
+    IndexHash, MsgId, ObjectType,
+};
+use re_query::{range_entity_with_primary, QueryError};
 
 use crate::{ui::SceneQuery, ViewerContext};
 
 use super::ui::ViewTextFilters;
 
 // ---
 
-/// A single text entry.
+#[derive(Debug, Clone)]
 pub struct TextEntry {
     // props
     pub msg_id: MsgId,
@@ -18,6 +24,9 @@ pub struct TextEntry {
     // text entry
     pub level: Option<String>,
     pub body: String,
+
+    // temp
+    pub is_arrow: bool,
 }
 
 /// A text scene, with everything needed to render it.
@@ -37,6 +46,8 @@ impl SceneText {
         crate::profile_function!();
 
         self.load_text_entries(ctx, query, filters);
+
+        self.load_text_entries_arrow(ctx, query, filters);
     }
 
     fn load_text_entries(
@@ -87,6 +98,7 @@ impl SceneText {
                                 color: color.copied(),
                                 level: level.map(ToOwned::to_owned),
                                 body: body.clone(),
+                                is_arrow: false,
                             });
                         }
                     },
@@ -97,7 +109,75 @@ impl SceneText {
         // The most important order is the the `time` for whatever timeline we are on.
         // For a tie-breaker, we use MsgId as that is ordered by a high-resolution wall-time.
         crate::profile_scope!("sort");
-        self.text_entries
-            .sort_by_key(|entry| (entry.time, entry.msg_id));
+        self.text_entries.sort_by_key(|te| (te.time, te.msg_id));
+    }
+
+    fn load_text_entries_arrow(
+        &mut self,
+        ctx: &ViewerContext<'_>,
+        query: &SceneQuery<'_>,
+        filters: &ViewTextFilters,
+    ) {
+        crate::profile_function!();
+
+        let store = &ctx.log_db.obj_db.arrow_store;
+
+        for obj_path in query.obj_paths {
+            let ent_path = obj_path;
+
+            // Early filtering: if we're not showing it the view, there isn't much point
+            // in querying it to begin with... at least for now.
+            if !filters.is_obj_path_visible(ent_path) {
+                return;
+            }
+
+            let query = re_arrow_store::RangeQuery::new(
+                query.timeline,
+                TimeRange::new(i64::MIN.into(), i64::MAX.into()),
+            );
+
+            let components = [
+                Instance::name(),
+                MsgId::name(),
+                field_types::TextEntry::name(),
+                field_types::ColorRGBA::name(),
+            ];
+            let ent_views = range_entity_with_primary::<field_types::TextEntry, 4>(
+                store, &query, ent_path, components,
+            );
+
+            for (time, ent_view) in ent_views {
+                match ent_view.visit3(
+                    |_instance,
+                     text_entry: field_types::TextEntry,
+                     msg_id: Option<MsgId>,
+                     color: Option<field_types::ColorRGBA>| {
+                        let field_types::TextEntry { body, level } = text_entry;
+
+                        // Early filtering once more, see above.
+                        let is_visible = level
+                            .as_ref()
+                            .map_or(true, |lvl| filters.is_log_level_visible(lvl));
+
+                        if is_visible {
+                            self.text_entries.push(TextEntry {
+                                msg_id: msg_id.unwrap(), // always present
+                                obj_path: obj_path.clone(),
+                                time: time.as_i64(),
+                                color: color.map(|c| c.to_array()),
+                                level,
+                                body,
+                                is_arrow: true,
+                            });
+                        }
+                    },
+                ) {
+                    Ok(_) | Err(QueryError::PrimaryNotFound) => {}
+                    Err(err) => {
+                        re_log::error_once!("Unexpected error querying '{ent_path:?}': {err:?}");
+                    }
+                }
+            }
+        }
     }
 }