diff --git a/CHANGELOG.md b/CHANGELOG.md index 8995c1fff69d..fda7ce402b41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ * 3D transform APIs: Previously, the transform component was represented as one of several variants (an Arrow union, `enum` in Rust) depending on how the transform was expressed. Instead, there are now several components for translation/scale/rotation/matrices that can live side-by-side in the 3D transform archetype. * Python: `NV12/YUY2` are now logged with the new `ImageChromaDownsampled` * [`ImageEncoded`](https://rerun.io/docs/reference/types/archetypes/image_encoded?speculative-link):s `format` parameter has been replaced with `media_type` (MIME) -* [`DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) is no longer encoded as a tensor, and expects its shape in `[width, height]` order +* [`DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) and [`SegmentationImage`](https://rerun.io/docs/reference/types/archetypes/segmentation_image) are no longer encoded as a tensors, and expects its shape in `[width, height]` order 🧳 Migration guide: http://rerun.io/docs/reference/migration/migration-0-18?speculative-link diff --git a/crates/store/re_types/definitions/rerun/archetypes/depth_image.fbs b/crates/store/re_types/definitions/rerun/archetypes/depth_image.fbs index 6a9829232c77..55b9f75e3834 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/depth_image.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/depth_image.fbs @@ -1,10 +1,9 @@ namespace rerun.archetypes; -/// A depth image. +/// A depth image, i.e. as captured by a depth camera. /// -/// The shape of the [components.TensorData] must be mappable to an `HxW` tensor. -/// Each pixel corresponds to a depth value in units specified by `meter`. +/// Each pixel corresponds to a depth value in units specified by [components.DepthMeter]. /// /// \cpp Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, /// \cpp data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays. diff --git a/crates/store/re_types/definitions/rerun/archetypes/segmentation_image.fbs b/crates/store/re_types/definitions/rerun/archetypes/segmentation_image.fbs index 11acce7333ad..0e1c7a932328 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/segmentation_image.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/segmentation_image.fbs @@ -3,15 +3,11 @@ namespace rerun.archetypes; /// An image made up of integer [components.ClassId]s. /// -/// The shape of the [components.TensorData] must be mappable to an `HxW` tensor. /// Each pixel corresponds to a [components.ClassId] that will be mapped to a color based on annotation context. /// /// In the case of floating point images, the label will be looked up based on rounding to the nearest /// integer value. /// -/// Leading and trailing unit-dimensions are ignored, so that -/// `1x640x480x1` is treated as a `640x480` image. -/// /// See also [archetypes.AnnotationContext] to associate each class with a color and a label. /// /// \cpp Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, @@ -27,8 +23,14 @@ table SegmentationImage ( ) { // --- Required --- - /// The image data. Should always be a 2-dimensional tensor. - data: rerun.components.TensorData ("attr.rerun.component_required", order: 1000); + /// The raw image data. + data: rerun.components.Blob ("attr.rerun.component_required", order: 1000); + + /// The size of the image. + resolution: rerun.components.Resolution2D ("attr.rerun.component_required", order: 1500); + + /// The data type of the segmentation image data (U16, U32, …). + data_type: rerun.components.ChannelDataType ("attr.rerun.component_required", order: 2000); // --- Optional --- diff --git a/crates/store/re_types/definitions/rerun/components/color_model.fbs b/crates/store/re_types/definitions/rerun/components/color_model.fbs index 1696a0367c05..751bddc940d8 100644 --- a/crates/store/re_types/definitions/rerun/components/color_model.fbs +++ b/crates/store/re_types/definitions/rerun/components/color_model.fbs @@ -5,7 +5,9 @@ namespace rerun.components; /// Specified what color components are present in an [archetypes.Image]. /// /// This combined with [components.ChannelDataType] determines the pixel format of an image. -enum ColorModel: byte { +enum ColorModel: byte ( + "attr.docs.unreleased" +) { /// Grayscale luminance intencity/brightness/value, sometimes called `Y` L (default), diff --git a/crates/store/re_types/definitions/rerun/components/resolution2d.fbs b/crates/store/re_types/definitions/rerun/components/resolution2d.fbs index 210e8d30cd80..cbedbef5490b 100644 --- a/crates/store/re_types/definitions/rerun/components/resolution2d.fbs +++ b/crates/store/re_types/definitions/rerun/components/resolution2d.fbs @@ -2,6 +2,7 @@ namespace rerun.components; /// The width and height of a 2D image. struct Resolution2D ( + "attr.docs.unreleased", "attr.python.aliases": "npt.NDArray[np.int], Sequence[int], Tuple[int, int]", "attr.python.array_aliases": "npt.NDArray[np.int], Sequence[int]", "attr.rust.derive": "Default, Copy, PartialEq, Eq, Hash, bytemuck::Pod, bytemuck::Zeroable", diff --git a/crates/store/re_types/src/archetypes/depth_image.rs b/crates/store/re_types/src/archetypes/depth_image.rs index 7412abd61d20..7d7b90628de1 100644 --- a/crates/store/re_types/src/archetypes/depth_image.rs +++ b/crates/store/re_types/src/archetypes/depth_image.rs @@ -18,10 +18,9 @@ use ::re_types_core::SerializationResult; use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; use ::re_types_core::{DeserializationError, DeserializationResult}; -/// **Archetype**: A depth image. +/// **Archetype**: A depth image, i.e. as captured by a depth camera. /// -/// The shape of the [`components::TensorData`][crate::components::TensorData] must be mappable to an `HxW` tensor. -/// Each pixel corresponds to a depth value in units specified by `meter`. +/// Each pixel corresponds to a depth value in units specified by [`components::DepthMeter`][crate::components::DepthMeter]. /// /// ## Example /// diff --git a/crates/store/re_types/src/archetypes/segmentation_image.rs b/crates/store/re_types/src/archetypes/segmentation_image.rs index f4bce07db378..a73efa9a6284 100644 --- a/crates/store/re_types/src/archetypes/segmentation_image.rs +++ b/crates/store/re_types/src/archetypes/segmentation_image.rs @@ -20,15 +20,11 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// **Archetype**: An image made up of integer [`components::ClassId`][crate::components::ClassId]s. /// -/// The shape of the [`components::TensorData`][crate::components::TensorData] must be mappable to an `HxW` tensor. /// Each pixel corresponds to a [`components::ClassId`][crate::components::ClassId] that will be mapped to a color based on annotation context. /// /// In the case of floating point images, the label will be looked up based on rounding to the nearest /// integer value. /// -/// Leading and trailing unit-dimensions are ignored, so that -/// `1x640x480x1` is treated as a `640x480` image. -/// /// See also [`archetypes::AnnotationContext`][crate::archetypes::AnnotationContext] to associate each class with a color and a label. /// /// ## Example @@ -70,8 +66,14 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// #[derive(Clone, Debug, PartialEq)] pub struct SegmentationImage { - /// The image data. Should always be a 2-dimensional tensor. - pub data: crate::components::TensorData, + /// The raw image data. + pub data: crate::components::Blob, + + /// The size of the image. + pub resolution: crate::components::Resolution2D, + + /// The data type of the segmentation image data (U16, U32, …). + pub data_type: crate::components::ChannelDataType, /// Opacity of the image, useful for layering the segmentation image on top of another image. /// @@ -88,20 +90,30 @@ impl ::re_types_core::SizeBytes for SegmentationImage { #[inline] fn heap_size_bytes(&self) -> u64 { self.data.heap_size_bytes() + + self.resolution.heap_size_bytes() + + self.data_type.heap_size_bytes() + self.opacity.heap_size_bytes() + self.draw_order.heap_size_bytes() } #[inline] fn is_pod() -> bool { - ::is_pod() + ::is_pod() + && ::is_pod() + && ::is_pod() && >::is_pod() && >::is_pod() } } -static REQUIRED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = - once_cell::sync::Lazy::new(|| ["rerun.components.TensorData".into()]); +static REQUIRED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 3usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.Blob".into(), + "rerun.components.Resolution2D".into(), + "rerun.components.ChannelDataType".into(), + ] + }); static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = once_cell::sync::Lazy::new(|| ["rerun.components.SegmentationImageIndicator".into()]); @@ -114,10 +126,12 @@ static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 2usize]> = ] }); -static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 4usize]> = +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 6usize]> = once_cell::sync::Lazy::new(|| { [ - "rerun.components.TensorData".into(), + "rerun.components.Blob".into(), + "rerun.components.Resolution2D".into(), + "rerun.components.ChannelDataType".into(), "rerun.components.SegmentationImageIndicator".into(), "rerun.components.Opacity".into(), "rerun.components.DrawOrder".into(), @@ -125,8 +139,8 @@ static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 4usize]> = }); impl SegmentationImage { - /// The total number of components in the archetype: 1 required, 1 recommended, 2 optional - pub const NUM_COMPONENTS: usize = 4usize; + /// The total number of components in the archetype: 3 required, 1 recommended, 2 optional + pub const NUM_COMPONENTS: usize = 6usize; } /// Indicator component for the [`SegmentationImage`] [`::re_types_core::Archetype`] @@ -183,10 +197,10 @@ impl ::re_types_core::Archetype for SegmentationImage { .collect(); let data = { let array = arrays_by_name - .get("rerun.components.TensorData") + .get("rerun.components.Blob") .ok_or_else(DeserializationError::missing_data) .with_context("rerun.archetypes.SegmentationImage#data")?; - ::from_arrow_opt(&**array) + ::from_arrow_opt(&**array) .with_context("rerun.archetypes.SegmentationImage#data")? .into_iter() .next() @@ -194,6 +208,32 @@ impl ::re_types_core::Archetype for SegmentationImage { .ok_or_else(DeserializationError::missing_data) .with_context("rerun.archetypes.SegmentationImage#data")? }; + let resolution = { + let array = arrays_by_name + .get("rerun.components.Resolution2D") + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.SegmentationImage#resolution")?; + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.SegmentationImage#resolution")? + .into_iter() + .next() + .flatten() + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.SegmentationImage#resolution")? + }; + let data_type = { + let array = arrays_by_name + .get("rerun.components.ChannelDataType") + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.SegmentationImage#data_type")?; + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.SegmentationImage#data_type")? + .into_iter() + .next() + .flatten() + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.SegmentationImage#data_type")? + }; let opacity = if let Some(array) = arrays_by_name.get("rerun.components.Opacity") { ::from_arrow_opt(&**array) .with_context("rerun.archetypes.SegmentationImage#opacity")? @@ -214,6 +254,8 @@ impl ::re_types_core::Archetype for SegmentationImage { }; Ok(Self { data, + resolution, + data_type, opacity, draw_order, }) @@ -227,6 +269,8 @@ impl ::re_types_core::AsComponents for SegmentationImage { [ Some(Self::indicator()), Some((&self.data as &dyn ComponentBatch).into()), + Some((&self.resolution as &dyn ComponentBatch).into()), + Some((&self.data_type as &dyn ComponentBatch).into()), self.opacity .as_ref() .map(|comp| (comp as &dyn ComponentBatch).into()), @@ -243,9 +287,15 @@ impl ::re_types_core::AsComponents for SegmentationImage { impl SegmentationImage { /// Create a new `SegmentationImage`. #[inline] - pub fn new(data: impl Into) -> Self { + pub fn new( + data: impl Into, + resolution: impl Into, + data_type: impl Into, + ) -> Self { Self { data: data.into(), + resolution: resolution.into(), + data_type: data_type.into(), opacity: None, draw_order: None, } diff --git a/crates/store/re_types/src/archetypes/segmentation_image_ext.rs b/crates/store/re_types/src/archetypes/segmentation_image_ext.rs index 77ea42f495f6..dfa393936605 100644 --- a/crates/store/re_types/src/archetypes/segmentation_image_ext.rs +++ b/crates/store/re_types/src/archetypes/segmentation_image_ext.rs @@ -1,5 +1,6 @@ use crate::{ - datatypes::TensorData, + components::{ChannelDataType, Resolution2D}, + datatypes::{Blob, TensorBuffer, TensorData}, image::{find_non_empty_dim_indices, ImageConstructionError}, }; @@ -16,62 +17,47 @@ impl SegmentationImage { where >::Error: std::error::Error, { - let mut data: TensorData = data + let tensor_data: TensorData = data .try_into() .map_err(ImageConstructionError::TensorDataConversion)?; - let non_empty_dim_inds = find_non_empty_dim_indices(&data.shape); + let non_empty_dim_inds = find_non_empty_dim_indices(&tensor_data.shape); - match non_empty_dim_inds.len() { - 2 => { - assign_if_none(&mut data.shape[non_empty_dim_inds[0]].name, "height"); - assign_if_none(&mut data.shape[non_empty_dim_inds[1]].name, "width"); + if non_empty_dim_inds.len() != 2 { + return Err(ImageConstructionError::BadImageShape(tensor_data.shape)); + } + + let (blob, data_type) = match tensor_data.buffer { + TensorBuffer::U8(buffer) => (Blob(buffer), ChannelDataType::U8), + TensorBuffer::U16(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::U16), + TensorBuffer::U32(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::U32), + TensorBuffer::U64(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::U64), + TensorBuffer::I8(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::I8), + TensorBuffer::I16(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::I16), + TensorBuffer::I32(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::I32), + TensorBuffer::I64(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::I64), + TensorBuffer::F16(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::F16), + TensorBuffer::F32(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::F32), + TensorBuffer::F64(buffer) => (Blob(buffer.cast_to_u8()), ChannelDataType::F64), + TensorBuffer::Nv12(_) | TensorBuffer::Yuy2(_) => { + return Err(ImageConstructionError::ChromaDownsamplingNotSupported); } - _ => return Err(ImageConstructionError::BadImageShape(data.shape)), }; + let (height, width) = ( + &tensor_data.shape[non_empty_dim_inds[0]], + &tensor_data.shape[non_empty_dim_inds[1]], + ); + let height = height.size as u32; + let width = width.size as u32; + let resolution = Resolution2D::from([width, height]); + Ok(Self { - data: data.into(), + data: blob.into(), + resolution, + data_type, draw_order: None, opacity: None, }) } } - -fn assign_if_none(name: &mut Option<::re_types_core::ArrowString>, new_name: &str) { - if name.is_none() { - *name = Some(new_name.into()); - } -} - -// ---------------------------------------------------------------------------- -// Make it possible to create an ArrayView directly from an Image. - -macro_rules! forward_array_views { - ($type:ty, $alias:ty) => { - impl<'a> TryFrom<&'a $alias> for ::ndarray::ArrayViewD<'a, $type> { - type Error = crate::tensor_data::TensorCastError; - - #[inline] - fn try_from(value: &'a $alias) -> Result { - (&value.data.0).try_into() - } - } - }; -} - -forward_array_views!(u8, SegmentationImage); -forward_array_views!(u16, SegmentationImage); -forward_array_views!(u32, SegmentationImage); -forward_array_views!(u64, SegmentationImage); - -forward_array_views!(i8, SegmentationImage); -forward_array_views!(i16, SegmentationImage); -forward_array_views!(i32, SegmentationImage); -forward_array_views!(i64, SegmentationImage); - -forward_array_views!(half::f16, SegmentationImage); -forward_array_views!(f32, SegmentationImage); -forward_array_views!(f64, SegmentationImage); - -// ---------------------------------------------------------------------------- diff --git a/crates/store/re_types/tests/segmentation_image.rs b/crates/store/re_types/tests/segmentation_image.rs index 4af0187bfa66..89f177d5311c 100644 --- a/crates/store/re_types/tests/segmentation_image.rs +++ b/crates/store/re_types/tests/segmentation_image.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use re_types::{ archetypes::SegmentationImage, - datatypes::{TensorBuffer, TensorData, TensorDimension}, + components::{ChannelDataType, Resolution2D}, Archetype as _, AsComponents as _, }; @@ -11,20 +11,9 @@ mod util; #[test] fn segmentation_image_roundtrip() { let all_expected = [SegmentationImage { - data: TensorData { - shape: vec![ - TensorDimension { - size: 2, - name: Some("height".into()), - }, - TensorDimension { - size: 3, - name: Some("width".into()), - }, - ], - buffer: TensorBuffer::U8(vec![1, 2, 3, 4, 5, 6].into()), - } - .into(), + data: vec![1, 2, 3, 4, 5, 6].into(), + resolution: Resolution2D::new(3, 2), + data_type: ChannelDataType::U8, draw_order: None, opacity: None, }]; @@ -37,7 +26,7 @@ fn segmentation_image_roundtrip() { .to_arrow() .unwrap()]; - let expected_extensions: HashMap<_, _> = [("data", vec!["rerun.components.TensorData"])].into(); + let expected_extensions: HashMap<_, _> = [("data", vec!["rerun.components.Blob"])].into(); for (expected, serialized) in all_expected.into_iter().zip(all_arch_serialized) { for (field, array) in &serialized { diff --git a/crates/viewer/re_space_view_spatial/src/ui.rs b/crates/viewer/re_space_view_spatial/src/ui.rs index 91299dd812f6..217b49b584f1 100644 --- a/crates/viewer/re_space_view_spatial/src/ui.rs +++ b/crates/viewer/re_space_view_spatial/src/ui.rs @@ -457,7 +457,7 @@ pub fn picking( depth_images.images.iter(), // images.images.iter(), // TODO(#6386) images_encoded.images.iter(), - // segmentation_images.images.iter(), // TODO(#6386) + segmentation_images.images.iter(), ) .find(|i| i.ent_path == instance_path.entity_path) { @@ -492,18 +492,7 @@ pub fn picking( } }) } else { - let meaning = if segmentation_images - .images - .iter() - .any(|i| i.ent_path == instance_path.entity_path) - { - TensorDataMeaning::ClassId - } else if is_depth_cloud - || depth_images - .images - .iter() - .any(|i| i.ent_path == instance_path.entity_path) - { + let meaning = if is_depth_cloud { TensorDataMeaning::Depth } else { TensorDataMeaning::Unknown diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/depth_images.rs b/crates/viewer/re_space_view_spatial/src/visualizers/depth_images.rs index 900b155d9cec..ceae1b8e1254 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/depth_images.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/depth_images.rs @@ -64,15 +64,20 @@ impl DepthImageVisualizer { let meaning = TensorDataMeaning::Depth; for data in images { - let depth_meter = data.depth_meter.unwrap_or_else(|| self.fallback_for(ctx)); - let mut image = data.image.clone(); + let DepthImageComponentData { + mut image, + depth_meter, + fill_ratio, + } = data; + + let depth_meter = depth_meter.unwrap_or_else(|| self.fallback_for(ctx)); // All depth images must have a colormap: image.colormap = Some(image.colormap.unwrap_or_else(|| self.fallback_for(ctx))); if is_3d_view { if let Some(parent_pinhole_path) = transforms.parent_pinhole(entity_path) { - let fill_ratio = data.fill_ratio.unwrap_or_default(); + let fill_ratio = fill_ratio.unwrap_or_default(); // NOTE: we don't pass in `world_from_obj` because this corresponds to the // transform of the projection plane, which is of no use to us here. diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/segmentation_images.rs b/crates/viewer/re_space_view_spatial/src/visualizers/segmentation_images.rs index ca95bbd2b5f9..e1d571116c81 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/segmentation_images.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/segmentation_images.rs @@ -1,27 +1,25 @@ use itertools::Itertools as _; -use re_chunk_store::RowId; -use re_log_types::TimeInt; -use re_query::range_zip_1x1; -use re_space_view::diff_component_filter; use re_types::{ archetypes::SegmentationImage, - components::{DrawOrder, Opacity, TensorData}, + components::{self, DrawOrder, Opacity}, tensor_data::TensorDataMeaning, }; use re_viewer_context::{ - ApplicableEntities, IdentifiedViewSystem, QueryContext, SpaceViewClass, + ApplicableEntities, IdentifiedViewSystem, ImageInfo, QueryContext, SpaceViewClass, SpaceViewSystemExecutionError, TypedComponentFallbackProvider, ViewContext, ViewContextCollection, ViewQuery, VisualizableEntities, VisualizableFilterContext, - VisualizerAdditionalApplicabilityFilter, VisualizerQueryInfo, VisualizerSystem, + VisualizerQueryInfo, VisualizerSystem, }; use crate::{ - ui::SpatialSpaceViewState, view_kind::SpatialSpaceViewKind, - visualizers::filter_visualizable_2d_entities, PickableImageRect, SpatialSpaceView2D, + ui::SpatialSpaceViewState, + view_kind::SpatialSpaceViewKind, + visualizers::{filter_visualizable_2d_entities, textured_rect_from_image}, + PickableImageRect, SpatialSpaceView2D, }; -use super::{bounding_box_for_textured_rect, textured_rect_from_tensor, SpatialViewVisualizerData}; +use super::{bounding_box_for_textured_rect, SpatialViewVisualizerData}; pub struct SegmentationImageVisualizer { pub data: SpatialViewVisualizerData, @@ -37,11 +35,9 @@ impl Default for SegmentationImageVisualizer { } } -struct SegmentationImageComponentData<'a> { - index: (TimeInt, RowId), - - tensor: &'a TensorData, - opacity: Option<&'a Opacity>, +struct SegmentationImageComponentData { + image: ImageInfo, + opacity: Option, } impl IdentifiedViewSystem for SegmentationImageVisualizer { @@ -50,25 +46,11 @@ impl IdentifiedViewSystem for SegmentationImageVisualizer { } } -struct SegmentationImageVisualizerEntityFilter; - -impl VisualizerAdditionalApplicabilityFilter for SegmentationImageVisualizerEntityFilter { - fn update_applicability(&mut self, event: &re_chunk_store::ChunkStoreEvent) -> bool { - diff_component_filter(event, |tensor: &re_types::components::TensorData| { - tensor.is_shaped_like_an_image() - }) - } -} - impl VisualizerSystem for SegmentationImageVisualizer { fn visualizer_query_info(&self) -> VisualizerQueryInfo { VisualizerQueryInfo::from_archetype::() } - fn applicability_filter(&self) -> Option> { - Some(Box::new(SegmentationImageVisualizerEntityFilter)) - } - fn filter_visualizable_entities( &self, entities: ApplicableEntities, @@ -98,53 +80,63 @@ impl VisualizerSystem for SegmentationImageVisualizer { let entity_path = ctx.target_entity_path; let resolver = ctx.recording().resolver(); - let tensors = match results.get_required_component_dense::(resolver) { - Some(tensors) => tensors?, + let blobs = match results.get_required_component_dense::(resolver) + { + Some(blobs) => blobs?, + _ => return Ok(()), + }; + let data_types = match results + .get_required_component_dense::(resolver) + { + Some(data_types) => data_types?, + _ => return Ok(()), + }; + let resolutions = match results + .get_required_component_dense::(resolver) + { + Some(resolutions) => resolutions?, _ => return Ok(()), }; let opacity = results.get_or_empty_dense(resolver)?; - let data = range_zip_1x1(tensors.range_indexed(), opacity.range_indexed()) - .filter_map(|(&index, tensors, opacity)| { - tensors - .first() - .map(|tensor| SegmentationImageComponentData { - index, - tensor, - opacity: opacity.and_then(|opacity| opacity.first()), - }) - }); + let data = re_query::range_zip_1x3( + blobs.range_indexed(), + data_types.range_indexed(), + resolutions.range_indexed(), + opacity.range_indexed(), + ) + .filter_map(|(&index, blobs, data_type, resolution, opacity)| { + let blob = blobs.first()?; + Some(SegmentationImageComponentData { + image: ImageInfo { + blob_row_id: index.1, + blob: blob.0.clone(), + resolution: first_copied(resolution)?.0 .0, + color_model: None, + data_type: first_copied(data_type)?, + colormap: None, + }, + opacity: first_copied(opacity), + }) + }); let meaning = TensorDataMeaning::ClassId; for data in data { - if !data.tensor.is_shaped_like_an_image() { - continue; - } + let SegmentationImageComponentData { image, opacity } = data; - let tensor_data_row_id = data.index.1; - let tensor = data.tensor.0.clone(); - - // TODO(andreas): colormap is only available for depth images right now. - let colormap = None; - - let opacity = data - .opacity - .copied() - .unwrap_or_else(|| self.fallback_for(ctx)); + let opacity = opacity.unwrap_or_else(|| self.fallback_for(ctx)); let multiplicative_tint = re_renderer::Rgba::from_white_alpha(opacity.0.clamp(0.0, 1.0)); - if let Some(textured_rect) = textured_rect_from_tensor( + if let Some(textured_rect) = textured_rect_from_image( ctx.viewer_ctx, entity_path, spatial_ctx, - tensor_data_row_id, - &tensor, + &image, meaning, multiplicative_tint, - colormap, ) { // Only update the bounding box if this is a 2D space view. // This is avoids a cyclic relationship where the image plane grows @@ -162,12 +154,12 @@ impl VisualizerSystem for SegmentationImageVisualizer { self.images.push(PickableImageRect { ent_path: entity_path.clone(), - row_id: tensor_data_row_id, + row_id: image.blob_row_id, textured_rect, - meaning: TensorDataMeaning::ClassId, + meaning, depth_meter: None, - tensor: Some(tensor), - image: None, + tensor: None, + image: Some(image), }); } } @@ -256,3 +248,7 @@ impl TypedComponentFallbackProvider for SegmentationImageVisualizer { } re_viewer_context::impl_component_fallback_provider!(SegmentationImageVisualizer => [DrawOrder, Opacity]); + +fn first_copied(slice: Option<&[T]>) -> Option { + slice.and_then(|element| element.first()).copied() +} diff --git a/docs/content/reference/migration/migration-0-18.md b/docs/content/reference/migration/migration-0-18.md index d5eb00947f49..9d12565a1f2a 100644 --- a/docs/content/reference/migration/migration-0-18.md +++ b/docs/content/reference/migration/migration-0-18.md @@ -7,8 +7,8 @@ NOTE! Rerun 0.18 has not yet been released ## ⚠️ Breaking changes -### [`DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) -The `DepthImage` archetype used to be encoded as a tensor, but now it is encoded as a blob of bytes, a resolution, and a datatype. +### [`DepthImage`](https://rerun.io/docs/reference/types/archetypes/depth_image) and [`SegmentationImage`](https://rerun.io/docs/reference/types/archetypes/segmentation_image) +The `DepthImage` and `SegmentationImage` archetypes used to be encoded as a tensor, but now it is encoded as a blob of bytes, a resolution, and a datatype. The constructs have changed to now expect the shape in `[width, height]` order. diff --git a/docs/content/reference/types/archetypes.md b/docs/content/reference/types/archetypes.md index 1a9c3d0ad744..916c9205a652 100644 --- a/docs/content/reference/types/archetypes.md +++ b/docs/content/reference/types/archetypes.md @@ -14,7 +14,7 @@ This page lists all built-in archetypes. ## Image & tensor -* [`DepthImage`](archetypes/depth_image.md): A depth image. +* [`DepthImage`](archetypes/depth_image.md): A depth image, i.e. as captured by a depth camera. * [`Image`](archetypes/image.md): A monochrome or color image. * [`ImageEncoded`](archetypes/image_encoded.md): An image encoded as e.g. a JPEG or PNG. * [`SegmentationImage`](archetypes/segmentation_image.md): An image made up of integer [`components.ClassId`](https://rerun.io/docs/reference/types/components/class_id)s. diff --git a/docs/content/reference/types/archetypes/depth_image.md b/docs/content/reference/types/archetypes/depth_image.md index d64506935860..1ba3674d57d2 100644 --- a/docs/content/reference/types/archetypes/depth_image.md +++ b/docs/content/reference/types/archetypes/depth_image.md @@ -3,10 +3,9 @@ title: "DepthImage" --- -A depth image. +A depth image, i.e. as captured by a depth camera. -The shape of the [`components.TensorData`](https://rerun.io/docs/reference/types/components/tensor_data) must be mappable to an `HxW` tensor. -Each pixel corresponds to a depth value in units specified by `meter`. +Each pixel corresponds to a depth value in units specified by [`components.DepthMeter`](https://rerun.io/docs/reference/types/components/depth_meter). ## Components diff --git a/docs/content/reference/types/archetypes/segmentation_image.md b/docs/content/reference/types/archetypes/segmentation_image.md index 455aa523e846..262365f857be 100644 --- a/docs/content/reference/types/archetypes/segmentation_image.md +++ b/docs/content/reference/types/archetypes/segmentation_image.md @@ -5,20 +5,16 @@ title: "SegmentationImage" An image made up of integer [`components.ClassId`](https://rerun.io/docs/reference/types/components/class_id)s. -The shape of the [`components.TensorData`](https://rerun.io/docs/reference/types/components/tensor_data) must be mappable to an `HxW` tensor. Each pixel corresponds to a [`components.ClassId`](https://rerun.io/docs/reference/types/components/class_id) that will be mapped to a color based on annotation context. In the case of floating point images, the label will be looked up based on rounding to the nearest integer value. -Leading and trailing unit-dimensions are ignored, so that -`1x640x480x1` is treated as a `640x480` image. - See also [`archetypes.AnnotationContext`](https://rerun.io/docs/reference/types/archetypes/annotation_context) to associate each class with a color and a label. ## Components -**Required**: [`TensorData`](../components/tensor_data.md) +**Required**: [`Blob`](../components/blob.md), [`Resolution2D`](../components/resolution2d.md), [`ChannelDataType`](../components/channel_data_type.md) **Optional**: [`Opacity`](../components/opacity.md), [`DrawOrder`](../components/draw_order.md) diff --git a/docs/content/reference/types/components/blob.md b/docs/content/reference/types/components/blob.md index 5b83396920ad..8354c5e3dab9 100644 --- a/docs/content/reference/types/components/blob.md +++ b/docs/content/reference/types/components/blob.md @@ -20,3 +20,4 @@ A binary blob of data. * [`Asset3D`](../archetypes/asset3d.md) * [`DepthImage`](../archetypes/depth_image.md) * [`ImageEncoded`](../archetypes/image_encoded.md?speculative-link) +* [`SegmentationImage`](../archetypes/segmentation_image.md) diff --git a/docs/content/reference/types/components/channel_data_type.md b/docs/content/reference/types/components/channel_data_type.md index 1863e32e1c47..eac44d478385 100644 --- a/docs/content/reference/types/components/channel_data_type.md +++ b/docs/content/reference/types/components/channel_data_type.md @@ -30,3 +30,4 @@ How individual color channel components are encoded. ## Used by * [`DepthImage`](../archetypes/depth_image.md) +* [`SegmentationImage`](../archetypes/segmentation_image.md) diff --git a/docs/content/reference/types/components/color_model.md b/docs/content/reference/types/components/color_model.md index 4c59a50d84d6..d0bd68f809cf 100644 --- a/docs/content/reference/types/components/color_model.md +++ b/docs/content/reference/types/components/color_model.md @@ -14,8 +14,8 @@ This combined with [`components.ChannelDataType`](https://rerun.io/docs/referenc * RGBA ## API reference links - * 🌊 [C++ API docs for `ColorModel`](https://ref.rerun.io/docs/cpp/stable/namespacererun_1_1components.html) - * 🐍 [Python API docs for `ColorModel`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.ColorModel) - * 🦀 [Rust API docs for `ColorModel`](https://docs.rs/rerun/latest/rerun/components/enum.ColorModel.html) + * 🌊 [C++ API docs for `ColorModel`](https://ref.rerun.io/docs/cpp/stable/namespacererun_1_1components.html?speculative-link) + * 🐍 [Python API docs for `ColorModel`](https://ref.rerun.io/docs/python/stable/common/components?speculative-link#rerun.components.ColorModel) + * 🦀 [Rust API docs for `ColorModel`](https://docs.rs/rerun/latest/rerun/components/enum.ColorModel.html?speculative-link) diff --git a/docs/content/reference/types/components/resolution2d.md b/docs/content/reference/types/components/resolution2d.md index c5498547279e..ad56e6a6e2d0 100644 --- a/docs/content/reference/types/components/resolution2d.md +++ b/docs/content/reference/types/components/resolution2d.md @@ -10,11 +10,12 @@ The width and height of a 2D image. * wh: [`UVec2D`](../datatypes/uvec2d.md) ## API reference links - * 🌊 [C++ API docs for `Resolution2D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1Resolution2D.html) - * 🐍 [Python API docs for `Resolution2D`](https://ref.rerun.io/docs/python/stable/common/components#rerun.components.Resolution2D) - * 🦀 [Rust API docs for `Resolution2D`](https://docs.rs/rerun/latest/rerun/components/struct.Resolution2D.html) + * 🌊 [C++ API docs for `Resolution2D`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1components_1_1Resolution2D.html?speculative-link) + * 🐍 [Python API docs for `Resolution2D`](https://ref.rerun.io/docs/python/stable/common/components?speculative-link#rerun.components.Resolution2D) + * 🦀 [Rust API docs for `Resolution2D`](https://docs.rs/rerun/latest/rerun/components/struct.Resolution2D.html?speculative-link) ## Used by * [`DepthImage`](../archetypes/depth_image.md) +* [`SegmentationImage`](../archetypes/segmentation_image.md) diff --git a/docs/content/reference/types/components/tensor_data.md b/docs/content/reference/types/components/tensor_data.md index 5a15d81869e3..1385344a1339 100644 --- a/docs/content/reference/types/components/tensor_data.md +++ b/docs/content/reference/types/components/tensor_data.md @@ -30,5 +30,4 @@ For chroma downsampled formats the shape has to be the shape of the decoded imag * [`BarChart`](../archetypes/bar_chart.md) * [`Image`](../archetypes/image.md) * [`Mesh3D`](../archetypes/mesh3d.md) -* [`SegmentationImage`](../archetypes/segmentation_image.md) * [`Tensor`](../archetypes/tensor.md) diff --git a/docs/content/reference/types/datatypes/uvec2d.md b/docs/content/reference/types/datatypes/uvec2d.md index 15c7bebcac4c..19002d6a6ab1 100644 --- a/docs/content/reference/types/datatypes/uvec2d.md +++ b/docs/content/reference/types/datatypes/uvec2d.md @@ -17,4 +17,4 @@ A uint32 vector in 2D space. ## Used by -* [`Resolution2D`](../components/resolution2d.md) +* [`Resolution2D`](../components/resolution2d.md?speculative-link) diff --git a/docs/snippets/all/archetypes/annotation_context_segmentation.cpp b/docs/snippets/all/archetypes/annotation_context_segmentation.cpp index 7a0b06dff736..acd88ef3288e 100644 --- a/docs/snippets/all/archetypes/annotation_context_segmentation.cpp +++ b/docs/snippets/all/archetypes/annotation_context_segmentation.cpp @@ -29,5 +29,5 @@ int main() { std::fill_n(data.begin() + y * WIDTH + 130, 150, static_cast(2)); } - rec.log("segmentation/image", rerun::SegmentationImage({HEIGHT, WIDTH}, std::move(data))); + rec.log("segmentation/image", rerun::SegmentationImage(std::move(data), {WIDTH, HEIGHT})); } diff --git a/docs/snippets/all/archetypes/segmentation_image_simple.cpp b/docs/snippets/all/archetypes/segmentation_image_simple.cpp index d879d13e7263..3e68a8466527 100644 --- a/docs/snippets/all/archetypes/segmentation_image_simple.cpp +++ b/docs/snippets/all/archetypes/segmentation_image_simple.cpp @@ -29,5 +29,5 @@ int main() { }) ); - rec.log("image", rerun::SegmentationImage({HEIGHT, WIDTH}, data)); + rec.log("image", rerun::SegmentationImage(data, {WIDTH, HEIGHT})); } diff --git a/rerun_cpp/src/rerun/archetypes/annotation_context.hpp b/rerun_cpp/src/rerun/archetypes/annotation_context.hpp index 04cf109b96f4..fa6ac3aa5fa1 100644 --- a/rerun_cpp/src/rerun/archetypes/annotation_context.hpp +++ b/rerun_cpp/src/rerun/archetypes/annotation_context.hpp @@ -59,7 +59,7 @@ namespace rerun::archetypes { /// std::fill_n(data.begin() + y * WIDTH + 130, 150, static_cast(2)); /// } /// - /// rec.log("segmentation/image", rerun::SegmentationImage({HEIGHT, WIDTH}, std::move(data))); + /// rec.log("segmentation/image", rerun::SegmentationImage(std::move(data), {WIDTH, HEIGHT})); /// } /// ``` struct AnnotationContext { diff --git a/rerun_cpp/src/rerun/archetypes/depth_image.hpp b/rerun_cpp/src/rerun/archetypes/depth_image.hpp index 6f9b84cc4ee8..42e0218ad4b3 100644 --- a/rerun_cpp/src/rerun/archetypes/depth_image.hpp +++ b/rerun_cpp/src/rerun/archetypes/depth_image.hpp @@ -23,10 +23,9 @@ #include namespace rerun::archetypes { - /// **Archetype**: A depth image. + /// **Archetype**: A depth image, i.e. as captured by a depth camera. /// - /// The shape of the `components::TensorData` must be mappable to an `HxW` tensor. - /// Each pixel corresponds to a depth value in units specified by `meter`. + /// Each pixel corresponds to a depth value in units specified by `components::DepthMeter`. /// /// Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, /// data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays. @@ -156,8 +155,6 @@ namespace rerun::archetypes { resolution{resolution_}, data_type{data_type_} {} - /// New depth image from an `ChannelDataType` and a pointer. - /// /// The length of the data should be `W * H * data_type.size` DepthImage( Collection data_, components::Resolution2D resolution_, diff --git a/rerun_cpp/src/rerun/archetypes/depth_image_ext.cpp b/rerun_cpp/src/rerun/archetypes/depth_image_ext.cpp index 1e8fd053b86a..cd840d820746 100644 --- a/rerun_cpp/src/rerun/archetypes/depth_image_ext.cpp +++ b/rerun_cpp/src/rerun/archetypes/depth_image_ext.cpp @@ -41,8 +41,6 @@ namespace rerun::archetypes { resolution{resolution_}, data_type{data_type_} {} - /// New depth image from an `ChannelDataType` and a pointer. - /// /// The length of the data should be `W * H * data_type.size` DepthImage( Collection data_, components::Resolution2D resolution_, diff --git a/rerun_cpp/src/rerun/archetypes/segmentation_image.cpp b/rerun_cpp/src/rerun/archetypes/segmentation_image.cpp index 79c856b9a3ef..54fa564560d8 100644 --- a/rerun_cpp/src/rerun/archetypes/segmentation_image.cpp +++ b/rerun_cpp/src/rerun/archetypes/segmentation_image.cpp @@ -14,13 +14,23 @@ namespace rerun { ) { using namespace archetypes; std::vector cells; - cells.reserve(4); + cells.reserve(6); { auto result = DataCell::from_loggable(archetype.data); RR_RETURN_NOT_OK(result.error); cells.push_back(std::move(result.value)); } + { + auto result = DataCell::from_loggable(archetype.resolution); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + { + auto result = DataCell::from_loggable(archetype.data_type); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } if (archetype.opacity.has_value()) { auto result = DataCell::from_loggable(archetype.opacity.value()); RR_RETURN_NOT_OK(result.error); diff --git a/rerun_cpp/src/rerun/archetypes/segmentation_image.hpp b/rerun_cpp/src/rerun/archetypes/segmentation_image.hpp index ab2552b164e6..81f31f05a3c6 100644 --- a/rerun_cpp/src/rerun/archetypes/segmentation_image.hpp +++ b/rerun_cpp/src/rerun/archetypes/segmentation_image.hpp @@ -5,10 +5,13 @@ #include "../collection.hpp" #include "../compiler_utils.hpp" +#include "../components/blob.hpp" +#include "../components/channel_data_type.hpp" #include "../components/draw_order.hpp" #include "../components/opacity.hpp" -#include "../components/tensor_data.hpp" +#include "../components/resolution2d.hpp" #include "../data_cell.hpp" +#include "../image_utils.hpp" #include "../indicator_component.hpp" #include "../result.hpp" @@ -20,15 +23,11 @@ namespace rerun::archetypes { /// **Archetype**: An image made up of integer `components::ClassId`s. /// - /// The shape of the `components::TensorData` must be mappable to an `HxW` tensor. /// Each pixel corresponds to a `components::ClassId` that will be mapped to a color based on annotation context. /// /// In the case of floating point images, the label will be looked up based on rounding to the nearest /// integer value. /// - /// Leading and trailing unit-dimensions are ignored, so that - /// `1x640x480x1` is treated as a `640x480` image. - /// /// See also `archetypes::AnnotationContext` to associate each class with a color and a label. /// /// Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, @@ -70,12 +69,18 @@ namespace rerun::archetypes { /// }) /// ); /// - /// rec.log("image", rerun::SegmentationImage({HEIGHT, WIDTH}, data)); + /// rec.log("image", rerun::SegmentationImage(data, {WIDTH, HEIGHT})); /// } /// ``` struct SegmentationImage { - /// The image data. Should always be a 2-dimensional tensor. - rerun::components::TensorData data; + /// The raw image data. + rerun::components::Blob data; + + /// The size of the image. + rerun::components::Resolution2D resolution; + + /// The data type of the segmentation image data (U16, U32, …). + rerun::components::ChannelDataType data_type; /// Opacity of the image, useful for layering the segmentation image on top of another image. /// @@ -97,40 +102,46 @@ namespace rerun::archetypes { public: // Extensions to generated type defined in 'segmentation_image_ext.cpp' - /// New segmentation image from height/width and tensor buffer. + /// Row-major. Borrows. /// - /// \param shape - /// Shape of the image. Calls `Error::handle()` if the tensor is not 2-dimensional - /// Sets the dimension names to "height" and "width" if they are not specified. - /// \param buffer - /// The tensor buffer containing the segmentation image data. - SegmentationImage( - Collection shape, datatypes::TensorBuffer buffer - ) - : SegmentationImage(datatypes::TensorData(std::move(shape), std::move(buffer))) {} + /// The length of the data should be `W * H`. + template + SegmentationImage(const TElement* pixels, components::Resolution2D resolution_) + : SegmentationImage{ + reinterpret_cast(pixels), resolution_, get_data_type(pixels)} {} - /// New segmentation image from tensor data. + /// Row-major. /// - /// \param data_ - /// The tensor buffer containing the segmentation image data. - /// Sets the dimension names to "height" and "width" if they are not specified. - /// Calls `Error::handle()` if the tensor is not 2-dimensional - explicit SegmentationImage(components::TensorData data_); + /// The length of the data should be `W * H`. + template + SegmentationImage(std::vector pixels, components::Resolution2D resolution_) + : SegmentationImage{ + Collection::take_ownership(std::move(pixels)), resolution_} {} - /// New segmentation image from dimensions and pointer to segmentation image data. + /// Row-major. /// - /// Type must be one of the types supported by `rerun::datatypes::TensorData`. - /// \param shape - /// Shape of the image. Calls `Error::handle()` if the tensor is not 2-dimensional - /// Sets the dimension names to "height", "width" and "channel" if they are not specified. - /// Determines the number of elements expected to be in `data`. - /// \param data_ - /// Target of the pointer must outlive the archetype. + /// The length of the data should be `W * H`. template - explicit SegmentationImage( - Collection shape, const TElement* data_ + SegmentationImage(Collection pixels, components::Resolution2D resolution_) + : SegmentationImage{pixels.to_uint8(), resolution_, get_data_type(pixels.data())} {} + + /// Row-major. Borrows. + /// + /// The length of the data should be `W * H * data_type.size` + SegmentationImage( + const void* data_, components::Resolution2D resolution_, + components::ChannelDataType data_type_ + ) + : data{Collection::borrow(data_, num_bytes(resolution_, data_type_))}, + resolution{resolution_}, + data_type{data_type_} {} + + /// The length of the data should be `W * H * data_type.size` + SegmentationImage( + Collection data_, components::Resolution2D resolution_, + components::ChannelDataType data_type_ ) - : SegmentationImage(datatypes::TensorData(std::move(shape), data_)) {} + : data{data_}, resolution{resolution_}, data_type{data_type_} {} public: SegmentationImage() = default; diff --git a/rerun_cpp/src/rerun/archetypes/segmentation_image_ext.cpp b/rerun_cpp/src/rerun/archetypes/segmentation_image_ext.cpp index eafb39b7f2d6..6136b88a93a2 100644 --- a/rerun_cpp/src/rerun/archetypes/segmentation_image_ext.cpp +++ b/rerun_cpp/src/rerun/archetypes/segmentation_image_ext.cpp @@ -1,75 +1,54 @@ -#include "segmentation_image.hpp" - -#include "../collection_adapter_builtins.hpp" #include "../error.hpp" - -#include +#include "segmentation_image.hpp" namespace rerun::archetypes { -#if 0 +#ifdef EDIT_EXTENSION // - /// New segmentation image from height/width and tensor buffer. +#include "../image_utils.hpp" + + /// Row-major. Borrows. /// - /// \param shape - /// Shape of the image. Calls `Error::handle()` if the tensor is not 2-dimensional - /// Sets the dimension names to "height" and "width" if they are not specified. - /// \param buffer - /// The tensor buffer containing the segmentation image data. - SegmentationImage(Collection shape, datatypes::TensorBuffer buffer) - : SegmentationImage(datatypes::TensorData(std::move(shape), std::move(buffer))) {} + /// The length of the data should be `W * H`. + template + SegmentationImage(const TElement* pixels, components::Resolution2D resolution_) + : SegmentationImage{ + reinterpret_cast(pixels), resolution_, get_data_type(pixels)} {} - /// New segmentation image from tensor data. + /// Row-major. /// - /// \param data_ - /// The tensor buffer containing the segmentation image data. - /// Sets the dimension names to "height" and "width" if they are not specified. - /// Calls `Error::handle()` if the tensor is not 2-dimensional - explicit SegmentationImage(components::TensorData data_); + /// The length of the data should be `W * H`. + template + SegmentationImage(std::vector pixels, components::Resolution2D resolution_) + : SegmentationImage{Collection::take_ownership(std::move(pixels)), resolution_} {} - /// New segmentation image from dimensions and pointer to segmentation image data. + /// Row-major. /// - /// Type must be one of the types supported by `rerun::datatypes::TensorData`. - /// \param shape - /// Shape of the image. Calls `Error::handle()` if the tensor is not 2-dimensional - /// Sets the dimension names to "height", "width" and "channel" if they are not specified. - /// Determines the number of elements expected to be in `data`. - /// \param data_ - /// Target of the pointer must outlive the archetype. + /// The length of the data should be `W * H`. template - explicit SegmentationImage(Collection shape, const TElement* data_) - : SegmentationImage(datatypes::TensorData(std::move(shape), data_)) {} + SegmentationImage(Collection pixels, components::Resolution2D resolution_) + : SegmentationImage{pixels.to_uint8(), resolution_, get_data_type(pixels.data())} {} + + /// Row-major. Borrows. + /// + /// The length of the data should be `W * H * data_type.size` + SegmentationImage( + const void* data_, components::Resolution2D resolution_, + components::ChannelDataType data_type_ + ) + : data{Collection::borrow(data_, num_bytes(resolution_, data_type_))}, + resolution{resolution_}, + data_type{data_type_} {} + + /// The length of the data should be `W * H * data_type.size` + SegmentationImage( + Collection data_, components::Resolution2D resolution_, + components::ChannelDataType data_type_ + ) + : data{data_}, resolution{resolution_}, data_type{data_type_} {} // #endif - SegmentationImage::SegmentationImage(components::TensorData data_) : data(std::move(data_)) { - auto& shape = data.data.shape; - if (shape.size() != 2) { - std::stringstream ss; - ss << "Expected 2-dimensional tensor, got " << shape.size() << " dimensions."; - Error(ErrorCode::InvalidTensorDimension, ss.str()).handle(); - return; - } - - // We want to change the dimension names if they are not specified. - // But rerun collections are strictly immutable, so create a new one if necessary. - bool overwrite_height = !shape[0].name.has_value(); - bool overwrite_width = !shape[1].name.has_value(); - - if (overwrite_height || overwrite_width) { - auto new_shape = shape.to_vector(); - - if (overwrite_height) { - new_shape[0].name = "height"; - } - if (overwrite_width) { - new_shape[1].name = "width"; - } - - shape = std::move(new_shape); - } - } - } // namespace rerun::archetypes diff --git a/rerun_cpp/tests/archetypes/segmentation_and_depth_image.cpp b/rerun_cpp/tests/archetypes/segmentation_and_depth_image.cpp index 0cbb5e58c5e9..c1245b3d8acf 100644 --- a/rerun_cpp/tests/archetypes/segmentation_and_depth_image.cpp +++ b/rerun_cpp/tests/archetypes/segmentation_and_depth_image.cpp @@ -25,115 +25,10 @@ void run_image_tests() { } } -template -void run_tensor_image_tests() { - GIVEN("tensor data with correct shape") { - rerun::datatypes::TensorData data({3, 7}, std::vector(3 * 7, 0)); - THEN("no error occurs on image construction") { - auto image = check_logged_error([&] { return ImageType(std::move(data)); }); - - AND_THEN("width and height got set") { - CHECK(image.data.data.shape[0].name == "height"); - CHECK(image.data.data.shape[1].name == "width"); - } - - AND_THEN("serialization succeeds") { - CHECK(rerun::AsComponents().serialize(image).is_ok()); - } - } - } - - GIVEN("tensor data with correct shape and named dimensions") { - rerun::datatypes::TensorData data( - {rerun::datatypes::TensorDimension(3, "rick"), - rerun::datatypes::TensorDimension(7, "morty")}, - std::vector(3 * 7, 0) - ); - THEN("no error occurs on image construction") { - auto image = check_logged_error([&] { return ImageType(std::move(data)); }); - - AND_THEN("tensor dimensions are unchanged") { - CHECK(image.data.data.shape[0].name == "rick"); - CHECK(image.data.data.shape[1].name == "morty"); - } - - AND_THEN("serialization succeeds") { - CHECK(rerun::AsComponents().serialize(image).is_ok()); - } - } - } - - GIVEN("tensor data with too high dimensionality") { - rerun::datatypes::TensorData data( - { - { - rerun::datatypes::TensorDimension(1, "tick"), - rerun::datatypes::TensorDimension(2, "trick"), - rerun::datatypes::TensorDimension(3, "track"), - }, - }, - std::vector(1 * 2 * 3, 0) - ); - THEN("a warning occurs on image construction") { - auto image = check_logged_error( - [&] { return ImageType(std::move(data)); }, - rerun::ErrorCode::InvalidTensorDimension - ); - - AND_THEN("tensor dimension names are unchanged") { - CHECK(image.data.data.shape[0].name == "tick"); - CHECK(image.data.data.shape[1].name == "trick"); - CHECK(image.data.data.shape[2].name == "track"); - } - - AND_THEN("serialization succeeds") { - CHECK(rerun::AsComponents().serialize(image).is_ok()); - } - } - } - - GIVEN("tensor data with too low dimensionality") { - rerun::datatypes::TensorData data( - { - rerun::datatypes::TensorDimension(1, "dr robotnik"), - }, - std::vector(1, 0) - ); - THEN("a warning occurs on image construction") { - auto image = check_logged_error( - [&] { return ImageType(std::move(data)); }, - rerun::ErrorCode::InvalidTensorDimension - ); - - AND_THEN("tensor dimension names are unchanged") { - CHECK(image.data.data.shape[0].name == "dr robotnik"); - } - - AND_THEN("serialization succeeds") { - CHECK(rerun::AsComponents().serialize(image).is_ok()); - } - } - } - - GIVEN("a vector of data") { - std::vector data(10 * 10, 0); - THEN("no error occurs on image construction with either the vector or a data pointer") { - auto image_from_vector = check_logged_error([&] { return ImageType({10, 10}, data); }); - auto image_from_ptr = check_logged_error([&] { - return ImageType({10, 10}, data.data()); - }); - - AND_THEN("serialization succeeds") { - test_compare_archetype_serialization(image_from_ptr, image_from_vector); - } - } - } -} - SCENARIO("Depth archetype image can be created." TEST_TAG) { run_image_tests(); } SCENARIO("Segmentation archetype image can be created from tensor data." TEST_TAG) { - run_tensor_image_tests(); + run_image_tests(); } diff --git a/rerun_py/rerun_sdk/rerun/archetypes/depth_image.py b/rerun_py/rerun_sdk/rerun/archetypes/depth_image.py index b4848ba650e5..4a13be97afe5 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/depth_image.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/depth_image.py @@ -19,10 +19,9 @@ @define(str=False, repr=False, init=False) class DepthImage(DepthImageExt, Archetype): """ - **Archetype**: A depth image. + **Archetype**: A depth image, i.e. as captured by a depth camera. - The shape of the [`components.TensorData`][rerun.components.TensorData] must be mappable to an `HxW` tensor. - Each pixel corresponds to a depth value in units specified by `meter`. + Each pixel corresponds to a depth value in units specified by [`components.DepthMeter`][rerun.components.DepthMeter]. Example ------- diff --git a/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image.py b/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image.py index 84af1851dc0c..51d0595756e6 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image.py @@ -5,15 +5,12 @@ from __future__ import annotations -from typing import Any - from attrs import define, field -from .. import components, datatypes +from .. import components from .._baseclasses import ( Archetype, ) -from ..error_utils import catch_and_log_exceptions from .segmentation_image_ext import SegmentationImageExt __all__ = ["SegmentationImage"] @@ -24,15 +21,11 @@ class SegmentationImage(SegmentationImageExt, Archetype): """ **Archetype**: An image made up of integer [`components.ClassId`][rerun.components.ClassId]s. - The shape of the [`components.TensorData`][rerun.components.TensorData] must be mappable to an `HxW` tensor. Each pixel corresponds to a [`components.ClassId`][rerun.components.ClassId] that will be mapped to a color based on annotation context. In the case of floating point images, the label will be looked up based on rounding to the nearest integer value. - Leading and trailing unit-dimensions are ignored, so that - `1x640x480x1` is treated as a `640x480` image. - See also [`archetypes.AnnotationContext`][rerun.archetypes.AnnotationContext] to associate each class with a color and a label. Example @@ -66,41 +59,14 @@ class SegmentationImage(SegmentationImageExt, Archetype): """ - def __init__( - self: Any, - data: datatypes.TensorDataLike, - *, - opacity: datatypes.Float32Like | None = None, - draw_order: datatypes.Float32Like | None = None, - ): - """ - Create a new instance of the SegmentationImage archetype. - - Parameters - ---------- - data: - The image data. Should always be a 2-dimensional tensor. - opacity: - Opacity of the image, useful for layering the segmentation image on top of another image. - - Defaults to 0.5 if there's any other images in the scene, otherwise 1.0. - draw_order: - An optional floating point value that specifies the 2D drawing order. - - Objects with higher values are drawn on top of those with lower values. - - """ - - # You can define your own __init__ function as a member of SegmentationImageExt in segmentation_image_ext.py - with catch_and_log_exceptions(context=self.__class__.__name__): - self.__attrs_init__(data=data, opacity=opacity, draw_order=draw_order) - return - self.__attrs_clear__() + # __init__ can be found in segmentation_image_ext.py def __attrs_clear__(self) -> None: """Convenience method for calling `__attrs_init__` with all `None`s.""" self.__attrs_init__( data=None, # type: ignore[arg-type] + resolution=None, # type: ignore[arg-type] + data_type=None, # type: ignore[arg-type] opacity=None, # type: ignore[arg-type] draw_order=None, # type: ignore[arg-type] ) @@ -112,11 +78,27 @@ def _clear(cls) -> SegmentationImage: inst.__attrs_clear__() return inst - data: components.TensorDataBatch = field( + data: components.BlobBatch = field( + metadata={"component": "required"}, + converter=components.BlobBatch._required, # type: ignore[misc] + ) + # The raw image data. + # + # (Docstring intentionally commented out to hide this field from the docs) + + resolution: components.Resolution2DBatch = field( + metadata={"component": "required"}, + converter=components.Resolution2DBatch._required, # type: ignore[misc] + ) + # The size of the image. + # + # (Docstring intentionally commented out to hide this field from the docs) + + data_type: components.ChannelDataTypeBatch = field( metadata={"component": "required"}, - converter=SegmentationImageExt.data__field_converter_override, # type: ignore[misc] + converter=components.ChannelDataTypeBatch._required, # type: ignore[misc] ) - # The image data. Should always be a 2-dimensional tensor. + # The data type of the segmentation image data (U16, U32, …). # # (Docstring intentionally commented out to hide this field from the docs) diff --git a/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image_ext.py index c69494f44ea4..66b30225aef6 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image_ext.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/segmentation_image_ext.py @@ -1,84 +1,86 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Union import numpy as np -import pyarrow as pa +import numpy.typing as npt -from .._validators import find_non_empty_dim_indices -from ..error_utils import _send_warning_or_raise, catch_and_log_exceptions +from ..components import ChannelDataType, Resolution2D +from ..datatypes import Float32Like if TYPE_CHECKING: - from ..components import TensorDataBatch - from ..datatypes import TensorDataArrayLike + ImageLike = Union[ + npt.NDArray[np.float16], + npt.NDArray[np.float32], + npt.NDArray[np.float64], + npt.NDArray[np.int16], + npt.NDArray[np.int32], + npt.NDArray[np.int64], + npt.NDArray[np.int8], + npt.NDArray[np.uint16], + npt.NDArray[np.uint32], + npt.NDArray[np.uint64], + npt.NDArray[np.uint8], + ] + + +def _to_numpy(tensor: ImageLike) -> npt.NDArray[Any]: + # isinstance is 4x faster than catching AttributeError + if isinstance(tensor, np.ndarray): + return tensor + + try: + # Make available to the cpu + return tensor.numpy(force=True) # type: ignore[union-attr] + except AttributeError: + return np.array(tensor, copy=False) class SegmentationImageExt: """Extension for [SegmentationImage][rerun.archetypes.SegmentationImage].""" - @staticmethod - @catch_and_log_exceptions("SegmentationImage converter") - def data__field_converter_override(data: TensorDataArrayLike) -> TensorDataBatch: - from ..components import TensorDataBatch - from ..datatypes import TensorDataType, TensorDimensionType - - tensor_data = TensorDataBatch(data) - tensor_data_arrow = tensor_data.as_arrow_array() - - # TODO(jleibs): Doing this on raw arrow data is not great. Clean this up - # once we coerce to a canonical non-arrow type. - shape = tensor_data_arrow.storage.field(0) - - shape_dims = shape[0].values.field(0).to_numpy() - shape_names = shape[0].values.field(1).to_numpy(zero_copy_only=False) - - non_empty_dims = find_non_empty_dim_indices(shape_dims) - - num_non_empty_dims = len(non_empty_dims) - - # TODO(#3239): What `recording` should we be passing here? How should we be getting it? - if num_non_empty_dims != 2: - _send_warning_or_raise(f"Expected segmentation image, got array of shape {shape_dims}", 1, recording=None) - - tensor_data_type = TensorDataType().storage_type - shape_data_type = TensorDimensionType().storage_type - - # IF no labels are set, add them - # TODO(jleibs): Again, needing to do this at the arrow level is awful - if all(label is None for label in shape_names): - for ind, label in zip(non_empty_dims, ["height", "width"]): - shape_names[ind] = label - - shape_names = pa.array( - shape_names, mask=np.array([n is None for n in shape_names]), type=shape_data_type.field("name").type - ) - - shape = pa.ListArray.from_arrays( - offsets=[0, len(shape_dims)], - values=pa.StructArray.from_arrays( - [ - tensor_data_arrow[0].value["shape"].values.field(0), - shape_names, - ], - fields=[shape_data_type.field("size"), shape_data_type.field("name")], - ), - ).cast(tensor_data_type.field("shape").type) - - buffer = tensor_data_arrow.storage.field(1) - - return TensorDataBatch( - pa.StructArray.from_arrays( - [ - shape, - buffer, - ], - fields=[ - tensor_data_type.field("shape"), - tensor_data_type.field("buffer"), - ], - ).cast(tensor_data_arrow.storage.type) + def __init__( + self: Any, + data: ImageLike, + *, + opacity: Float32Like | None = None, + ): + channel_dtype_from_np_dtype = { + np.uint8: ChannelDataType.U8, + np.uint16: ChannelDataType.U16, + np.uint32: ChannelDataType.U32, + np.uint64: ChannelDataType.U64, + np.int8: ChannelDataType.I8, + np.int16: ChannelDataType.I16, + np.int32: ChannelDataType.I32, + np.int64: ChannelDataType.I64, + np.float16: ChannelDataType.F16, + np.float32: ChannelDataType.F32, + np.float64: ChannelDataType.F64, + } + + data = _to_numpy(data) + + shape = data.shape + + # Ignore leading and trailing dimensions of size 1: + while 2 < len(shape) and shape[0] == 1: + shape = shape[1:] + while 2 < len(shape) and shape[-1] == 1: + shape = shape[:-1] + + if len(shape) != 2: + raise ValueError(f"SegmentationImage must be 2D, got shape {data.shape}") + height, width = shape + + try: + data_type = channel_dtype_from_np_dtype[data.dtype.type] + except KeyError: + raise ValueError(f"Unsupported dtype {data.dtype} for SegmentationImage") + + self.__attrs_init__( + data=data.tobytes(), + resolution=Resolution2D(width=width, height=height), + data_type=data_type, + opacity=opacity, ) - - # TODO(jleibs): Should we enforce specific names on images? Specifically, what if the existing names are wrong. - - return tensor_data diff --git a/rerun_py/tests/unit/test_segmentation_image.py b/rerun_py/tests/unit/test_segmentation_image.py index 6f732e02b69b..801f62ff4ba0 100644 --- a/rerun_py/tests/unit/test_segmentation_image.py +++ b/rerun_py/tests/unit/test_segmentation_image.py @@ -6,40 +6,31 @@ import pytest import rerun as rr import torch -from rerun.datatypes import TensorBuffer, TensorData, TensorDataLike, TensorDimension rng = np.random.default_rng(12345) RANDOM_IMAGE_SOURCE = rng.integers(0, 255, size=(10, 20)) -IMAGE_INPUTS: list[TensorDataLike] = [ - # Full explicit construction - TensorData( - shape=[ - TensorDimension(10, "height"), - TensorDimension(20, "width"), - ], - buffer=TensorBuffer(RANDOM_IMAGE_SOURCE), - ), - # Implicit construction from ndarray +IMAGE_INPUTS: list[Any] = [ + RANDOM_IMAGE_SOURCE, RANDOM_IMAGE_SOURCE, ] def segmentation_image_image_expected() -> Any: - return rr.SegmentationImage(data=RANDOM_IMAGE_SOURCE) + return rr.SegmentationImage(RANDOM_IMAGE_SOURCE) def test_image() -> None: expected = segmentation_image_image_expected() for img in IMAGE_INPUTS: - arch = rr.SegmentationImage(data=img) + arch = rr.SegmentationImage(img) assert arch == expected -GOOD_IMAGE_INPUTS: list[TensorDataLike] = [ +GOOD_IMAGE_INPUTS: list[Any] = [ # Mono rng.integers(0, 255, (10, 20)), # Assorted Extra Dimensions @@ -49,7 +40,7 @@ def test_image() -> None: torch.randint(0, 255, (10, 20)), ] -BAD_IMAGE_INPUTS: list[TensorDataLike] = [ +BAD_IMAGE_INPUTS: list[Any] = [ rng.integers(0, 255, (10, 20, 3)), rng.integers(0, 255, (10, 20, 4)), rng.integers(0, 255, (10,)), diff --git a/tests/cpp/roundtrips/segmentation_image/main.cpp b/tests/cpp/roundtrips/segmentation_image/main.cpp index 2871e3ae0231..8df09aad8d76 100644 --- a/tests/cpp/roundtrips/segmentation_image/main.cpp +++ b/tests/cpp/roundtrips/segmentation_image/main.cpp @@ -8,7 +8,8 @@ int main(int, char** argv) { rec.save(argv[1]).exit_on_failure(); // 3x2 image. Each pixel is incremented down each row - auto img = rerun::datatypes::TensorData({2, 3}, std::vector{0, 1, 2, 3, 4, 5}); - - rec.log("segmentation_image", rerun::archetypes::SegmentationImage(img)); + rec.log( + "segmentation_image", + rerun::archetypes::SegmentationImage(std::vector{0, 1, 2, 3, 4, 5}, {3, 2}) + ); } diff --git a/tests/python/roundtrips/segmentation_image/main.py b/tests/python/roundtrips/segmentation_image/main.py index 20a2f2815aa5..287129188e41 100755 --- a/tests/python/roundtrips/segmentation_image/main.py +++ b/tests/python/roundtrips/segmentation_image/main.py @@ -22,7 +22,7 @@ def main() -> None: image[0, :] = [0, 1, 2] image[1, :] = [3, 4, 5] - rr.log("segmentation_image", rr.SegmentationImage(data=image)) + rr.log("segmentation_image", rr.SegmentationImage(image)) rr.script_teardown(args)