diff --git a/Cargo.lock b/Cargo.lock index a009d59d43870..4b75ff5d177be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5297,6 +5297,7 @@ dependencies = [ "egui_tiles", "glam", "half 2.3.1", + "image", "indexmap 2.1.0", "itertools 0.13.0", "linked-hash-map", diff --git a/crates/store/re_types/definitions/rerun/archetypes.fbs b/crates/store/re_types/definitions/rerun/archetypes.fbs index 4ddccbed02f3e..708b6549131ed 100644 --- a/crates/store/re_types/definitions/rerun/archetypes.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes.fbs @@ -8,6 +8,7 @@ include "./archetypes/boxes3d.fbs"; include "./archetypes/clear.fbs"; include "./archetypes/depth_image.fbs"; include "./archetypes/disconnected_space.fbs"; +include "./archetypes/image_encoded.fbs"; include "./archetypes/image.fbs"; include "./archetypes/line_strips2d.fbs"; include "./archetypes/line_strips3d.fbs"; diff --git a/crates/store/re_types/definitions/rerun/archetypes/image.fbs b/crates/store/re_types/definitions/rerun/archetypes/image.fbs index 060affc454445..af50c2863de64 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/image.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/image.fbs @@ -20,12 +20,12 @@ namespace rerun.archetypes; /// Leading and trailing unit-dimensions are ignored, so that /// `1x480x640x3x1` is treated as a `480x640x3` RGB image. /// -/// Rerun also supports compressed image encoded as JPEG, N12, and YUY2. -/// Using these formats can save a lot of bandwidth and memory. -/// \py To compress an image, use [`rerun.Image.compress`][]. -/// \py To pass in an already encoded image, use [`rerun.ImageEncoded`][]. -/// \rs See [`crate::components::TensorData`] for more. -/// \cpp See [`rerun::datatypes::TensorBuffer`] for more. +/// Rerun also supports compressed images (JPEG, PNG, …), using [archetypes.ImageEncoded]. +/// Compressing images can save a lot of bandwidth and memory. +/// +/// \py You can compress an image using [`rerun.Image.compress`][]. +/// +/// See also [components.TensorData] and [datatypes.TensorBuffer]. /// /// \cpp Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, /// \cpp data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays. diff --git a/crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs b/crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs new file mode 100644 index 0000000000000..7e0b8311daf3c --- /dev/null +++ b/crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs @@ -0,0 +1,51 @@ +include "fbs/attributes.fbs"; + +include "rerun/datatypes.fbs"; +include "rerun/components.fbs"; + +namespace rerun.archetypes; + + +/// An image encoded as e.g. a JPEG or PNG. +/// +/// Rerun also supports uncompressed images with the [archetypes.Image]. +/// +/// \py To compress an image, use [`rerun.Image.compress`][]. +/// +/// \example archetypes/image_encoded +table ImageEncoded ( + "attr.cpp.no_field_ctors", + "attr.docs.category": "Image & tensor", + "attr.docs.unreleased", + "attr.docs.view_types": "Spatial2DView, Spatial3DView: if logged under a projection", + "attr.rust.derive": "PartialEq" +) { + // --- Required --- + + /// The encoded content of some image file, e.g. a PNG or JPEG. + blob: rerun.components.Blob ("attr.rerun.component_required", order: 1000); + + // --- Recommended --- + + /// The Media Type of the asset. + /// + /// Supported values: + /// * `image/jpeg` + /// * `image/png` + /// + /// If omitted, the viewer will try to guess from the data blob. + /// If it cannot guess, it won't be able to render the asset. + media_type: rerun.components.MediaType ("attr.rerun.component_recommended", nullable, order: 2000); + + // --- Optional --- + + /// Opacity of the image, useful for layering several images. + /// + /// Defaults to 1.0 (fully opaque). + opacity: rerun.components.Opacity ("attr.rerun.component_optional", nullable, order: 3000); + + /// An optional floating point value that specifies the 2D drawing order. + /// + /// Objects with higher values are drawn on top of those with lower values. + draw_order: rerun.components.DrawOrder ("attr.rerun.component_optional", nullable, order: 3100); +} diff --git a/crates/store/re_types/src/archetypes/.gitattributes b/crates/store/re_types/src/archetypes/.gitattributes index 56544ae96577d..4a71b07037b70 100644 --- a/crates/store/re_types/src/archetypes/.gitattributes +++ b/crates/store/re_types/src/archetypes/.gitattributes @@ -11,6 +11,7 @@ boxes3d.rs linguist-generated=true depth_image.rs linguist-generated=true disconnected_space.rs linguist-generated=true image.rs linguist-generated=true +image_encoded.rs linguist-generated=true line_strips2d.rs linguist-generated=true line_strips3d.rs linguist-generated=true mesh3d.rs linguist-generated=true diff --git a/crates/store/re_types/src/archetypes/image.rs b/crates/store/re_types/src/archetypes/image.rs index a245be1ddf05a..b35e1279f4af2 100644 --- a/crates/store/re_types/src/archetypes/image.rs +++ b/crates/store/re_types/src/archetypes/image.rs @@ -32,9 +32,10 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// Leading and trailing unit-dimensions are ignored, so that /// `1x480x640x3x1` is treated as a `480x640x3` RGB image. /// -/// Rerun also supports compressed image encoded as JPEG, N12, and YUY2. -/// Using these formats can save a lot of bandwidth and memory. -/// See [`crate::components::TensorData`] for more. +/// Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes::ImageEncoded`][crate::archetypes::ImageEncoded]. +/// Compressing images can save a lot of bandwidth and memory. +/// +/// See also [`components::TensorData`][crate::components::TensorData] and [`datatypes::TensorBuffer`][crate::datatypes::TensorBuffer]. /// /// ## Example /// diff --git a/crates/store/re_types/src/archetypes/image_encoded.rs b/crates/store/re_types/src/archetypes/image_encoded.rs new file mode 100644 index 0000000000000..425a49ae58eab --- /dev/null +++ b/crates/store/re_types/src/archetypes/image_encoded.rs @@ -0,0 +1,285 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/rust/api.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs". + +#![allow(unused_imports)] +#![allow(unused_parens)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::cloned_instead_of_copied)] +#![allow(clippy::map_flatten)] +#![allow(clippy::needless_question_mark)] +#![allow(clippy::new_without_default)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::too_many_lines)] + +use ::re_types_core::external::arrow2; +use ::re_types_core::ComponentName; +use ::re_types_core::SerializationResult; +use ::re_types_core::{ComponentBatch, MaybeOwnedComponentBatch}; +use ::re_types_core::{DeserializationError, DeserializationResult}; + +/// **Archetype**: An image encoded as e.g. a JPEG or PNG. +/// +/// Rerun also supports uncompressed images with the [`archetypes::Image`][crate::archetypes::Image]. +/// +/// ## Example +/// +/// ### `image_encoded`: +/// ```ignore +/// fn main() -> Result<(), Box> { +/// let rec = rerun::RecordingStreamBuilder::new("rerun_example_image_encoded").spawn()?; +/// +/// let image = include_bytes!("../../../../crates/viewer/re_ui/data/logo_dark_mode.png"); +/// +/// rec.log("image", &rerun::ImageEncoded::from_bytes(image.to_vec()))?; +/// +/// Ok(()) +/// } +/// ``` +#[derive(Clone, Debug, PartialEq)] +pub struct ImageEncoded { + /// The encoded content of some image file, e.g. a PNG or JPEG. + pub blob: crate::components::Blob, + + /// The Media Type of the asset. + /// + /// Supported values: + /// * `image/jpeg` + /// * `image/png` + /// + /// If omitted, the viewer will try to guess from the data blob. + /// If it cannot guess, it won't be able to render the asset. + pub media_type: Option, + + /// Opacity of the image, useful for layering several images. + /// + /// Defaults to 1.0 (fully opaque). + pub opacity: Option, + + /// An optional floating point value that specifies the 2D drawing order. + /// + /// Objects with higher values are drawn on top of those with lower values. + pub draw_order: Option, +} + +impl ::re_types_core::SizeBytes for ImageEncoded { + #[inline] + fn heap_size_bytes(&self) -> u64 { + self.blob.heap_size_bytes() + + self.media_type.heap_size_bytes() + + self.opacity.heap_size_bytes() + + self.draw_order.heap_size_bytes() + } + + #[inline] + fn is_pod() -> bool { + ::is_pod() + && >::is_pod() + && >::is_pod() + && >::is_pod() + } +} + +static REQUIRED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 1usize]> = + once_cell::sync::Lazy::new(|| ["rerun.components.Blob".into()]); + +static RECOMMENDED_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 2usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.MediaType".into(), + "rerun.components.ImageEncodedIndicator".into(), + ] + }); + +static OPTIONAL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 2usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.Opacity".into(), + "rerun.components.DrawOrder".into(), + ] + }); + +static ALL_COMPONENTS: once_cell::sync::Lazy<[ComponentName; 5usize]> = + once_cell::sync::Lazy::new(|| { + [ + "rerun.components.Blob".into(), + "rerun.components.MediaType".into(), + "rerun.components.ImageEncodedIndicator".into(), + "rerun.components.Opacity".into(), + "rerun.components.DrawOrder".into(), + ] + }); + +impl ImageEncoded { + /// The total number of components in the archetype: 1 required, 2 recommended, 2 optional + pub const NUM_COMPONENTS: usize = 5usize; +} + +/// Indicator component for the [`ImageEncoded`] [`::re_types_core::Archetype`] +pub type ImageEncodedIndicator = ::re_types_core::GenericIndicatorComponent; + +impl ::re_types_core::Archetype for ImageEncoded { + type Indicator = ImageEncodedIndicator; + + #[inline] + fn name() -> ::re_types_core::ArchetypeName { + "rerun.archetypes.ImageEncoded".into() + } + + #[inline] + fn display_name() -> &'static str { + "Image encoded" + } + + #[inline] + fn indicator() -> MaybeOwnedComponentBatch<'static> { + static INDICATOR: ImageEncodedIndicator = ImageEncodedIndicator::DEFAULT; + MaybeOwnedComponentBatch::Ref(&INDICATOR) + } + + #[inline] + fn required_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + REQUIRED_COMPONENTS.as_slice().into() + } + + #[inline] + fn recommended_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + RECOMMENDED_COMPONENTS.as_slice().into() + } + + #[inline] + fn optional_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + OPTIONAL_COMPONENTS.as_slice().into() + } + + #[inline] + fn all_components() -> ::std::borrow::Cow<'static, [ComponentName]> { + ALL_COMPONENTS.as_slice().into() + } + + #[inline] + fn from_arrow_components( + arrow_data: impl IntoIterator)>, + ) -> DeserializationResult { + re_tracing::profile_function!(); + use ::re_types_core::{Loggable as _, ResultExt as _}; + let arrays_by_name: ::std::collections::HashMap<_, _> = arrow_data + .into_iter() + .map(|(name, array)| (name.full_name(), array)) + .collect(); + let blob = { + let array = arrays_by_name + .get("rerun.components.Blob") + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.ImageEncoded#blob")?; + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.ImageEncoded#blob")? + .into_iter() + .next() + .flatten() + .ok_or_else(DeserializationError::missing_data) + .with_context("rerun.archetypes.ImageEncoded#blob")? + }; + let media_type = if let Some(array) = arrays_by_name.get("rerun.components.MediaType") { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.ImageEncoded#media_type")? + .into_iter() + .next() + .flatten() + } else { + None + }; + let opacity = if let Some(array) = arrays_by_name.get("rerun.components.Opacity") { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.ImageEncoded#opacity")? + .into_iter() + .next() + .flatten() + } else { + None + }; + let draw_order = if let Some(array) = arrays_by_name.get("rerun.components.DrawOrder") { + ::from_arrow_opt(&**array) + .with_context("rerun.archetypes.ImageEncoded#draw_order")? + .into_iter() + .next() + .flatten() + } else { + None + }; + Ok(Self { + blob, + media_type, + opacity, + draw_order, + }) + } +} + +impl ::re_types_core::AsComponents for ImageEncoded { + fn as_component_batches(&self) -> Vec> { + re_tracing::profile_function!(); + use ::re_types_core::Archetype as _; + [ + Some(Self::indicator()), + Some((&self.blob as &dyn ComponentBatch).into()), + self.media_type + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), + self.opacity + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), + self.draw_order + .as_ref() + .map(|comp| (comp as &dyn ComponentBatch).into()), + ] + .into_iter() + .flatten() + .collect() + } +} + +impl ImageEncoded { + /// Create a new `ImageEncoded`. + #[inline] + pub fn new(blob: impl Into) -> Self { + Self { + blob: blob.into(), + media_type: None, + opacity: None, + draw_order: None, + } + } + + /// The Media Type of the asset. + /// + /// Supported values: + /// * `image/jpeg` + /// * `image/png` + /// + /// If omitted, the viewer will try to guess from the data blob. + /// If it cannot guess, it won't be able to render the asset. + #[inline] + pub fn with_media_type(mut self, media_type: impl Into) -> Self { + self.media_type = Some(media_type.into()); + self + } + + /// Opacity of the image, useful for layering several images. + /// + /// Defaults to 1.0 (fully opaque). + #[inline] + pub fn with_opacity(mut self, opacity: impl Into) -> Self { + self.opacity = Some(opacity.into()); + self + } + + /// An optional floating point value that specifies the 2D drawing order. + /// + /// Objects with higher values are drawn on top of those with lower values. + #[inline] + pub fn with_draw_order(mut self, draw_order: impl Into) -> Self { + self.draw_order = Some(draw_order.into()); + self + } +} diff --git a/crates/store/re_types/src/archetypes/image_encoded_ext.rs b/crates/store/re_types/src/archetypes/image_encoded_ext.rs new file mode 100644 index 0000000000000..beb178b32cdaa --- /dev/null +++ b/crates/store/re_types/src/archetypes/image_encoded_ext.rs @@ -0,0 +1,19 @@ +use crate::components::Blob; + +use super::ImageEncoded; + +impl ImageEncoded { + /// Construct an image given the encoded content of some image file, e.g. a PNG or JPEG. + /// + /// [`Self::media_type`] will be guessed from the bytes. + pub fn from_bytes(bytes: Vec) -> Self { + Self { + #[cfg(feature = "image")] + media_type: image::guess_format(&bytes) + .ok() + .map(|format| crate::components::MediaType::from(format.to_mime_type())), + + ..Self::new(Blob::from(bytes)) + } + } +} diff --git a/crates/store/re_types/src/archetypes/mod.rs b/crates/store/re_types/src/archetypes/mod.rs index a29b3b0e4e246..d8e511bc75b3c 100644 --- a/crates/store/re_types/src/archetypes/mod.rs +++ b/crates/store/re_types/src/archetypes/mod.rs @@ -16,6 +16,8 @@ mod depth_image; mod depth_image_ext; mod disconnected_space; mod image; +mod image_encoded; +mod image_encoded_ext; mod image_ext; mod line_strips2d; mod line_strips3d; @@ -51,6 +53,7 @@ pub use self::boxes3d::Boxes3D; pub use self::depth_image::DepthImage; pub use self::disconnected_space::DisconnectedSpace; pub use self::image::Image; +pub use self::image_encoded::ImageEncoded; pub use self::line_strips2d::LineStrips2D; pub use self::line_strips3d::LineStrips3D; pub use self::mesh3d::Mesh3D; diff --git a/crates/store/re_types/src/components/media_type_ext.rs b/crates/store/re_types/src/components/media_type_ext.rs index f2850f6a0fe97..6c710d068dd7c 100644 --- a/crates/store/re_types/src/components/media_type_ext.rs +++ b/crates/store/re_types/src/components/media_type_ext.rs @@ -11,6 +11,20 @@ impl MediaType { /// pub const MARKDOWN: &'static str = "text/markdown"; + // ------------------------------------------------------- + // Images: + + /// [JPEG image](https://en.wikipedia.org/wiki/JPEG): `image/jpeg`. + pub const JPEG: &'static str = "image/jpeg"; + + /// [PNG image](https://en.wikipedia.org/wiki/PNG): `image/png`. + /// + /// + pub const PNG: &'static str = "image/png"; + + // ------------------------------------------------------- + // Meshes: + /// [`glTF`](https://en.wikipedia.org/wiki/GlTF). /// /// @@ -46,6 +60,24 @@ impl MediaType { Self(Self::MARKDOWN.into()) } + // ------------------------------------------------------- + // Images: + + /// `image/jpeg` + #[inline] + pub fn jpeg() -> Self { + Self(Self::JPEG.into()) + } + + /// `image/png` + #[inline] + pub fn png() -> Self { + Self(Self::PNG.into()) + } + + // ------------------------------------------------------- + // Meshes: + /// `model/gltf+json` #[inline] pub fn gltf() -> Self { diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/image_encoded.rs b/crates/viewer/re_space_view_spatial/src/visualizers/image_encoded.rs new file mode 100644 index 0000000000000..be6cc98a8e2d8 --- /dev/null +++ b/crates/viewer/re_space_view_spatial/src/visualizers/image_encoded.rs @@ -0,0 +1,227 @@ +use itertools::Itertools as _; + +use re_query::range_zip_1x2; +use re_space_view::HybridResults; +use re_types::{ + archetypes::ImageEncoded, + components::{Blob, DrawOrder, MediaType, Opacity}, + tensor_data::TensorDataMeaning, +}; +use re_viewer_context::{ + ApplicableEntities, IdentifiedViewSystem, ImageDecodeCache, QueryContext, SpaceViewClass, + SpaceViewSystemExecutionError, TypedComponentFallbackProvider, ViewContext, + ViewContextCollection, ViewQuery, VisualizableEntities, VisualizableFilterContext, + VisualizerQueryInfo, VisualizerSystem, +}; + +use crate::{ + contexts::SpatialSceneEntityContext, view_kind::SpatialSpaceViewKind, + visualizers::filter_visualizable_2d_entities, PickableImageRect, SpatialSpaceView2D, +}; + +use super::{ + bounding_box_for_textured_rect, entity_iterator::process_archetype, tensor_to_textured_rect, + SpatialViewVisualizerData, +}; + +pub struct ImageEncodedVisualizer { + pub data: SpatialViewVisualizerData, + pub images: Vec, +} + +impl Default for ImageEncodedVisualizer { + fn default() -> Self { + Self { + data: SpatialViewVisualizerData::new(Some(SpatialSpaceViewKind::TwoD)), + images: Vec::new(), + } + } +} + +impl IdentifiedViewSystem for ImageEncodedVisualizer { + fn identifier() -> re_viewer_context::ViewSystemIdentifier { + "ImageEncoded".into() + } +} + +impl VisualizerSystem for ImageEncodedVisualizer { + fn visualizer_query_info(&self) -> VisualizerQueryInfo { + VisualizerQueryInfo::from_archetype::() + } + + fn filter_visualizable_entities( + &self, + entities: ApplicableEntities, + context: &dyn VisualizableFilterContext, + ) -> VisualizableEntities { + re_tracing::profile_function!(); + filter_visualizable_2d_entities(entities, context) + } + + fn execute( + &mut self, + ctx: &ViewContext<'_>, + view_query: &ViewQuery<'_>, + context_systems: &ViewContextCollection, + ) -> Result, SpaceViewSystemExecutionError> { + let Some(render_ctx) = ctx.viewer_ctx.render_ctx else { + return Err(SpaceViewSystemExecutionError::NoRenderContextError); + }; + + process_archetype::( + ctx, + view_query, + context_systems, + |ctx, spatial_ctx, results| self.process_image_encoded(ctx, results, spatial_ctx), + )?; + + // TODO(#702): draw order is translated to depth offset, which works fine for opaque images, + // but for everything with transparency, actual drawing order is still important. + // We mitigate this a bit by at least sorting the images within each other. + // Sorting of Images vs DepthImage vs SegmentationImage uses the fact that + // visualizers are executed in the order of their identifiers. + // -> The draw order is always DepthImage then Image then SegmentationImage, + // which happens to be exactly what we want 🙈 + self.images.sort_by_key(|image| { + ( + image.textured_rect.options.depth_offset, + egui::emath::OrderedFloat(image.textured_rect.options.multiplicative_tint.a()), + ) + }); + + let mut draw_data_list = Vec::new(); + + // TODO(wumpf): Can we avoid this copy, maybe let DrawData take an iterator? + let rectangles = self + .images + .iter() + .map(|image| image.textured_rect.clone()) + .collect_vec(); + match re_renderer::renderer::RectangleDrawData::new(render_ctx, &rectangles) { + Ok(draw_data) => { + draw_data_list.push(draw_data.into()); + } + Err(err) => { + re_log::error_once!("Failed to create rectangle draw data from images: {err}"); + } + } + + Ok(draw_data_list) + } + + fn data(&self) -> Option<&dyn std::any::Any> { + Some(self.data.as_any()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_fallback_provider(&self) -> &dyn re_viewer_context::ComponentFallbackProvider { + self + } +} + +impl ImageEncodedVisualizer { + fn process_image_encoded( + &mut self, + ctx: &QueryContext<'_>, + results: &HybridResults<'_>, + spatial_ctx: &SpatialSceneEntityContext<'_>, + ) -> Result<(), SpaceViewSystemExecutionError> { + use re_space_view::RangeResultsExt as _; + + let resolver = ctx.recording().resolver(); + let entity_path = ctx.target_entity_path; + + let blobs = match results.get_required_component_dense::(resolver) { + Some(blobs) => blobs?, + _ => return Ok(()), + }; + + // Unknown is currently interpreted as "Some Color" in most cases. + // TODO(jleibs): Make this more explicit + let meaning = TensorDataMeaning::Unknown; + + let media_types = results.get_or_empty_dense::(resolver)?; + let opacities = results.get_or_empty_dense::(resolver)?; + + for (&(_time, tensor_data_row_id), blobs, media_types, opacities) in range_zip_1x2( + blobs.range_indexed(), + media_types.range_indexed(), + opacities.range_indexed(), + ) { + let Some(blob) = blobs.first() else { + continue; + }; + let media_type = media_types.and_then(|media_types| media_types.first()); + + let tensor = ctx.viewer_ctx.cache.entry(|c: &mut ImageDecodeCache| { + c.entry(tensor_data_row_id, blob, media_type.map(|mt| mt.as_str())) + }); + + let tensor = match tensor { + Ok(tensor) => tensor, + Err(err) => { + re_log::warn_once!( + "Failed to decode ImageEncoded at path {entity_path}: {err}" + ); + continue; + } + }; + + // TODO(andreas): We only support colormap for depth image at this point. + let colormap = None; + + let opacity = opacities.and_then(|opacity| opacity.first()); + + let opacity = opacity.copied().unwrap_or_else(|| self.fallback_for(ctx)); + let multiplicative_tint = + re_renderer::Rgba::from_white_alpha(opacity.0.clamp(0.0, 1.0)); + + if let Some(textured_rect) = tensor_to_textured_rect( + ctx.viewer_ctx, + entity_path, + spatial_ctx, + tensor_data_row_id, + &tensor, + meaning, + multiplicative_tint, + colormap, + ) { + // Only update the bounding box if this is a 2D space view. + // This is avoids a cyclic relationship where the image plane grows + // the bounds which in turn influence the size of the image plane. + // See: https://github.com/rerun-io/rerun/issues/3728 + if spatial_ctx.space_view_class_identifier == SpatialSpaceView2D::identifier() { + self.data.add_bounding_box( + entity_path.hash(), + bounding_box_for_textured_rect(&textured_rect), + spatial_ctx.world_from_entity, + ); + } + + self.images.push(PickableImageRect { + ent_path: entity_path.clone(), + textured_rect, + }); + } + } + + Ok(()) + } +} + +impl TypedComponentFallbackProvider for ImageEncodedVisualizer { + fn fallback_for(&self, _ctx: &re_viewer_context::QueryContext<'_>) -> Opacity { + 1.0.into() + } +} + +impl TypedComponentFallbackProvider for ImageEncodedVisualizer { + fn fallback_for(&self, _ctx: &QueryContext<'_>) -> DrawOrder { + DrawOrder::DEFAULT_IMAGE + } +} + +re_viewer_context::impl_component_fallback_provider!(ImageEncodedVisualizer => [DrawOrder, Opacity]); diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs index 9df2935a88cd6..fd6e6b0820c7c 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs @@ -7,6 +7,7 @@ mod boxes2d; mod boxes3d; mod cameras; mod depth_images; +mod image_encoded; mod images; mod lines2d; mod lines3d; @@ -71,6 +72,7 @@ pub fn register_2d_spatial_visualizers( system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; + system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; system_registry.register_visualizer::()?; diff --git a/crates/viewer/re_viewer_context/Cargo.toml b/crates/viewer/re_viewer_context/Cargo.toml index 63d7564748470..aae44b9719152 100644 --- a/crates/viewer/re_viewer_context/Cargo.toml +++ b/crates/viewer/re_viewer_context/Cargo.toml @@ -48,6 +48,7 @@ egui-wgpu.workspace = true egui.workspace = true glam = { workspace = true, features = ["serde"] } half.workspace = true +image = { workspace = true, features = ["jpeg", "png"] } indexmap = { workspace = true, features = ["std", "serde"] } itertools.workspace = true linked-hash-map.workspace = true diff --git a/crates/viewer/re_viewer_context/src/lib.rs b/crates/viewer/re_viewer_context/src/lib.rs index 13dbf09596d67..7abf12e4c2f45 100644 --- a/crates/viewer/re_viewer_context/src/lib.rs +++ b/crates/viewer/re_viewer_context/src/lib.rs @@ -73,7 +73,7 @@ pub use space_view::{ }; pub use store_context::StoreContext; pub use store_hub::StoreHub; -pub use tensor::{TensorDecodeCache, TensorStats, TensorStatsCache}; +pub use tensor::{ImageDecodeCache, TensorDecodeCache, TensorStats, TensorStatsCache}; pub use time_control::{Looping, PlayState, TimeControl, TimeView}; pub use typed_entity_collections::{ ApplicableEntities, IndicatedEntities, PerVisualizer, VisualizableEntities, diff --git a/crates/viewer/re_viewer_context/src/tensor/image_decode_cache.rs b/crates/viewer/re_viewer_context/src/tensor/image_decode_cache.rs new file mode 100644 index 0000000000000..7c6fb0cf0b2c5 --- /dev/null +++ b/crates/viewer/re_viewer_context/src/tensor/image_decode_cache.rs @@ -0,0 +1,132 @@ +use re_chunk::RowId; +use re_types::tensor_data::{DecodedTensor, TensorImageLoadError}; + +use crate::Cache; + +struct DecodedImageResult { + /// Cached `Result` from decoding the image + tensor_result: Result, + + /// Total memory used by this `Tensor`.\ + memory_used: u64, + + /// Which [`ImageDecodeCache::generation`] was this `Tensor` last used? + last_use_generation: u64, +} + +/// Caches decoded tensors using a [`RowId`], i.e. a specific instance of +/// a `TensorData` component. +#[derive(Default)] +pub struct ImageDecodeCache { + cache: ahash::HashMap, + memory_used: u64, + generation: u64, +} + +#[allow(clippy::map_err_ignore)] +impl ImageDecodeCache { + /// Decode some image data and cache the result. + /// + /// The key should be the `RowId` of the blob. + /// NOTE: images are never batched atm (they are mono-archetypes), + /// so we don't need the instance id here. + pub fn entry( + &mut self, + key: RowId, + image_bytes: &[u8], + media_type: Option<&str>, + ) -> Result { + re_tracing::profile_function!(); + + let lookup = self.cache.entry(key).or_insert_with(|| { + let tensor_result = decode_image(image_bytes, media_type); + let memory_used = match &tensor_result { + Ok(tensor) => tensor.size_in_bytes() as u64, + Err(_) => 0, + }; + + self.memory_used += memory_used; + + DecodedImageResult { + tensor_result, + memory_used, + last_use_generation: 0, + } + }); + lookup.last_use_generation = self.generation; + lookup.tensor_result.clone() + } +} + +fn decode_image( + image_bytes: &[u8], + media_type: Option<&str>, +) -> Result { + re_tracing::profile_function!(); + + let mut reader = image::io::Reader::new(std::io::Cursor::new(image_bytes)); + + if let Some(media_type) = media_type { + if let Some(format) = image::ImageFormat::from_mime_type(media_type) { + reader.set_format(format); + } else { + re_log::warn!("Unsupported image MediaType/MIME: {media_type:?}"); + } + } + + if reader.format().is_none() { + if let Ok(format) = image::guess_format(image_bytes) { + // Weirdly enough, `reader.decode` doesn't do this for us. + reader.set_format(format); + } + } + + let img = reader.decode()?; + + DecodedTensor::from_image(img) +} + +impl Cache for ImageDecodeCache { + fn begin_frame(&mut self) { + #[cfg(not(target_arch = "wasm32"))] + let max_decode_cache_use = 4_000_000_000; + + #[cfg(target_arch = "wasm32")] + let max_decode_cache_use = 1_000_000_000; + + // TODO(jleibs): a more incremental purging mechanism, maybe switching to an LRU Cache + // would likely improve the behavior. + + if self.memory_used > max_decode_cache_use { + self.purge_memory(); + } + + self.generation += 1; + } + + fn purge_memory(&mut self) { + re_tracing::profile_function!(); + + // Very aggressively flush everything not used in this frame + + let before = self.memory_used; + + self.cache.retain(|_, ci| { + let retain = ci.last_use_generation == self.generation; + if !retain { + self.memory_used -= ci.memory_used; + } + retain + }); + + re_log::trace!( + "Flushed tensor decode cache. Before: {:.2} GB. After: {:.2} GB", + before as f64 / 1e9, + self.memory_used as f64 / 1e9, + ); + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } +} diff --git a/crates/viewer/re_viewer_context/src/tensor/mod.rs b/crates/viewer/re_viewer_context/src/tensor/mod.rs index dfba69bb5223d..cc4bb474acfe1 100644 --- a/crates/viewer/re_viewer_context/src/tensor/mod.rs +++ b/crates/viewer/re_viewer_context/src/tensor/mod.rs @@ -1,9 +1,11 @@ // TODO(andreas): Move tensor utilities to a tensor specific crate? +mod image_decode_cache; mod tensor_decode_cache; mod tensor_stats; mod tensor_stats_cache; +pub use image_decode_cache::ImageDecodeCache; pub use tensor_decode_cache::TensorDecodeCache; pub use tensor_stats::TensorStats; pub use tensor_stats_cache::TensorStatsCache; diff --git a/docs/content/reference/types/archetypes.md b/docs/content/reference/types/archetypes.md index 2b9fa3cca4efb..ecb3095dcc081 100644 --- a/docs/content/reference/types/archetypes.md +++ b/docs/content/reference/types/archetypes.md @@ -16,6 +16,7 @@ This page lists all built-in archetypes. * [`DepthImage`](archetypes/depth_image.md): A depth image. * [`Image`](archetypes/image.md): A monochrome or color image. +* [`ImageEncoded`](archetypes/image_encoded.md): An image encoded as e.g. a JPEG or PNG. * [`SegmentationImage`](archetypes/segmentation_image.md): An image made up of integer [`components.ClassId`](https://rerun.io/docs/reference/types/components/class_id)s. * [`Tensor`](archetypes/tensor.md): An N-dimensional array of numbers. diff --git a/docs/content/reference/types/archetypes/.gitattributes b/docs/content/reference/types/archetypes/.gitattributes index c8dbda543c26e..60a307da06cad 100644 --- a/docs/content/reference/types/archetypes/.gitattributes +++ b/docs/content/reference/types/archetypes/.gitattributes @@ -12,6 +12,7 @@ clear.md linguist-generated=true depth_image.md linguist-generated=true disconnected_space.md linguist-generated=true image.md linguist-generated=true +image_encoded.md linguist-generated=true line_strips2d.md linguist-generated=true line_strips3d.md linguist-generated=true mesh3d.md linguist-generated=true diff --git a/docs/content/reference/types/archetypes/image.md b/docs/content/reference/types/archetypes/image.md index e53fb64356d17..3d1a999afc1f9 100644 --- a/docs/content/reference/types/archetypes/image.md +++ b/docs/content/reference/types/archetypes/image.md @@ -17,8 +17,10 @@ As such, the shape of the [`components.TensorData`](https://rerun.io/docs/refere Leading and trailing unit-dimensions are ignored, so that `1x480x640x3x1` is treated as a `480x640x3` RGB image. -Rerun also supports compressed image encoded as JPEG, N12, and YUY2. -Using these formats can save a lot of bandwidth and memory. +Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.ImageEncoded`](https://rerun.io/docs/reference/types/archetypes/image_encoded?speculative-link). +Compressing images can save a lot of bandwidth and memory. + +See also [`components.TensorData`](https://rerun.io/docs/reference/types/components/tensor_data) and [`datatypes.TensorBuffer`](https://rerun.io/docs/reference/types/datatypes/tensor_buffer). ## Components diff --git a/docs/content/reference/types/archetypes/image_encoded.md b/docs/content/reference/types/archetypes/image_encoded.md new file mode 100644 index 0000000000000..06ebfe7de85a1 --- /dev/null +++ b/docs/content/reference/types/archetypes/image_encoded.md @@ -0,0 +1,32 @@ +--- +title: "ImageEncoded" +--- + + +An image encoded as e.g. a JPEG or PNG. + +Rerun also supports uncompressed images with the [`archetypes.Image`](https://rerun.io/docs/reference/types/archetypes/image). + +## Components + +**Required**: [`Blob`](../components/blob.md) + +**Recommended**: [`MediaType`](../components/media_type.md) + +**Optional**: [`Opacity`](../components/opacity.md), [`DrawOrder`](../components/draw_order.md) + +## Shown in +* [Spatial2DView](../views/spatial2d_view.md) +* [Spatial3DView](../views/spatial3d_view.md) (if logged under a projection) + +## API reference links + * 🌊 [C++ API docs for `ImageEncoded`](https://ref.rerun.io/docs/cpp/stable/structrerun_1_1archetypes_1_1ImageEncoded.html?speculative-link) + * 🐍 [Python API docs for `ImageEncoded`](https://ref.rerun.io/docs/python/stable/common/archetypes?speculative-link#rerun.archetypes.ImageEncoded) + * 🦀 [Rust API docs for `ImageEncoded`](https://docs.rs/rerun/latest/rerun/archetypes/struct.ImageEncoded.html?speculative-link) + +## Example + +### image_encoded + +snippet: archetypes/image_encoded + diff --git a/docs/content/reference/types/components/blob.md b/docs/content/reference/types/components/blob.md index d1d50af1974d8..b26ecd8ae7baf 100644 --- a/docs/content/reference/types/components/blob.md +++ b/docs/content/reference/types/components/blob.md @@ -18,3 +18,4 @@ A binary blob of data. ## Used by * [`Asset3D`](../archetypes/asset3d.md) +* [`ImageEncoded`](../archetypes/image_encoded.md?speculative-link) diff --git a/docs/content/reference/types/components/draw_order.md b/docs/content/reference/types/components/draw_order.md index d184b91b56ebb..5baf786ab68f4 100644 --- a/docs/content/reference/types/components/draw_order.md +++ b/docs/content/reference/types/components/draw_order.md @@ -25,6 +25,7 @@ Draw order for entities with the same draw order is generally undefined. * [`Arrows2D`](../archetypes/arrows2d.md) * [`Boxes2D`](../archetypes/boxes2d.md) * [`DepthImage`](../archetypes/depth_image.md) +* [`ImageEncoded`](../archetypes/image_encoded.md?speculative-link) * [`Image`](../archetypes/image.md) * [`LineStrips2D`](../archetypes/line_strips2d.md) * [`Points2D`](../archetypes/points2d.md) diff --git a/docs/content/reference/types/components/media_type.md b/docs/content/reference/types/components/media_type.md index ffc257da809ee..33cf60cf5bf7c 100644 --- a/docs/content/reference/types/components/media_type.md +++ b/docs/content/reference/types/components/media_type.md @@ -21,4 +21,5 @@ consulted at . ## Used by * [`Asset3D`](../archetypes/asset3d.md) +* [`ImageEncoded`](../archetypes/image_encoded.md?speculative-link) * [`TextDocument`](../archetypes/text_document.md) diff --git a/docs/content/reference/types/components/opacity.md b/docs/content/reference/types/components/opacity.md index 202b35662c593..5632a4613dc53 100644 --- a/docs/content/reference/types/components/opacity.md +++ b/docs/content/reference/types/components/opacity.md @@ -20,5 +20,6 @@ Unless otherwise specified, the default value is 1. ## Used by +* [`ImageEncoded`](../archetypes/image_encoded.md?speculative-link) * [`Image`](../archetypes/image.md) * [`SegmentationImage`](../archetypes/segmentation_image.md) diff --git a/docs/content/reference/types/views/spatial2d_view.md b/docs/content/reference/types/views/spatial2d_view.md index 48774402817e4..c50dffa1fa0ae 100644 --- a/docs/content/reference/types/views/spatial2d_view.md +++ b/docs/content/reference/types/views/spatial2d_view.md @@ -50,6 +50,7 @@ snippet: views/spatial2d * [`DepthImage`](../archetypes/depth_image.md) * [`DisconnectedSpace`](../archetypes/disconnected_space.md) * [`Image`](../archetypes/image.md) +* [`ImageEncoded`](../archetypes/image_encoded.md) * [`LineStrips2D`](../archetypes/line_strips2d.md) * [`Pinhole`](../archetypes/pinhole.md) * [`Pinhole`](../archetypes/pinhole.md) diff --git a/docs/content/reference/types/views/spatial3d_view.md b/docs/content/reference/types/views/spatial3d_view.md index 90ab22d654ef0..3432fb65cad51 100644 --- a/docs/content/reference/types/views/spatial3d_view.md +++ b/docs/content/reference/types/views/spatial3d_view.md @@ -53,6 +53,7 @@ snippet: views/spatial3d * [`Boxes2D`](../archetypes/boxes2d.md) (if logged under a projection) * [`DepthImage`](../archetypes/depth_image.md) (if logged under a projection) * [`Image`](../archetypes/image.md) (if logged under a projection) +* [`ImageEncoded`](../archetypes/image_encoded.md) (if logged under a projection) * [`LineStrips2D`](../archetypes/line_strips2d.md) (if logged under a projection) * [`Points2D`](../archetypes/points2d.md) (if logged under a projection) * [`SegmentationImage`](../archetypes/segmentation_image.md) (if logged under a projection) diff --git a/docs/snippets/all/archetypes/image_encoded.cpp b/docs/snippets/all/archetypes/image_encoded.cpp new file mode 100644 index 0000000000000..66702bbf106ad --- /dev/null +++ b/docs/snippets/all/archetypes/image_encoded.cpp @@ -0,0 +1,20 @@ +// Create and log a image. + +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +int main() { + const auto rec = rerun::RecordingStream("rerun_example_image_encoded"); + rec.spawn().exit_on_failure(); + + fs::path image_filepath = fs::path(__FILE__).parent_path() / + "../../../../crates/viewer/re_ui/data/logo_dark_mode.png"; + + rec.log("image", rerun::ImageEncoded::from_file(image_filepath).value_or_throw()); +} diff --git a/docs/snippets/all/archetypes/image_encoded.py b/docs/snippets/all/archetypes/image_encoded.py new file mode 100644 index 0000000000000..e7bd69977243f --- /dev/null +++ b/docs/snippets/all/archetypes/image_encoded.py @@ -0,0 +1,14 @@ +"""Create and log an image.""" + +from pathlib import Path + +import rerun as rr + +current_file_dir = Path(__file__).parent +target_file_path = current_file_dir / "../../../../crates/viewer/re_ui/data/logo_dark_mode.png" +with open(target_file_path, "rb") as file: + file_bytes = file.read() + +rr.init("rerun_example_image_encoded", spawn=True) + +rr.log("image", rr.ImageEncoded(file_bytes)) diff --git a/docs/snippets/all/archetypes/image_encoded.rs b/docs/snippets/all/archetypes/image_encoded.rs new file mode 100644 index 0000000000000..4b7893ec2bcb0 --- /dev/null +++ b/docs/snippets/all/archetypes/image_encoded.rs @@ -0,0 +1,11 @@ +//! Log a PNG image + +fn main() -> Result<(), Box> { + let rec = rerun::RecordingStreamBuilder::new("rerun_example_image_encoded").spawn()?; + + let image = include_bytes!("../../../../crates/viewer/re_ui/data/logo_dark_mode.png"); + + rec.log("image", &rerun::ImageEncoded::from_bytes(image.to_vec()))?; + + Ok(()) +} diff --git a/rerun_cpp/src/rerun/archetypes.hpp b/rerun_cpp/src/rerun/archetypes.hpp index 1075260a8335c..80af29be5264e 100644 --- a/rerun_cpp/src/rerun/archetypes.hpp +++ b/rerun_cpp/src/rerun/archetypes.hpp @@ -13,6 +13,7 @@ #include "archetypes/depth_image.hpp" #include "archetypes/disconnected_space.hpp" #include "archetypes/image.hpp" +#include "archetypes/image_encoded.hpp" #include "archetypes/line_strips2d.hpp" #include "archetypes/line_strips3d.hpp" #include "archetypes/mesh3d.hpp" diff --git a/rerun_cpp/src/rerun/archetypes/.gitattributes b/rerun_cpp/src/rerun/archetypes/.gitattributes index bb3c86280a874..33a93142ec768 100644 --- a/rerun_cpp/src/rerun/archetypes/.gitattributes +++ b/rerun_cpp/src/rerun/archetypes/.gitattributes @@ -23,6 +23,8 @@ disconnected_space.cpp linguist-generated=true disconnected_space.hpp linguist-generated=true image.cpp linguist-generated=true image.hpp linguist-generated=true +image_encoded.cpp linguist-generated=true +image_encoded.hpp linguist-generated=true line_strips2d.cpp linguist-generated=true line_strips2d.hpp linguist-generated=true line_strips3d.cpp linguist-generated=true diff --git a/rerun_cpp/src/rerun/archetypes/asset3d.hpp b/rerun_cpp/src/rerun/archetypes/asset3d.hpp index 86cc96fc763a5..22131f95f6e9f 100644 --- a/rerun_cpp/src/rerun/archetypes/asset3d.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset3d.hpp @@ -80,10 +80,6 @@ namespace rerun::archetypes { public: // Extensions to generated type defined in 'asset3d_ext.cpp' - static std::optional guess_media_type( - const std::filesystem::path& path - ); - /// Creates a new `Asset3D` from the file contents at `path`. /// /// The `MediaType` will be guessed from the file extension. @@ -97,7 +93,8 @@ namespace rerun::archetypes { /// If no `MediaType` is specified, the Rerun Viewer will try to guess one from the data /// at render-time. If it can't, rendering will fail with an error. static Asset3D from_bytes( - rerun::Collection bytes, std::optional media_type + rerun::Collection bytes, + std::optional media_type = {} ) { // TODO(cmc): we could try and guess using magic bytes here, like rust does. Asset3D asset = Asset3D(std::move(bytes)); @@ -105,6 +102,10 @@ namespace rerun::archetypes { return asset; } + static std::optional guess_media_type( + const std::filesystem::path& path + ); + public: Asset3D() = default; Asset3D(Asset3D&& other) = default; diff --git a/rerun_cpp/src/rerun/archetypes/asset3d_ext.cpp b/rerun_cpp/src/rerun/archetypes/asset3d_ext.cpp index d6edc70a31a85..b64ee16b62e68 100644 --- a/rerun_cpp/src/rerun/archetypes/asset3d_ext.cpp +++ b/rerun_cpp/src/rerun/archetypes/asset3d_ext.cpp @@ -17,10 +17,6 @@ namespace rerun::archetypes { #if 0 // - static std::optional guess_media_type( - const std::filesystem::path& path - ); - /// Creates a new `Asset3D` from the file contents at `path`. /// /// The `MediaType` will be guessed from the file extension. @@ -34,7 +30,7 @@ namespace rerun::archetypes { /// If no `MediaType` is specified, the Rerun Viewer will try to guess one from the data /// at render-time. If it can't, rendering will fail with an error. static Asset3D from_bytes( - rerun::Collection bytes, std::optional media_type + rerun::Collection bytes, std::optional media_type = {} ) { // TODO(cmc): we could try and guess using magic bytes here, like rust does. Asset3D asset = Asset3D(std::move(bytes)); @@ -42,6 +38,11 @@ namespace rerun::archetypes { return asset; } + + static std::optional guess_media_type( + const std::filesystem::path& path + ); + // #endif diff --git a/rerun_cpp/src/rerun/archetypes/image.hpp b/rerun_cpp/src/rerun/archetypes/image.hpp index 37325b09f0176..1a83f50146803 100644 --- a/rerun_cpp/src/rerun/archetypes/image.hpp +++ b/rerun_cpp/src/rerun/archetypes/image.hpp @@ -32,9 +32,10 @@ namespace rerun::archetypes { /// Leading and trailing unit-dimensions are ignored, so that /// `1x480x640x3x1` is treated as a `480x640x3` RGB image. /// - /// Rerun also supports compressed image encoded as JPEG, N12, and YUY2. - /// Using these formats can save a lot of bandwidth and memory. - /// See [`rerun::datatypes::TensorBuffer`] for more. + /// Rerun also supports compressed images (JPEG, PNG, …), using `archetypes::ImageEncoded`. + /// Compressing images can save a lot of bandwidth and memory. + /// + /// See also `components::TensorData` and `datatypes::TensorBuffer`. /// /// Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally, /// data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays. diff --git a/rerun_cpp/src/rerun/archetypes/image_encoded.cpp b/rerun_cpp/src/rerun/archetypes/image_encoded.cpp new file mode 100644 index 0000000000000..576b622408c61 --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/image_encoded.cpp @@ -0,0 +1,48 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs". + +#include "image_encoded.hpp" + +#include "../collection_adapter_builtins.hpp" + +namespace rerun::archetypes {} + +namespace rerun { + + Result> AsComponents::serialize( + const archetypes::ImageEncoded& archetype + ) { + using namespace archetypes; + std::vector cells; + cells.reserve(5); + + { + auto result = DataCell::from_loggable(archetype.blob); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + if (archetype.media_type.has_value()) { + auto result = DataCell::from_loggable(archetype.media_type.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + if (archetype.opacity.has_value()) { + auto result = DataCell::from_loggable(archetype.opacity.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + if (archetype.draw_order.has_value()) { + auto result = DataCell::from_loggable(archetype.draw_order.value()); + RR_RETURN_NOT_OK(result.error); + cells.push_back(std::move(result.value)); + } + { + auto indicator = ImageEncoded::IndicatorComponent(); + auto result = DataCell::from_loggable(indicator); + RR_RETURN_NOT_OK(result.error); + cells.emplace_back(std::move(result.value)); + } + + return cells; + } +} // namespace rerun diff --git a/rerun_cpp/src/rerun/archetypes/image_encoded.hpp b/rerun_cpp/src/rerun/archetypes/image_encoded.hpp new file mode 100644 index 0000000000000..7897eae1f2f8e --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/image_encoded.hpp @@ -0,0 +1,155 @@ +// DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/cpp/mod.rs +// Based on "crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs". + +#pragma once + +#include "../collection.hpp" +#include "../compiler_utils.hpp" +#include "../components/blob.hpp" +#include "../components/draw_order.hpp" +#include "../components/media_type.hpp" +#include "../components/opacity.hpp" +#include "../data_cell.hpp" +#include "../indicator_component.hpp" +#include "../result.hpp" + +#include +#include +#include +#include +#include + +namespace rerun::archetypes { + /// **Archetype**: An image encoded as e.g. a JPEG or PNG. + /// + /// Rerun also supports uncompressed images with the `archetypes::Image`. + /// + /// ## Example + /// + /// ### image_encoded: + /// ```cpp + /// #include + /// + /// #include + /// #include + /// #include + /// #include + /// + /// namespace fs = std::filesystem; + /// + /// int main() { + /// const auto rec = rerun::RecordingStream("rerun_example_image_encoded"); + /// rec.spawn().exit_on_failure(); + /// + /// fs::path image_filepath = fs::path(__FILE__).parent_path() / + /// "../../../../crates/viewer/re_ui/data/logo_dark_mode.png"; + /// + /// rec.log("image", rerun::ImageEncoded::from_file(image_filepath).value_or_throw()); + /// } + /// ``` + struct ImageEncoded { + /// The encoded content of some image file, e.g. a PNG or JPEG. + rerun::components::Blob blob; + + /// The Media Type of the asset. + /// + /// Supported values: + /// * `image/jpeg` + /// * `image/png` + /// + /// If omitted, the viewer will try to guess from the data blob. + /// If it cannot guess, it won't be able to render the asset. + std::optional media_type; + + /// Opacity of the image, useful for layering several images. + /// + /// Defaults to 1.0 (fully opaque). + std::optional opacity; + + /// An optional floating point value that specifies the 2D drawing order. + /// + /// Objects with higher values are drawn on top of those with lower values. + std::optional draw_order; + + public: + static constexpr const char IndicatorComponentName[] = + "rerun.components.ImageEncodedIndicator"; + + /// Indicator component, used to identify the archetype when converting to a list of components. + using IndicatorComponent = rerun::components::IndicatorComponent; + + public: + // Extensions to generated type defined in 'image_encoded_ext.cpp' + + /// Create a new `ImageEncoded` from the contents of a file on disk, e.g. a PNG or JPEG. + static Result from_file(const std::filesystem::path& filepath); + + /// Create a new `ImageEncoded` from the contents of an image file, like a PNG or JPEG. + /// + /// If no `MediaType` is specified, the Rerun Viewer will try to guess one from the data + /// at render-time. If it can't, rendering will fail with an error. + static ImageEncoded from_bytes( + rerun::Collection image_contents, + std::optional media_type = {} + ) { + ImageEncoded image; + image.blob = image_contents; + image.media_type = media_type; + return image; + } + + static std::optional guess_media_type( + const std::filesystem::path& path + ); + + public: + ImageEncoded() = default; + ImageEncoded(ImageEncoded&& other) = default; + + /// The Media Type of the asset. + /// + /// Supported values: + /// * `image/jpeg` + /// * `image/png` + /// + /// If omitted, the viewer will try to guess from the data blob. + /// If it cannot guess, it won't be able to render the asset. + ImageEncoded with_media_type(rerun::components::MediaType _media_type) && { + media_type = std::move(_media_type); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + + /// Opacity of the image, useful for layering several images. + /// + /// Defaults to 1.0 (fully opaque). + ImageEncoded with_opacity(rerun::components::Opacity _opacity) && { + opacity = std::move(_opacity); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + + /// An optional floating point value that specifies the 2D drawing order. + /// + /// Objects with higher values are drawn on top of those with lower values. + ImageEncoded with_draw_order(rerun::components::DrawOrder _draw_order) && { + draw_order = std::move(_draw_order); + // See: https://github.com/rerun-io/rerun/issues/4027 + RR_WITH_MAYBE_UNINITIALIZED_DISABLED(return std::move(*this);) + } + }; + +} // namespace rerun::archetypes + +namespace rerun { + /// \private + template + struct AsComponents; + + /// \private + template <> + struct AsComponents { + /// Serialize all set component batches. + static Result> serialize(const archetypes::ImageEncoded& archetype); + }; +} // namespace rerun diff --git a/rerun_cpp/src/rerun/archetypes/image_encoded_ext.cpp b/rerun_cpp/src/rerun/archetypes/image_encoded_ext.cpp new file mode 100644 index 0000000000000..c2fa0b3b01737 --- /dev/null +++ b/rerun_cpp/src/rerun/archetypes/image_encoded_ext.cpp @@ -0,0 +1,85 @@ +#include "../error.hpp" +#include "image_encoded.hpp" + +#include "../collection_adapter_builtins.hpp" + +#include +#include +#include + +// It's undefined behavior to pre-declare std types, see http://www.gotw.ca/gotw/034.htm +// We want to use `std::filesystem::path`, so we have it include it in the header. +// + +#include + +// + +// Uncomment for better auto-complete while editing the extension. +// #define EDIT_EXTENSION + +namespace rerun::archetypes { + +#ifdef EDIT_EXTENSION + // + + /// Create a new `ImageEncoded` from the contents of a file on disk, e.g. a PNG or JPEG. + static Result from_file(const std::filesystem::path& filepath); + + /// Create a new `ImageEncoded` from the contents of an image file, like a PNG or JPEG. + /// + /// If no `MediaType` is specified, the Rerun Viewer will try to guess one from the data + /// at render-time. If it can't, rendering will fail with an error. + static ImageEncoded from_bytes( + rerun::Collection image_contents, + std::optional media_type = {} + ) { + ImageEncoded image; + image.blob = image_contents; + image.media_type = media_type; + return image; + } + + static std::optional guess_media_type( + const std::filesystem::path& path + ); + + // +#endif + + Result ImageEncoded::from_file(const std::filesystem::path& filepath) { + std::ifstream file(filepath, std::ios::binary); + if (!file) { + return Error(ErrorCode::FileRead, filepath.string()); + } + + // Get the size of the file: + file.seekg(0, std::ios::end); + auto file_size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector file_bytes(static_cast(file_size)); + + if (!file.read(file_bytes.data(), static_cast(file_size))) { + return Error(ErrorCode::FileRead, filepath.string()); + } + + return ImageEncoded::from_bytes(file_bytes, ImageEncoded::guess_media_type(filepath)); + } + + std::optional ImageEncoded::guess_media_type( + const std::filesystem::path& path + ) { + std::filesystem::path file_path(path); + std::string ext = file_path.extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); + + if (ext == ".jpg" || ext == ".jpeg") { + return rerun::components::MediaType::jpeg(); + } else if (ext == ".png") { + return rerun::components::MediaType::png(); + } else { + return std::nullopt; + } + } +} // namespace rerun::archetypes diff --git a/rerun_cpp/src/rerun/components/media_type.hpp b/rerun_cpp/src/rerun/components/media_type.hpp index db79264eb48ca..e66c902359955 100644 --- a/rerun_cpp/src/rerun/components/media_type.hpp +++ b/rerun_cpp/src/rerun/components/media_type.hpp @@ -40,6 +40,24 @@ namespace rerun::components { return "text/markdown"; } + // ------------------------------------------------ + // Images: + + /// [JPEG image](https://en.wikipedia.org/wiki/JPEG): `image/jpeg`. + static MediaType jpeg() { + return "image/jpeg"; + } + + /// [PNG image](https://en.wikipedia.org/wiki/PNG): `image/png`. + /// + /// + static MediaType png() { + return "image/png"; + } + + // ------------------------------------------------ + // Meshes: + /// [`glTF`](https://en.wikipedia.org/wiki/GlTF): `model/gltf+json`. /// /// diff --git a/rerun_cpp/src/rerun/components/media_type_ext.cpp b/rerun_cpp/src/rerun/components/media_type_ext.cpp index 6e9d8579e89d2..4380a520597f9 100644 --- a/rerun_cpp/src/rerun/components/media_type_ext.cpp +++ b/rerun_cpp/src/rerun/components/media_type_ext.cpp @@ -32,6 +32,24 @@ namespace rerun { return "text/markdown"; } + // ------------------------------------------------ + // Images: + + /// [JPEG image](https://en.wikipedia.org/wiki/JPEG): `image/jpeg`. + static MediaType jpeg() { + return "image/jpeg"; + } + + /// [PNG image](https://en.wikipedia.org/wiki/PNG): `image/png`. + /// + /// + static MediaType png() { + return "image/png"; + } + + // ------------------------------------------------ + // Meshes: + /// [`glTF`](https://en.wikipedia.org/wiki/GlTF): `model/gltf+json`. /// /// diff --git a/rerun_cpp/src/rerun/error.hpp b/rerun_cpp/src/rerun/error.hpp index d669423d81420..baa1d83ac268b 100644 --- a/rerun_cpp/src/rerun/error.hpp +++ b/rerun_cpp/src/rerun/error.hpp @@ -40,6 +40,7 @@ namespace rerun { InvalidSocketAddress, InvalidComponentTypeHandle, InvalidTensorDimension, + FileRead, // Recording stream errors _CategoryRecordingStream = 0x0000'0100, diff --git a/rerun_notebook/package-lock.json b/rerun_notebook/package-lock.json index 3f7ce291c5567..8b1e9db972a58 100644 --- a/rerun_notebook/package-lock.json +++ b/rerun_notebook/package-lock.json @@ -17,7 +17,7 @@ }, "../rerun_js/web-viewer": { "name": "@rerun-io/web-viewer", - "version": "0.17.0-alpha.8", + "version": "0.18.0-alpha.1+dev", "license": "MIT", "devDependencies": { "dts-buddy": "^0.3.0", diff --git a/rerun_py/docs/gen_common_index.py b/rerun_py/docs/gen_common_index.py index 92aeaa0e22673..919f66f43980b 100755 --- a/rerun_py/docs/gen_common_index.py +++ b/rerun_py/docs/gen_common_index.py @@ -155,6 +155,7 @@ class Section: class_list=[ "archetypes.DepthImage", "archetypes.Image", + "archetypes.ImageEncoded", "ImageEncoded", "archetypes.SegmentationImage", ], diff --git a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes index 689720b6cbf05..9d3630483f6b3 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes +++ b/rerun_py/rerun_sdk/rerun/archetypes/.gitattributes @@ -13,6 +13,7 @@ clear.py linguist-generated=true depth_image.py linguist-generated=true disconnected_space.py linguist-generated=true image.py linguist-generated=true +image_encoded.py linguist-generated=true line_strips2d.py linguist-generated=true line_strips3d.py linguist-generated=true mesh3d.py linguist-generated=true diff --git a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py index 890f04d766bb6..0b80ae96e39c4 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/__init__.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/__init__.py @@ -13,6 +13,7 @@ from .depth_image import DepthImage from .disconnected_space import DisconnectedSpace from .image import Image +from .image_encoded import ImageEncoded from .line_strips2d import LineStrips2D from .line_strips3d import LineStrips3D from .mesh3d import Mesh3D @@ -41,6 +42,7 @@ "DepthImage", "DisconnectedSpace", "Image", + "ImageEncoded", "LineStrips2D", "LineStrips3D", "Mesh3D", diff --git a/rerun_py/rerun_sdk/rerun/archetypes/image.py b/rerun_py/rerun_sdk/rerun/archetypes/image.py index 4ffc52bd63a94..e3e9e6882468b 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/image.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/image.py @@ -36,10 +36,12 @@ class Image(ImageExt, Archetype): Leading and trailing unit-dimensions are ignored, so that `1x480x640x3x1` is treated as a `480x640x3` RGB image. - Rerun also supports compressed image encoded as JPEG, N12, and YUY2. - Using these formats can save a lot of bandwidth and memory. - To compress an image, use [`rerun.Image.compress`][]. - To pass in an already encoded image, use [`rerun.ImageEncoded`][]. + Rerun also supports compressed images (JPEG, PNG, …), using [`archetypes.ImageEncoded`][rerun.archetypes.ImageEncoded]. + Compressing images can save a lot of bandwidth and memory. + + You can compress an image using [`rerun.Image.compress`][]. + + See also [`components.TensorData`][rerun.components.TensorData] and [`datatypes.TensorBuffer`][rerun.datatypes.TensorBuffer]. Example ------- diff --git a/rerun_py/rerun_sdk/rerun/archetypes/image_encoded.py b/rerun_py/rerun_sdk/rerun/archetypes/image_encoded.py new file mode 100644 index 0000000000000..510dcd99ec702 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/archetypes/image_encoded.py @@ -0,0 +1,113 @@ +# DO NOT EDIT! This file was auto-generated by crates/build/re_types_builder/src/codegen/python/mod.rs +# Based on "crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs". + +# You can extend this class by creating a "ImageEncodedExt" class in "image_encoded_ext.py". + +from __future__ import annotations + +from attrs import define, field + +from .. import components +from .._baseclasses import ( + Archetype, +) +from .image_encoded_ext import ImageEncodedExt + +__all__ = ["ImageEncoded"] + + +@define(str=False, repr=False, init=False) +class ImageEncoded(ImageEncodedExt, Archetype): + """ + **Archetype**: An image encoded as e.g. a JPEG or PNG. + + Rerun also supports uncompressed images with the [`archetypes.Image`][rerun.archetypes.Image]. + + To compress an image, use [`rerun.Image.compress`][]. + + Example + ------- + ### `image_encoded`: + ```python + from pathlib import Path + + import rerun as rr + + current_file_dir = Path(__file__).parent + target_file_path = current_file_dir / "../../../../crates/viewer/re_ui/data/logo_dark_mode.png" + with open(target_file_path, "rb") as file: + file_bytes = file.read() + + rr.init("rerun_example_image_encoded", spawn=True) + + rr.log("image", rr.ImageEncoded(file_bytes)) + ``` + + """ + + # __init__ can be found in image_encoded_ext.py + + def __attrs_clear__(self) -> None: + """Convenience method for calling `__attrs_init__` with all `None`s.""" + self.__attrs_init__( + blob=None, # type: ignore[arg-type] + media_type=None, # type: ignore[arg-type] + opacity=None, # type: ignore[arg-type] + draw_order=None, # type: ignore[arg-type] + ) + + @classmethod + def _clear(cls) -> ImageEncoded: + """Produce an empty ImageEncoded, bypassing `__init__`.""" + inst = cls.__new__(cls) + inst.__attrs_clear__() + return inst + + blob: components.BlobBatch = field( + metadata={"component": "required"}, + converter=components.BlobBatch._required, # type: ignore[misc] + ) + # The encoded content of some image file, e.g. a PNG or JPEG. + # + # (Docstring intentionally commented out to hide this field from the docs) + + media_type: components.MediaTypeBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.MediaTypeBatch._optional, # type: ignore[misc] + ) + # The Media Type of the asset. + # + # Supported values: + # * `image/jpeg` + # * `image/png` + # + # If omitted, the viewer will try to guess from the data blob. + # If it cannot guess, it won't be able to render the asset. + # + # (Docstring intentionally commented out to hide this field from the docs) + + opacity: components.OpacityBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.OpacityBatch._optional, # type: ignore[misc] + ) + # Opacity of the image, useful for layering several images. + # + # Defaults to 1.0 (fully opaque). + # + # (Docstring intentionally commented out to hide this field from the docs) + + draw_order: components.DrawOrderBatch | None = field( + metadata={"component": "optional"}, + default=None, + converter=components.DrawOrderBatch._optional, # type: ignore[misc] + ) + # An optional floating point value that specifies the 2D drawing order. + # + # Objects with higher values are drawn on top of those with lower values. + # + # (Docstring intentionally commented out to hide this field from the docs) + + __str__ = Archetype.__str__ + __repr__ = Archetype.__repr__ # type: ignore[assignment] diff --git a/rerun_py/rerun_sdk/rerun/archetypes/image_encoded_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/image_encoded_ext.py new file mode 100644 index 0000000000000..31641b209ce99 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/archetypes/image_encoded_ext.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import pathlib +from typing import TYPE_CHECKING, Any + +from .. import datatypes +from ..error_utils import catch_and_log_exceptions + +if TYPE_CHECKING: + from ..components import MediaType + + +def guess_media_type(path: str | pathlib.Path) -> MediaType | None: + from ..components import MediaType + + ext = pathlib.Path(path).suffix.lower() + if ext == ".jpg" or ext == ".jpeg": + return MediaType.JPEG + elif ext == ".png": + return MediaType.PNG + else: + return None + + +class ImageEncodedExt: + """Extension for [ImageEncoded][rerun.archetypes.ImageEncoded].""" + + def __init__( + self: Any, + *, + path: str | pathlib.Path | None = None, + contents: datatypes.BlobLike | None = None, + media_type: datatypes.Utf8Like | None = None, + ): + """ + Create a new instance of the ImageEncoded archetype. + + Parameters + ---------- + path: + A path to an file stored on the local filesystem. Mutually + exclusive with `contents`. + + contents: + The contents of the file. Can be a BufferedReader, BytesIO, or + bytes. Mutually exclusive with `path`. + + media_type: + The Media Type of the asset. + + For instance: + * `image/jpeg` + * `image/png` + + If omitted, it will be guessed from the `path` (if any), + or the viewer will try to guess from the contents (magic header). + If the media type cannot be guessed, the viewer won't be able to render the asset. + + """ + + with catch_and_log_exceptions(context=self.__class__.__name__): + if (path is None) == (contents is None): + raise ValueError("Must provide exactly one of 'path' or 'contents'") + + if path is None: + blob = contents + else: + blob = pathlib.Path(path).read_bytes() + if media_type is None: + media_type = guess_media_type(str(path)) + + self.__attrs_init__(blob=blob, media_type=media_type) + return + + self.__attrs_clear__() diff --git a/rerun_py/rerun_sdk/rerun/components/media_type_ext.py b/rerun_py/rerun_sdk/rerun/components/media_type_ext.py index 6b3a74d095dc9..374664355211b 100644 --- a/rerun_py/rerun_sdk/rerun/components/media_type_ext.py +++ b/rerun_py/rerun_sdk/rerun/components/media_type_ext.py @@ -20,6 +20,24 @@ class MediaTypeExt: """ + # -------------------------- + # Images: + + JPEG: MediaType = None # type: ignore[assignment] + """ + [JPEG image](https://en.wikipedia.org/wiki/JPEG): `image/jpeg`. + """ + + PNG: MediaType = None # type: ignore[assignment] + """ + [PNG image](https://en.wikipedia.org/wiki/PNG): `image/png`. + + + """ + + # -------------------------- + # Meshes: + GLB: MediaType = None # type: ignore[assignment] """ Binary [`glTF`](https://en.wikipedia.org/wiki/GlTF): `model/gltf-binary`. @@ -53,6 +71,10 @@ class MediaTypeExt: def deferred_patch_class(cls: Any) -> None: cls.TEXT = cls("text/plain") cls.MARKDOWN = cls("text/markdown") + + cls.JPEG = cls("image/jpeg") + cls.PNG = cls("image/png") + cls.GLB = cls("model/gltf-binary") cls.GLTF = cls("model/gltf+json") cls.OBJ = cls("model/obj")