From d32aa44cee13e3aebae5bea1789f910da18d4d47 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Fri, 27 Sep 2024 15:23:57 +0200 Subject: [PATCH] Show spinner icon while waiting for video decoder (#7541) ### What * Closes #7478 ![video-spinner-3](https://github.com/user-attachments/assets/f0448e4b-63b1-4582-8085-c685fc2fa686) This uses the default `egui::Spinner` loading animation. It plays it directly, even if there is only a single frame of delay in decoding. I thought this could be annoying, but I think it feels great, and it is a lot simpler than keeping track of for how long we've been waiting. ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7541?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7541?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7541) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- crates/viewer/re_data_ui/src/blob.rs | 53 +++++---- crates/viewer/re_data_ui/src/image.rs | 7 +- .../re_space_view_spatial/src/picking_ui.rs | 6 +- .../src/scene_bounding_boxes.rs | 4 +- crates/viewer/re_space_view_spatial/src/ui.rs | 22 +++- .../viewer/re_space_view_spatial/src/ui_2d.rs | 5 +- .../viewer/re_space_view_spatial/src/ui_3d.rs | 9 +- .../src/visualizers/mod.rs | 7 +- .../src/visualizers/utilities/mod.rs | 2 +- .../utilities/spatial_view_visualizer.rs | 20 ++-- .../src/visualizers/videos.rs | 102 ++++++++++-------- .../src/space_view/visualizer_system.rs | 11 ++ 12 files changed, 159 insertions(+), 89 deletions(-) diff --git a/crates/viewer/re_data_ui/src/blob.rs b/crates/viewer/re_data_ui/src/blob.rs index 41e93fe068a3..a6a4caf6d3e1 100644 --- a/crates/viewer/re_data_ui/src/blob.rs +++ b/crates/viewer/re_data_ui/src/blob.rs @@ -237,31 +237,46 @@ fn show_video_blob_info( }; let decode_stream_id = re_renderer::video::VideoDecodingStreamId( - egui::Id::new("video_miniplayer").value(), + ui.id().with("video_player").value(), ); - if let Some(texture) = - match video.frame_at(render_ctx, decode_stream_id, timestamp_in_seconds) { - Ok(VideoFrameTexture::Ready(texture)) => Some(texture), + match video.frame_at(render_ctx, decode_stream_id, timestamp_in_seconds) { + Ok(frame) => { + let is_pending; + let texture = match frame { + VideoFrameTexture::Ready(texture) => { + is_pending = false; + texture + } - Ok(VideoFrameTexture::Pending(texture)) => { - ui.ctx().request_repaint(); - Some(texture) - } + VideoFrameTexture::Pending(placeholder) => { + is_pending = true; + ui.ctx().request_repaint(); + placeholder + } + }; + + let response = crate::image::texture_preview_ui( + render_ctx, + ui, + ui_layout, + "video_preview", + re_renderer::renderer::ColormappedTexture::from_unorm_rgba(texture), + ); - Err(err) => { - ui.error_label_long(&err.to_string()); - None + if is_pending { + // Shrink slightly: + let smaller_rect = egui::Rect::from_center_size( + response.rect.center(), + 0.75 * response.rect.size(), + ); + egui::Spinner::new().paint_at(ui, smaller_rect); } } - { - crate::image::texture_preview_ui( - render_ctx, - ui, - ui_layout, - "video_preview", - re_renderer::renderer::ColormappedTexture::from_unorm_rgba(texture), - ); + + Err(err) => { + ui.error_label_long(&err.to_string()); + } } } }); diff --git a/crates/viewer/re_data_ui/src/image.rs b/crates/viewer/re_data_ui/src/image.rs index 2185291cf2c9..56dd270e1455 100644 --- a/crates/viewer/re_data_ui/src/image.rs +++ b/crates/viewer/re_data_ui/src/image.rs @@ -54,7 +54,7 @@ pub fn texture_preview_ui( ui_layout: UiLayout, debug_name: &str, texture: ColormappedTexture, -) { +) -> egui::Response { if ui_layout.is_single_line() { let preview_size = Vec2::splat(ui.available_height()); ui.allocate_ui_with_layout( @@ -73,7 +73,8 @@ pub fn texture_preview_ui( Err((response, err)) => response.on_hover_text(err.to_string()), } }, - ); + ) + .inner } else { let size_range = if ui_layout == UiLayout::Tooltip { egui::Rangef::new(64.0, 128.0) @@ -90,7 +91,7 @@ pub fn texture_preview_ui( re_log::warn_once!("Failed to show texture {debug_name}: {err}"); response }, - ); + ) } } diff --git a/crates/viewer/re_space_view_spatial/src/picking_ui.rs b/crates/viewer/re_space_view_spatial/src/picking_ui.rs index 4d8695752b7d..8aa10dde8581 100644 --- a/crates/viewer/re_space_view_spatial/src/picking_ui.rs +++ b/crates/viewer/re_space_view_spatial/src/picking_ui.rs @@ -17,7 +17,7 @@ use crate::{ picking_ui_pixel::{textured_rect_hover_ui, PickedPixelInfo}, ui::SpatialSpaceViewState, view_kind::SpatialSpaceViewKind, - visualizers::{iter_spatial_visualizer_data, CamerasVisualizer, DepthImageVisualizer}, + visualizers::{CamerasVisualizer, DepthImageVisualizer, SpatialViewVisualizerData}, PickableRectSourceData, PickableTexturedRect, }; @@ -217,7 +217,9 @@ pub fn picking( fn iter_pickable_rects( visualizers: &VisualizerCollection, ) -> impl Iterator { - iter_spatial_visualizer_data(visualizers).flat_map(|data| data.pickable_rects.iter()) + visualizers + .iter_visualizer_data::() + .flat_map(|data| data.pickable_rects.iter()) } /// If available, finds pixel info for a picking hit. diff --git a/crates/viewer/re_space_view_spatial/src/scene_bounding_boxes.rs b/crates/viewer/re_space_view_spatial/src/scene_bounding_boxes.rs index 77af7138894f..986106e14f14 100644 --- a/crates/viewer/re_space_view_spatial/src/scene_bounding_boxes.rs +++ b/crates/viewer/re_space_view_spatial/src/scene_bounding_boxes.rs @@ -3,7 +3,7 @@ use nohash_hasher::IntMap; use re_log_types::EntityPathHash; use re_viewer_context::VisualizerCollection; -use crate::{view_kind::SpatialSpaceViewKind, visualizers::iter_spatial_visualizer_data}; +use crate::{view_kind::SpatialSpaceViewKind, visualizers::SpatialViewVisualizerData}; #[derive(Clone)] pub struct SceneBoundingBoxes { @@ -42,7 +42,7 @@ impl SceneBoundingBoxes { self.current = re_math::BoundingBox::NOTHING; self.per_entity.clear(); - for data in iter_spatial_visualizer_data(visualizers) { + for data in visualizers.iter_visualizer_data::() { // If we're in a 3D space, but the visualizer is distintivly 2D, don't count it towards the bounding box. // These visualizers show up when we're on a pinhole camera plane which itself is heuristically fed by the // bounding box, creating a feedback loop if we were to add it here. diff --git a/crates/viewer/re_space_view_spatial/src/ui.rs b/crates/viewer/re_space_view_spatial/src/ui.rs index 6ce07265f18c..1c34d4b76be9 100644 --- a/crates/viewer/re_space_view_spatial/src/ui.rs +++ b/crates/viewer/re_space_view_spatial/src/ui.rs @@ -19,7 +19,7 @@ use crate::{ picking::{PickableUiRect, PickingResult}, scene_bounding_boxes::SceneBoundingBoxes, view_kind::SpatialSpaceViewKind, - visualizers::{iter_spatial_visualizer_data, UiLabel, UiLabelTarget}, + visualizers::{SpatialViewVisualizerData, UiLabel, UiLabelTarget}, }; use super::{eye::Eye, ui_3d::View3DState}; @@ -84,7 +84,8 @@ impl SpatialSpaceViewState { .update(ui, &system_output.view_systems, space_kind); let view_systems = &system_output.view_systems; - self.num_non_segmentation_images_last_frame = iter_spatial_visualizer_data(view_systems) + self.num_non_segmentation_images_last_frame = view_systems + .iter_visualizer_data::() .flat_map(|data| { data.pickable_rects.iter().map(|pickable_rect| { if let PickableRectSourceData::Image { image, .. } = &pickable_rect.source_data @@ -268,6 +269,23 @@ pub fn create_labels( (label_shapes, ui_rects) } +pub fn paint_loading_spinners( + ui: &egui::Ui, + ui_from_scene: egui::emath::RectTransform, + visualizers: &re_viewer_context::VisualizerCollection, +) { + for data in visualizers.iter_visualizer_data::() { + for &rect_in_scene in &data.loading_rects { + let rect_in_ui = ui_from_scene.transform_rect(rect_in_scene); + + // Shrink slightly: + let rect = egui::Rect::from_center_size(rect_in_ui.center(), 0.75 * rect_in_ui.size()); + + egui::Spinner::new().paint_at(ui, rect); + } + } +} + pub fn outline_config(gui_ctx: &egui::Context) -> OutlineConfig { // Use the exact same colors we have in the ui! let hover_outline = gui_ctx.hover_stroke(); diff --git a/crates/viewer/re_space_view_spatial/src/ui_2d.rs b/crates/viewer/re_space_view_spatial/src/ui_2d.rs index 16fcff735d3c..c0999aa2e9bb 100644 --- a/crates/viewer/re_space_view_spatial/src/ui_2d.rs +++ b/crates/viewer/re_space_view_spatial/src/ui_2d.rs @@ -286,7 +286,10 @@ impl SpatialSpaceView2D { )); } - // Add egui driven labels on top of re_renderer content. + // Add egui-rendered spinners/loaders on top of re_renderer content: + crate::ui::paint_loading_spinners(ui, ui_from_scene, &system_output.view_systems); + + // Add egui-rendered labels on top of everything else: painter.extend(label_shapes); Ok(()) diff --git a/crates/viewer/re_space_view_spatial/src/ui_3d.rs b/crates/viewer/re_space_view_spatial/src/ui_3d.rs index cd2a2e7ede51..19ca39c75310 100644 --- a/crates/viewer/re_space_view_spatial/src/ui_3d.rs +++ b/crates/viewer/re_space_view_spatial/src/ui_3d.rs @@ -687,7 +687,14 @@ impl SpatialSpaceView3D { clear_color, )); - // Add egui driven labels on top of re_renderer content. + // Add egui-rendered spinners/loaders on top of re_renderer content: + crate::ui::paint_loading_spinners( + ui, + RectTransform::from_to(ui_rect, ui_rect), + &system_output.view_systems, + ); + + // Add egui-rendered labels on top of everything else: let painter = ui.painter().with_clip_rect(ui.max_rect()); painter.extend(label_shapes); diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs index 59e4d25a7e40..93db22658852 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/mod.rs @@ -24,8 +24,8 @@ pub use cameras::CamerasVisualizer; pub use depth_images::DepthImageVisualizer; pub use transform3d_arrows::{add_axis_arrows, AxisLengthDetector, Transform3DArrowsVisualizer}; pub use utilities::{ - entity_iterator, iter_spatial_visualizer_data, process_labels_3d, textured_rect_from_image, - SpatialViewVisualizerData, UiLabel, UiLabelTarget, + entity_iterator, process_labels_3d, textured_rect_from_image, SpatialViewVisualizerData, + UiLabel, UiLabelTarget, }; // --- @@ -129,7 +129,8 @@ pub fn visualizers_processing_draw_order() -> impl Iterator Vec { - iter_spatial_visualizer_data(visualizers) + visualizers + .iter_visualizer_data::() .flat_map(|data| data.ui_labels.iter().cloned()) .collect() } diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/utilities/mod.rs b/crates/viewer/re_space_view_spatial/src/visualizers/utilities/mod.rs index fcb409fa9019..127a706ccdad 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/utilities/mod.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/utilities/mod.rs @@ -9,5 +9,5 @@ pub use labels::{ UiLabel, UiLabelTarget, }; pub use proc_mesh_vis::{ProcMeshBatch, ProcMeshDrawableBuilder}; -pub use spatial_view_visualizer::{iter_spatial_visualizer_data, SpatialViewVisualizerData}; +pub use spatial_view_visualizer::SpatialViewVisualizerData; pub use textured_rect::textured_rect_from_image; diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/utilities/spatial_view_visualizer.rs b/crates/viewer/re_space_view_spatial/src/visualizers/utilities/spatial_view_visualizer.rs index 040698533700..96c97710b7a4 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/utilities/spatial_view_visualizer.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/utilities/spatial_view_visualizer.rs @@ -7,6 +7,9 @@ use crate::{view_kind::SpatialSpaceViewKind, PickableTexturedRect}; /// /// Each spatial scene element is expected to fill an instance of this struct with its data. pub struct SpatialViewVisualizerData { + /// Loading icons/spinners shown using egui, in world/scene coordinates. + pub loading_rects: Vec, + /// Labels that should be shown using egui. pub ui_labels: Vec, @@ -23,9 +26,10 @@ pub struct SpatialViewVisualizerData { impl SpatialViewVisualizerData { pub fn new(preferred_view_kind: Option) -> Self { Self { - ui_labels: Vec::new(), - bounding_boxes: Vec::new(), - pickable_rects: Vec::new(), + loading_rects: Default::default(), + ui_labels: Default::default(), + bounding_boxes: Default::default(), + pickable_rects: Default::default(), preferred_view_kind, } } @@ -58,13 +62,3 @@ impl SpatialViewVisualizerData { self } } - -pub fn iter_spatial_visualizer_data( - visualizers: &re_viewer_context::VisualizerCollection, -) -> impl Iterator { - visualizers.iter().filter_map(|visualizer| { - visualizer - .data() - .and_then(|data| data.downcast_ref::()) - }) -} diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index 867473d774d6..8fdf14b1711f 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -191,49 +191,67 @@ impl VideoFrameReferenceVisualizer { Some(Ok(video)) => { video_resolution = glam::vec2(video.width() as _, video.height() as _); - if let Some(texture) = - match video.frame_at(render_ctx, decode_stream_id, video_timestamp.as_seconds()) - { - Ok(VideoFrameTexture::Ready(texture)) => Some(texture), - Ok(VideoFrameTexture::Pending(texture)) => { - ctx.viewer_ctx.egui_ctx.request_repaint(); - Some(texture) - } - Err(err) => { - self.show_video_error( - ctx, - spatial_ctx, - world_from_entity, - err.to_string(), - video_resolution, - entity_path, - ); - None - } - } - { - let textured_rect = TexturedRect { - top_left_corner_position: world_from_entity - .transform_point3(glam::Vec3::ZERO), + + match video.frame_at(render_ctx, decode_stream_id, video_timestamp.as_seconds()) { + Ok(frame) => { // Make sure to use the video instead of texture size here, - // since it may be a placeholder which doesn't have the full size yet. - extent_u: world_from_entity - .transform_vector3(glam::Vec3::X * video_resolution.x), - extent_v: world_from_entity - .transform_vector3(glam::Vec3::Y * video_resolution.y), - colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), - options: RectangleOptions { - texture_filter_magnification: TextureFilterMag::Nearest, - texture_filter_minification: TextureFilterMin::Linear, - outline_mask: spatial_ctx.highlight.overall, - ..Default::default() - }, - }; - self.data.pickable_rects.push(PickableTexturedRect { - ent_path: entity_path.clone(), - textured_rect, - source_data: PickableRectSourceData::Video, - }); + // since the texture may be a placeholder which doesn't have the full size yet. + let top_left_corner_position = + world_from_entity.transform_point3(glam::Vec3::ZERO); + let extent_u = + world_from_entity.transform_vector3(glam::Vec3::X * video_resolution.x); + let extent_v = + world_from_entity.transform_vector3(glam::Vec3::Y * video_resolution.y); + + let texture = match frame { + VideoFrameTexture::Ready(texture) => texture, + VideoFrameTexture::Pending(placeholder) => { + // Show loading rectangle: + let min = top_left_corner_position; + let max = top_left_corner_position + extent_u + extent_v; + let center = 0.5 * (min + max); + let diameter = (max - min).truncate().abs().min_element(); + self.data.loading_rects.push(egui::Rect::from_center_size( + egui::pos2(center.x, center.y), + egui::Vec2::splat(diameter), + )); + + // Keep polling for the decoded result: + ctx.viewer_ctx.egui_ctx.request_repaint(); + + placeholder + } + }; + + let textured_rect = TexturedRect { + top_left_corner_position, + extent_u, + extent_v, + colormapped_texture: ColormappedTexture::from_unorm_rgba(texture), + options: RectangleOptions { + texture_filter_magnification: TextureFilterMag::Nearest, + texture_filter_minification: TextureFilterMin::Linear, + outline_mask: spatial_ctx.highlight.overall, + ..Default::default() + }, + }; + self.data.pickable_rects.push(PickableTexturedRect { + ent_path: entity_path.clone(), + textured_rect, + source_data: PickableRectSourceData::Video, + }); + } + + Err(err) => { + self.show_video_error( + ctx, + spatial_ctx, + world_from_entity, + err.to_string(), + video_resolution, + entity_path, + ); + } } } Some(Err(err)) => { diff --git a/crates/viewer/re_viewer_context/src/space_view/visualizer_system.rs b/crates/viewer/re_viewer_context/src/space_view/visualizer_system.rs index 29000a17d4fd..10eec441f940 100644 --- a/crates/viewer/re_viewer_context/src/space_view/visualizer_system.rs +++ b/crates/viewer/re_viewer_context/src/space_view/visualizer_system.rs @@ -174,4 +174,15 @@ impl VisualizerCollection { ) -> impl Iterator { self.systems.iter().map(|s| (*s.0, s.1.as_ref())) } + + /// Iterate over all visualizer data that can be downcast to the given type. + pub fn iter_visualizer_data( + &self, + ) -> impl Iterator { + self.iter().filter_map(|visualizer| { + visualizer + .data() + .and_then(|data| data.downcast_ref::()) + }) + } }