Skip to content

Commit

Permalink
Show spinner icon while waiting for video decoder (#7541)
Browse files Browse the repository at this point in the history
### What
* Closes #7478


![video-spinner-3](https://github.com/user-attachments/assets/f0448e4b-63b1-4582-8085-c685fc2fa686)

This uses the default `egui::Spinner` loading animation.

It plays it directly, even if there is only a single frame of delay in
decoding. I thought this could be annoying, but I think it feels great,
and it is a lot simpler than keeping track of for how long we've been
waiting.

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7541?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7541?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7541)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.
  • Loading branch information
emilk committed Sep 27, 2024
1 parent 0199946 commit d32aa44
Show file tree
Hide file tree
Showing 12 changed files with 159 additions and 89 deletions.
53 changes: 34 additions & 19 deletions crates/viewer/re_data_ui/src/blob.rs
Original file line number Diff line number Diff line change
Expand Up @@ -237,31 +237,46 @@ fn show_video_blob_info(
};

let decode_stream_id = re_renderer::video::VideoDecodingStreamId(
egui::Id::new("video_miniplayer").value(),
ui.id().with("video_player").value(),
);

if let Some(texture) =
match video.frame_at(render_ctx, decode_stream_id, timestamp_in_seconds) {
Ok(VideoFrameTexture::Ready(texture)) => Some(texture),
match video.frame_at(render_ctx, decode_stream_id, timestamp_in_seconds) {
Ok(frame) => {
let is_pending;
let texture = match frame {
VideoFrameTexture::Ready(texture) => {
is_pending = false;
texture
}

Ok(VideoFrameTexture::Pending(texture)) => {
ui.ctx().request_repaint();
Some(texture)
}
VideoFrameTexture::Pending(placeholder) => {
is_pending = true;
ui.ctx().request_repaint();
placeholder
}
};

let response = crate::image::texture_preview_ui(
render_ctx,
ui,
ui_layout,
"video_preview",
re_renderer::renderer::ColormappedTexture::from_unorm_rgba(texture),
);

Err(err) => {
ui.error_label_long(&err.to_string());
None
if is_pending {
// Shrink slightly:
let smaller_rect = egui::Rect::from_center_size(
response.rect.center(),
0.75 * response.rect.size(),
);
egui::Spinner::new().paint_at(ui, smaller_rect);
}
}
{
crate::image::texture_preview_ui(
render_ctx,
ui,
ui_layout,
"video_preview",
re_renderer::renderer::ColormappedTexture::from_unorm_rgba(texture),
);

Err(err) => {
ui.error_label_long(&err.to_string());
}
}
}
});
Expand Down
7 changes: 4 additions & 3 deletions crates/viewer/re_data_ui/src/image.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ pub fn texture_preview_ui(
ui_layout: UiLayout,
debug_name: &str,
texture: ColormappedTexture,
) {
) -> egui::Response {
if ui_layout.is_single_line() {
let preview_size = Vec2::splat(ui.available_height());
ui.allocate_ui_with_layout(
Expand All @@ -73,7 +73,8 @@ pub fn texture_preview_ui(
Err((response, err)) => response.on_hover_text(err.to_string()),
}
},
);
)
.inner
} else {
let size_range = if ui_layout == UiLayout::Tooltip {
egui::Rangef::new(64.0, 128.0)
Expand All @@ -90,7 +91,7 @@ pub fn texture_preview_ui(
re_log::warn_once!("Failed to show texture {debug_name}: {err}");
response
},
);
)
}
}

Expand Down
6 changes: 4 additions & 2 deletions crates/viewer/re_space_view_spatial/src/picking_ui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::{
picking_ui_pixel::{textured_rect_hover_ui, PickedPixelInfo},
ui::SpatialSpaceViewState,
view_kind::SpatialSpaceViewKind,
visualizers::{iter_spatial_visualizer_data, CamerasVisualizer, DepthImageVisualizer},
visualizers::{CamerasVisualizer, DepthImageVisualizer, SpatialViewVisualizerData},
PickableRectSourceData, PickableTexturedRect,
};

Expand Down Expand Up @@ -217,7 +217,9 @@ pub fn picking(
fn iter_pickable_rects(
visualizers: &VisualizerCollection,
) -> impl Iterator<Item = &PickableTexturedRect> {
iter_spatial_visualizer_data(visualizers).flat_map(|data| data.pickable_rects.iter())
visualizers
.iter_visualizer_data::<SpatialViewVisualizerData>()
.flat_map(|data| data.pickable_rects.iter())
}

/// If available, finds pixel info for a picking hit.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use nohash_hasher::IntMap;
use re_log_types::EntityPathHash;
use re_viewer_context::VisualizerCollection;

use crate::{view_kind::SpatialSpaceViewKind, visualizers::iter_spatial_visualizer_data};
use crate::{view_kind::SpatialSpaceViewKind, visualizers::SpatialViewVisualizerData};

#[derive(Clone)]
pub struct SceneBoundingBoxes {
Expand Down Expand Up @@ -42,7 +42,7 @@ impl SceneBoundingBoxes {
self.current = re_math::BoundingBox::NOTHING;
self.per_entity.clear();

for data in iter_spatial_visualizer_data(visualizers) {
for data in visualizers.iter_visualizer_data::<SpatialViewVisualizerData>() {
// If we're in a 3D space, but the visualizer is distintivly 2D, don't count it towards the bounding box.
// These visualizers show up when we're on a pinhole camera plane which itself is heuristically fed by the
// bounding box, creating a feedback loop if we were to add it here.
Expand Down
22 changes: 20 additions & 2 deletions crates/viewer/re_space_view_spatial/src/ui.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use crate::{
picking::{PickableUiRect, PickingResult},
scene_bounding_boxes::SceneBoundingBoxes,
view_kind::SpatialSpaceViewKind,
visualizers::{iter_spatial_visualizer_data, UiLabel, UiLabelTarget},
visualizers::{SpatialViewVisualizerData, UiLabel, UiLabelTarget},
};

use super::{eye::Eye, ui_3d::View3DState};
Expand Down Expand Up @@ -84,7 +84,8 @@ impl SpatialSpaceViewState {
.update(ui, &system_output.view_systems, space_kind);

let view_systems = &system_output.view_systems;
self.num_non_segmentation_images_last_frame = iter_spatial_visualizer_data(view_systems)
self.num_non_segmentation_images_last_frame = view_systems
.iter_visualizer_data::<SpatialViewVisualizerData>()
.flat_map(|data| {
data.pickable_rects.iter().map(|pickable_rect| {
if let PickableRectSourceData::Image { image, .. } = &pickable_rect.source_data
Expand Down Expand Up @@ -268,6 +269,23 @@ pub fn create_labels(
(label_shapes, ui_rects)
}

pub fn paint_loading_spinners(
ui: &egui::Ui,
ui_from_scene: egui::emath::RectTransform,
visualizers: &re_viewer_context::VisualizerCollection,
) {
for data in visualizers.iter_visualizer_data::<SpatialViewVisualizerData>() {
for &rect_in_scene in &data.loading_rects {
let rect_in_ui = ui_from_scene.transform_rect(rect_in_scene);

// Shrink slightly:
let rect = egui::Rect::from_center_size(rect_in_ui.center(), 0.75 * rect_in_ui.size());

egui::Spinner::new().paint_at(ui, rect);
}
}
}

pub fn outline_config(gui_ctx: &egui::Context) -> OutlineConfig {
// Use the exact same colors we have in the ui!
let hover_outline = gui_ctx.hover_stroke();
Expand Down
5 changes: 4 additions & 1 deletion crates/viewer/re_space_view_spatial/src/ui_2d.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,10 @@ impl SpatialSpaceView2D {
));
}

// Add egui driven labels on top of re_renderer content.
// Add egui-rendered spinners/loaders on top of re_renderer content:
crate::ui::paint_loading_spinners(ui, ui_from_scene, &system_output.view_systems);

// Add egui-rendered labels on top of everything else:
painter.extend(label_shapes);

Ok(())
Expand Down
9 changes: 8 additions & 1 deletion crates/viewer/re_space_view_spatial/src/ui_3d.rs
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,14 @@ impl SpatialSpaceView3D {
clear_color,
));

// Add egui driven labels on top of re_renderer content.
// Add egui-rendered spinners/loaders on top of re_renderer content:
crate::ui::paint_loading_spinners(
ui,
RectTransform::from_to(ui_rect, ui_rect),
&system_output.view_systems,
);

// Add egui-rendered labels on top of everything else:
let painter = ui.painter().with_clip_rect(ui.max_rect());
painter.extend(label_shapes);

Expand Down
7 changes: 4 additions & 3 deletions crates/viewer/re_space_view_spatial/src/visualizers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ pub use cameras::CamerasVisualizer;
pub use depth_images::DepthImageVisualizer;
pub use transform3d_arrows::{add_axis_arrows, AxisLengthDetector, Transform3DArrowsVisualizer};
pub use utilities::{
entity_iterator, iter_spatial_visualizer_data, process_labels_3d, textured_rect_from_image,
SpatialViewVisualizerData, UiLabel, UiLabelTarget,
entity_iterator, process_labels_3d, textured_rect_from_image, SpatialViewVisualizerData,
UiLabel, UiLabelTarget,
};

// ---
Expand Down Expand Up @@ -129,7 +129,8 @@ pub fn visualizers_processing_draw_order() -> impl Iterator<Item = ViewSystemIde
}

pub fn collect_ui_labels(visualizers: &VisualizerCollection) -> Vec<UiLabel> {
iter_spatial_visualizer_data(visualizers)
visualizers
.iter_visualizer_data::<SpatialViewVisualizerData>()
.flat_map(|data| data.ui_labels.iter().cloned())
.collect()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ pub use labels::{
UiLabel, UiLabelTarget,
};
pub use proc_mesh_vis::{ProcMeshBatch, ProcMeshDrawableBuilder};
pub use spatial_view_visualizer::{iter_spatial_visualizer_data, SpatialViewVisualizerData};
pub use spatial_view_visualizer::SpatialViewVisualizerData;
pub use textured_rect::textured_rect_from_image;
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ use crate::{view_kind::SpatialSpaceViewKind, PickableTexturedRect};
///
/// Each spatial scene element is expected to fill an instance of this struct with its data.
pub struct SpatialViewVisualizerData {
/// Loading icons/spinners shown using egui, in world/scene coordinates.
pub loading_rects: Vec<egui::Rect>,

/// Labels that should be shown using egui.
pub ui_labels: Vec<UiLabel>,

Expand All @@ -23,9 +26,10 @@ pub struct SpatialViewVisualizerData {
impl SpatialViewVisualizerData {
pub fn new(preferred_view_kind: Option<SpatialSpaceViewKind>) -> Self {
Self {
ui_labels: Vec::new(),
bounding_boxes: Vec::new(),
pickable_rects: Vec::new(),
loading_rects: Default::default(),
ui_labels: Default::default(),
bounding_boxes: Default::default(),
pickable_rects: Default::default(),
preferred_view_kind,
}
}
Expand Down Expand Up @@ -58,13 +62,3 @@ impl SpatialViewVisualizerData {
self
}
}

pub fn iter_spatial_visualizer_data(
visualizers: &re_viewer_context::VisualizerCollection,
) -> impl Iterator<Item = &SpatialViewVisualizerData> {
visualizers.iter().filter_map(|visualizer| {
visualizer
.data()
.and_then(|data| data.downcast_ref::<SpatialViewVisualizerData>())
})
}
102 changes: 60 additions & 42 deletions crates/viewer/re_space_view_spatial/src/visualizers/videos.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,49 +191,67 @@ impl VideoFrameReferenceVisualizer {

Some(Ok(video)) => {
video_resolution = glam::vec2(video.width() as _, video.height() as _);
if let Some(texture) =
match video.frame_at(render_ctx, decode_stream_id, video_timestamp.as_seconds())
{
Ok(VideoFrameTexture::Ready(texture)) => Some(texture),
Ok(VideoFrameTexture::Pending(texture)) => {
ctx.viewer_ctx.egui_ctx.request_repaint();
Some(texture)
}
Err(err) => {
self.show_video_error(
ctx,
spatial_ctx,
world_from_entity,
err.to_string(),
video_resolution,
entity_path,
);
None
}
}
{
let textured_rect = TexturedRect {
top_left_corner_position: world_from_entity
.transform_point3(glam::Vec3::ZERO),

match video.frame_at(render_ctx, decode_stream_id, video_timestamp.as_seconds()) {
Ok(frame) => {
// Make sure to use the video instead of texture size here,
// since it may be a placeholder which doesn't have the full size yet.
extent_u: world_from_entity
.transform_vector3(glam::Vec3::X * video_resolution.x),
extent_v: world_from_entity
.transform_vector3(glam::Vec3::Y * video_resolution.y),
colormapped_texture: ColormappedTexture::from_unorm_rgba(texture),
options: RectangleOptions {
texture_filter_magnification: TextureFilterMag::Nearest,
texture_filter_minification: TextureFilterMin::Linear,
outline_mask: spatial_ctx.highlight.overall,
..Default::default()
},
};
self.data.pickable_rects.push(PickableTexturedRect {
ent_path: entity_path.clone(),
textured_rect,
source_data: PickableRectSourceData::Video,
});
// since the texture may be a placeholder which doesn't have the full size yet.
let top_left_corner_position =
world_from_entity.transform_point3(glam::Vec3::ZERO);
let extent_u =
world_from_entity.transform_vector3(glam::Vec3::X * video_resolution.x);
let extent_v =
world_from_entity.transform_vector3(glam::Vec3::Y * video_resolution.y);

let texture = match frame {
VideoFrameTexture::Ready(texture) => texture,
VideoFrameTexture::Pending(placeholder) => {
// Show loading rectangle:
let min = top_left_corner_position;
let max = top_left_corner_position + extent_u + extent_v;
let center = 0.5 * (min + max);
let diameter = (max - min).truncate().abs().min_element();
self.data.loading_rects.push(egui::Rect::from_center_size(
egui::pos2(center.x, center.y),
egui::Vec2::splat(diameter),
));

// Keep polling for the decoded result:
ctx.viewer_ctx.egui_ctx.request_repaint();

placeholder
}
};

let textured_rect = TexturedRect {
top_left_corner_position,
extent_u,
extent_v,
colormapped_texture: ColormappedTexture::from_unorm_rgba(texture),
options: RectangleOptions {
texture_filter_magnification: TextureFilterMag::Nearest,
texture_filter_minification: TextureFilterMin::Linear,
outline_mask: spatial_ctx.highlight.overall,
..Default::default()
},
};
self.data.pickable_rects.push(PickableTexturedRect {
ent_path: entity_path.clone(),
textured_rect,
source_data: PickableRectSourceData::Video,
});
}

Err(err) => {
self.show_video_error(
ctx,
spatial_ctx,
world_from_entity,
err.to_string(),
video_resolution,
entity_path,
);
}
}
}
Some(Err(err)) => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,4 +174,15 @@ impl VisualizerCollection {
) -> impl Iterator<Item = (ViewSystemIdentifier, &dyn VisualizerSystem)> {
self.systems.iter().map(|s| (*s.0, s.1.as_ref()))
}

/// Iterate over all visualizer data that can be downcast to the given type.
pub fn iter_visualizer_data<SpecificData: 'static>(
&self,
) -> impl Iterator<Item = &'_ SpecificData> {
self.iter().filter_map(|visualizer| {
visualizer
.data()
.and_then(|data| data.downcast_ref::<SpecificData>())
})
}
}

0 comments on commit d32aa44

Please sign in to comment.