Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show all samples/frames in a video in a nice table #8102

Merged
merged 3 commits into from
Nov 12, 2024
+179 −96
Merged

Show all samples/frames in a video in a nice table #8102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f0d6233
Break up long function to prevent rightward drift
emilk Nov 12, 2024
b5643c4
Better naming of function
emilk Nov 12, 2024
65aece6
Show table of all samples in the video
emilk Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/store/re_video/src/demux/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ impl VideoData {
// Segments are guaranteed to be sorted among each other, but within a segment,
// presentation timestamps may not be sorted since this is sorted by decode timestamps.
self.gops.iter().flat_map(|seg| {
self.samples[seg.decode_time_range()]
self.samples[seg.sample_range_usize()]
.iter()
.map(|sample| sample.presentation_timestamp)
.sorted()
Expand Down Expand Up @@ -432,7 +432,7 @@ pub struct GroupOfPictures {

impl GroupOfPictures {
/// The GOP's `sample_range` mapped to `usize` for slicing.
pub fn decode_time_range(&self) -> Range<usize> {
pub fn sample_range_usize(&self) -> Range<usize> {
Range {
start: self.sample_range.start as usize,
end: self.sample_range.end as usize,
Expand Down
4 changes: 2 additions & 2 deletions crates/viewer/re_data_ui/src/blob.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use re_viewer_context::UiLayout;

use crate::{
image::image_preview_ui,
video::{show_decoded_frame_info, show_video_blob_info},
video::{show_decoded_frame_info, video_result_ui},
EntityDataUi,
};

Expand Down Expand Up @@ -131,7 +131,7 @@ pub fn blob_preview_and_save_ui(
ctx.app_options.video_decoder_hw_acceleration,
)
});
show_video_blob_info(ui, ui_layout, &video_result);
video_result_ui(ui, ui_layout, &video_result);
video_result_for_frame_preview = Some(video_result);
}
}
Expand Down
265 changes: 174 additions & 91 deletions crates/viewer/re_data_ui/src/video.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,109 +1,28 @@
use egui_extras::Column;
use re_renderer::{
external::re_video::VideoLoadError, resource_managers::SourceImageDataFormat,
video::VideoFrameTexture,
};
use re_types::components::VideoTimestamp;
use re_ui::{list_item::PropertyContent, UiExt};
use re_video::{decode::FrameInfo, demux::SamplesStatistics};
use re_ui::{list_item::PropertyContent, DesignTokens, UiExt};
use re_video::{decode::FrameInfo, demux::SamplesStatistics, VideoData};
use re_viewer_context::UiLayout;

pub fn show_video_blob_info(
pub fn video_result_ui(
ui: &mut egui::Ui,
ui_layout: UiLayout,
video_result: &Result<re_renderer::video::Video, VideoLoadError>,
) {
re_tracing::profile_function!();

#[allow(clippy::match_same_arms)]
match video_result {
Ok(video) => {
if ui_layout.is_single_line() {
return;
if !ui_layout.is_single_line() {
re_ui::list_item::list_item_scope(ui, "video_blob_info", |ui| {
video_data_ui(ui, ui_layout, video.data());
});
}

let data = video.data();

re_ui::list_item::list_item_scope(ui, "video_blob_info", |ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Dimensions").value_text(format!(
"{}x{}",
data.width(),
data.height()
)),
);
if let Some(bit_depth) = data.config.stsd.contents.bit_depth() {
ui.list_item_flat_noninteractive(PropertyContent::new("Bit depth").value_fn(
|ui, _| {
ui.label(bit_depth.to_string());
if 8 < bit_depth {
// TODO(#7594): HDR videos
ui.warning_label("HDR").on_hover_ui(|ui| {
ui.label(
"High-dynamic-range videos not yet supported by Rerun",
);
ui.hyperlink("https://github.com/rerun-io/rerun/issues/7594");
});
}
if data.is_monochrome() == Some(true) {
ui.label("(monochrome)");
}
},
));
}
if let Some(subsampling_mode) = data.subsampling_mode() {
// Don't show subsampling mode for monochrome, doesn't make sense usually.
if data.is_monochrome() != Some(true) {
ui.list_item_flat_noninteractive(
PropertyContent::new("Subsampling")
.value_text(subsampling_mode.to_string()),
);
}
}
ui.list_item_flat_noninteractive(
PropertyContent::new("Duration")
.value_text(format!("{}", re_log_types::Duration::from(data.duration()))),
);
// Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate.
// Video containers and codecs like talking about samples or chunks rather than frames, but for how we define a chunk today,
// a frame is always a single chunk of data is always a single sample, see [`re_video::decode::Chunk`].
// So for all practical purposes the sample count _is_ the number of frames, at least how we use it today.
ui.list_item_flat_noninteractive(
PropertyContent::new("Frame count")
.value_text(re_format::format_uint(data.num_samples())),
);
ui.list_item_flat_noninteractive(
PropertyContent::new("Codec").value_text(data.human_readable_codec_string()),
);

if ui_layout != UiLayout::Tooltip {
ui.list_item_collapsible_noninteractive_label("MP4 tracks", true, |ui| {
for (track_id, track_kind) in &data.mp4_tracks {
let track_kind_string = match track_kind {
Some(re_video::TrackKind::Audio) => "audio",
Some(re_video::TrackKind::Subtitle) => "subtitle",
Some(re_video::TrackKind::Video) => "video",
None => "unknown",
};
ui.list_item_flat_noninteractive(
PropertyContent::new(format!("Track {track_id}"))
.value_text(track_kind_string),
);
}
});
ui.list_item_collapsible_noninteractive_label(
"More video statistics",
false,
|ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Number of GOPs")
.value_text(data.gops.len().to_string()),
)
.on_hover_text(
"The total number of Group of Pictures (GOPs) in the video.",
);
samples_statistics_ui(ui, &data.samples_statistics);
},
);
}
});
}
Err(VideoLoadError::MimeTypeIsNotAVideo { .. }) => {
// Don't show an error if this wasn't a video in the first place.
Expand All @@ -126,6 +45,170 @@ pub fn show_video_blob_info(
}
}

fn video_data_ui(ui: &mut egui::Ui, ui_layout: UiLayout, video_data: &VideoData) {
re_tracing::profile_function!();

ui.list_item_flat_noninteractive(PropertyContent::new("Dimensions").value_text(format!(
"{}x{}",
video_data.width(),
video_data.height()
)));
if let Some(bit_depth) = video_data.config.stsd.contents.bit_depth() {
ui.list_item_flat_noninteractive(PropertyContent::new("Bit depth").value_fn(|ui, _| {
ui.label(bit_depth.to_string());
if 8 < bit_depth {
// TODO(#7594): HDR videos
ui.warning_label("HDR").on_hover_ui(|ui| {
ui.label("High-dynamic-range videos not yet supported by Rerun");
ui.hyperlink("https://github.com/rerun-io/rerun/issues/7594");
});
}
if video_data.is_monochrome() == Some(true) {
ui.label("(monochrome)");
}
}));
}
if let Some(subsampling_mode) = video_data.subsampling_mode() {
// Don't show subsampling mode for monochrome, doesn't make sense usually.
if video_data.is_monochrome() != Some(true) {
ui.list_item_flat_noninteractive(
PropertyContent::new("Subsampling").value_text(subsampling_mode.to_string()),
);
}
}
ui.list_item_flat_noninteractive(PropertyContent::new("Duration").value_text(format!(
"{}",
re_log_types::Duration::from(video_data.duration())
)));
// Some people may think that num_frames / duration = fps, but that's not true, videos may have variable frame rate.
// Video containers and codecs like talking about samples or chunks rather than frames, but for how we define a chunk today,
// a frame is always a single chunk of data is always a single sample, see [`re_video::decode::Chunk`].
// So for all practical purposes the sample count _is_ the number of frames, at least how we use it today.
ui.list_item_flat_noninteractive(
PropertyContent::new("Frame count")
.value_text(re_format::format_uint(video_data.num_samples())),
);
ui.list_item_flat_noninteractive(
PropertyContent::new("Codec").value_text(video_data.human_readable_codec_string()),
);

if ui_layout != UiLayout::Tooltip {
ui.list_item_collapsible_noninteractive_label("MP4 tracks", true, |ui| {
for (track_id, track_kind) in &video_data.mp4_tracks {
let track_kind_string = match track_kind {
Some(re_video::TrackKind::Audio) => "audio",
Some(re_video::TrackKind::Subtitle) => "subtitle",
Some(re_video::TrackKind::Video) => "video",
None => "unknown",
};
ui.list_item_flat_noninteractive(
PropertyContent::new(format!("Track {track_id}")).value_text(track_kind_string),
);
}
});

ui.list_item_collapsible_noninteractive_label("More video statistics", false, |ui| {
ui.list_item_flat_noninteractive(
PropertyContent::new("Number of GOPs")
.value_text(video_data.gops.len().to_string()),
)
.on_hover_text("The total number of Group of Pictures (GOPs) in the video.");
samples_statistics_ui(ui, &video_data.samples_statistics);
});

ui.list_item_collapsible_noninteractive_label("Video samples", false, |ui| {
samples_table_ui(ui, video_data);
});
}
}

fn samples_table_ui(ui: &mut egui::Ui, video_data: &VideoData) {
re_tracing::profile_function!();

egui_extras::TableBuilder::new(ui)
.auto_shrink([false, true])
.vscroll(true)
.max_scroll_height(800.0)
.columns(Column::auto(), 7)
.cell_layout(egui::Layout::left_to_right(egui::Align::Center))
.header(DesignTokens::table_header_height(), |mut header| {
DesignTokens::setup_table_header(&mut header);
header.col(|ui| {
ui.strong("Sample");
});
header.col(|ui| {
ui.strong("GOP");
});
header.col(|ui| {
ui.strong("Sync");
});
header.col(|ui| {
ui.strong("DTS").on_hover_text("Decode timestamp");
});
header.col(|ui| {
ui.strong("PTS").on_hover_text("Presentation timestamp");
});
header.col(|ui| {
ui.strong("Duration");
});
header.col(|ui| {
ui.strong("Size");
});
})
.body(|mut body| {
DesignTokens::setup_table_body(&mut body);

body.rows(
DesignTokens::table_line_height(),
video_data.samples.len(),
|mut row| {
let sample_idx = row.index();
let sample = &video_data.samples[sample_idx];
let re_video::Sample {
is_sync,
decode_timestamp,
presentation_timestamp,
duration,
byte_offset: _,
byte_length,
} = *sample;

row.col(|ui| {
ui.monospace(sample_idx.to_string());
});
row.col(|ui| {
if let Some(gop_index) = video_data
.gop_index_containing_presentation_timestamp(presentation_timestamp)
{
ui.monospace(re_format::format_uint(gop_index));
}
});
row.col(|ui| {
if is_sync {
ui.label("sync");
}
});
row.col(|ui| {
ui.monospace(re_format::format_int(decode_timestamp.0));
});
row.col(|ui| {
ui.monospace(re_format::format_int(presentation_timestamp.0));
});

row.col(|ui| {
ui.monospace(
re_log_types::Duration::from(duration.duration(video_data.timescale))
.to_string(),
);
});
row.col(|ui| {
ui.monospace(re_format::format_bytes(byte_length as _));
});
},
);
});
}

pub fn show_decoded_frame_info(
render_ctx: Option<&re_renderer::RenderContext>,
ui: &mut egui::Ui,
Expand Down
2 changes: 1 addition & 1 deletion crates/viewer/re_renderer/src/video/player.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ impl VideoPlayer {
return Ok(());
};

let samples = &self.data.samples[gop.decode_time_range()];
let samples = &self.data.samples[gop.sample_range_usize()];

for sample in samples {
let chunk = sample.get(video_data).ok_or(VideoPlayerError::BadData)?;
Expand Down
Loading