Support H.264 on native via user installed ffmpeg executable (#7962)

### What

* Most of #7607 
* Replaces #7658 

Tested this with videos containing b frames, both long and short.

<img width="1403" alt="image"
src="https://github.com/user-attachments/assets/12da809b-5aee-4c06-ba30-394d82073a2a">

(note how PTS != DTS, indicating that there's a b-frame _somewhere_ ;-))

Biggest risk overall with the approach taken is that we can't detect
dropped frames properly

Left to do before landing:
* [x] test 4k 60fps video material
* works smoothly in release, not at 60fps in debug builds on my mac
because of rgb->rgba conversions on our side (see todos below)
* [x] test black & white (yuv400)
* [x] works but my video test material gave me:
`re_video::decode::ffmpeg]
/\//Users/andreas/dev/rerun-io/internal-test-assets/video/greyscale_360p_h264.mp4,
codec: H.264 (avc1.64001E) decoder: [swscaler @ 0x3001d8000] [warning]
deprecated pixel format used, make sure you did set range correctly`
every time ffmpeg started, need to fix that to a log_once (if ffmpeg
says it's deprecated then I figure we may report that 🤷, just don't
spam)
* [x] test yuv444/rgb video
* worked, but the test 4.4s test video I used stops working starting at
3.2s, recovers fine though
* Direct play browser: Firefox says the file is corrupt, Safari refuses
to play. Chrome handles it
        * Rerun play browser: Same, but different errors
        * ... letting this issue slide for now 🤷 

What's left to do (beyond general video bugs I logged while working on
this) before calling this "done":
* download ffmpeg for the user or at least tell users what to do when
there's no ffmpeg (haven't checked even yet on the error message)
* add version & ability check for ffmpeg?
* consume 420p and friends directly to speed up processing
* try to provoke frame dropping and see if we can detect & handle that -
one of the shortcomings of the overall approach today is that we don't
track this!

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7962?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7962?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7962)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

---------

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

Cargo.lock

@@ -2370,6 +2370,15 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "ffmpeg-sidecar"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bd1e249e0ceeb0f5c9f84a3c6941c3bde3ebc2815f4b94531a7e806af61c4c0"
+dependencies = [
+ "anyhow",
+]
+
 [[package]]
 name = "filetime"
 version = "0.2.25"
@@ -6319,6 +6328,7 @@ dependencies = [
  "criterion",
  "crossbeam",
  "econtext",
+ "ffmpeg-sidecar",
  "indicatif",
  "itertools 0.13.0",
  "js-sys",

Cargo.toml

@@ -178,6 +178,7 @@ econtext = "0.2" # Prints error contexts on crashes
 ehttp = "0.5.0"
 enumset = "1.0.12"
 env_logger = { version = "0.10", default-features = false }
+ffmpeg-sidecar = "1.1.2"
 fixed = { version = "1.28", default-features = false }
 flatbuffers = "23.0"
 futures-channel = "0.3"

crates/store/re_video/Cargo.toml

@@ -23,14 +23,17 @@ features = ["all"]
 
 
 [features]
-default = ["av1"]
+default = ["av1", "ffmpeg"]
 
 ## Enable serialization for data structures that support it.
 serde = ["dep:serde"]
 
 ## Native AV1 decoding.
 av1 = ["dep:dav1d"]
 
+## Decode H.264 using ffmpeg over CLI.
+ffmpeg = ["dep:ffmpeg-sidecar"]
+
 ## Enable faster native video decoding with assembly.
 ## You need to install [nasm](https://nasm.us/) to compile with this feature.
 nasm = [
@@ -49,9 +52,11 @@ econtext.workspace = true
 itertools.workspace = true
 parking_lot.workspace = true
 re_mp4.workspace = true
-serde = { workspace = true, optional = true }
 thiserror.workspace = true
 
+ffmpeg-sidecar = { workspace = true, optional = true }
+serde = { workspace = true, optional = true }
+
 # We enable re_rav1d on native, UNLESS we're on Linux Arm64
 # See https://github.com/rerun-io/rerun/issues/7755
 [target.'cfg(all(not(target_arch = "wasm32"), not(all(target_os = "linux", target_arch = "aarch64"))))'.dependencies]
@@ -83,6 +88,7 @@ web-sys = { workspace = true, features = [
 ] }
 
 [dev-dependencies]
+# For the `frames` example:
 indicatif.workspace = true
 criterion.workspace = true
 

crates/store/re_video/build.rs

@@ -10,5 +10,6 @@ fn main() {
         native: { not(target_arch = "wasm32") },
         linux_arm64: { all(target_os = "linux", target_arch = "aarch64") },
         with_dav1d: { all(feature = "av1", native, not(linux_arm64)) }, // https://github.com/rerun-io/rerun/issues/7755
+        with_ffmpeg: { all(feature= "ffmpeg", native) }
     }
 }

crates/store/re_video/examples/frames.rs

@@ -14,6 +14,8 @@ use indicatif::ProgressBar;
 use parking_lot::Mutex;
 
 fn main() {
+    re_log::setup_logging();
+
     // frames <video.mp4>
     let args: Vec<_> = std::env::args().collect();
     let Some(video_path) = args.get(1) else {
@@ -83,13 +85,20 @@ fn main() {
                 .create(true)
                 .truncate(true)
                 .open(output_dir.join(format!("{i:0width$}.ppm")))
-                .expect("failed to open file");
-            write_binary_ppm(
-                &mut file,
-                frame.content.width,
-                frame.content.height,
-                &frame.content.data,
-            );
+                .expect("failed to oformatpen file");
+
+            let frame = &frame.content;
+            match frame.format {
+                re_video::PixelFormat::Rgb8Unorm => {
+                    write_ppm_rgb24(&mut file, frame.width, frame.height, &frame.data);
+                }
+                re_video::PixelFormat::Rgba8Unorm => {
+                    write_ppm_rgba32(&mut file, frame.width, frame.height, &frame.data);
+                }
+                re_video::PixelFormat::Yuv { .. } => {
+                    re_log::error_once!("YUV frame writing is not supported");
+                }
+            }
         }
     }
 }
@@ -98,7 +107,24 @@ fn num_digits(n: usize) -> usize {
     (n as f64).log10().floor() as usize + 1
 }
 
-fn write_binary_ppm(file: &mut File, width: u32, height: u32, rgba: &[u8]) {
+fn write_ppm_rgb24(file: &mut File, width: u32, height: u32, rgb: &[u8]) {
+    assert_eq!(width as usize * height as usize * 3, rgb.len());
+
+    let header = format!("P6\n{width} {height}\n255\n");
+
+    let mut data = Vec::with_capacity(header.len() + width as usize * height as usize * 3);
+    data.extend_from_slice(header.as_bytes());
+
+    for rgb in rgb.chunks(3) {
+        data.extend_from_slice(&[rgb[0], rgb[1], rgb[2]]);
+    }
+
+    file.write_all(&data).expect("failed to write frame data");
+}
+
+fn write_ppm_rgba32(file: &mut File, width: u32, height: u32, rgba: &[u8]) {
+    assert_eq!(width as usize * height as usize * 4, rgba.len());
+
     let header = format!("P6\n{width} {height}\n255\n");
 
     let mut data = Vec::with_capacity(header.len() + width as usize * height as usize * 3);

crates/store/re_video/src/decode/async_decoder_wrapper.rs

@@ -73,7 +73,7 @@ impl AsyncDecoderWrapper {
         let comms = Comms::default();
 
         let thread = std::thread::Builder::new()
-            .name("av1_decoder".into())
+            .name(format!("decoder of {debug_name}"))
             .spawn({
                 let comms = comms.clone();
                 move || {

crates/store/re_video/src/decode/av1.rs

@@ -76,14 +76,14 @@ impl SyncDav1dDecoder {
         re_tracing::profile_function!();
         econtext::econtext_function_data!(format!(
             "chunk timestamp: {:?}",
-            chunk.composition_timestamp
+            chunk.presentation_timestamp
         ));
 
         re_tracing::profile_scope!("send_data");
         match self.decoder.send_data(
             chunk.data,
             None,
-            Some(chunk.composition_timestamp.0),
+            Some(chunk.presentation_timestamp.0),
             Some(chunk.duration.0),
         ) {
             Ok(()) => {}
@@ -255,6 +255,7 @@ fn create_frame(debug_name: &str, picture: &dav1d::Picture) -> Result<Frame> {
         info: FrameInfo {
             presentation_timestamp: Time(picture.timestamp().unwrap_or(0)),
             duration: Time(picture.duration()),
+            ..Default::default()
         },
     })
 }