SDK DataLoaders 3: barebones C and C++ support (#5330)

Exposes raw `DataLoader`s to the C++ SDK through 2 new methods:
`RecordingStream::log_file_from_path` &
`RecordingStream::log_file_from_contents`.

Everything streams asynchronously, as expected.

There's no way to configure the behavior of the `DataLoader` at all,
yet. That comes next.

:warning: I'm carrying bytes around in a `string_view` (see `rr_bytes`)
and related, and the internet has been unhelpful to tell e whether I
should or shouldn't.

```cpp
rec.log_file_from_path(filepath);
```

![image](https://github.com/rerun-io/rerun/assets/2910679/1933f08b-1b2b-483b-9576-ce2e0421cb62)



---

Part of series of PR to expose configurable `DataLoader`s to our SDKs:
- #5327 
- #5328 
- #5330
- #5337
- #5351
- #5355

---------

Co-authored-by: Andreas Reich <andreas@rerun.io>

crates/rerun_c/Cargo.toml

@@ -32,7 +32,7 @@ test = false
 
 [dependencies]
 re_log = { workspace = true, features = ["setup"] }
-re_sdk.workspace = true
+re_sdk = { workspace = true, features = ["data_loaders"] }
 
 ahash.workspace = true
 arrow2.workspace = true

crates/rerun_c/src/lib.rs

@@ -10,7 +10,7 @@ mod error;
 mod ptr;
 mod recording_streams;
 
-use std::ffi::{c_char, CString};
+use std::ffi::{c_char, c_uchar, CString};
 
 use component_type_registry::COMPONENT_TYPES;
 use once_cell::sync::Lazy;
@@ -48,6 +48,29 @@ impl CStringView {
     }
 }
 
+/// This is called `rr_bytes` in the C API.
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+pub struct CBytesView {
+    pub bytes: *const c_uchar,
+    pub length: u32,
+}
+
+impl CBytesView {
+    #[allow(clippy::result_large_err)]
+    pub fn as_bytes<'a>(&self, argument_name: &'a str) -> Result<&'a [u8], CError> {
+        ptr::try_ptr_as_slice(self.bytes, self.length, argument_name)
+    }
+
+    pub fn is_null(&self) -> bool {
+        self.bytes.is_null()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.length == 0
+    }
+}
+
 pub type CRecordingStream = u32;
 
 pub type CComponentTypeHandle = u32;
@@ -159,10 +182,10 @@ pub enum CErrorCode {
     InvalidComponentTypeHandle,
 
     _CategoryRecordingStream = 0x0000_00100,
+    RecordingStreamRuntimeFailure,
     RecordingStreamCreationFailure,
     RecordingStreamSaveFailure,
     RecordingStreamStdoutFailure,
-    // TODO(cmc): Really this should be its own category…
     RecordingStreamSpawnFailure,
 
     _CategoryArrow = 0x0000_1000,
@@ -703,6 +726,74 @@ pub unsafe extern "C" fn rr_recording_stream_log(
     }
 }
 
+#[allow(unsafe_code)]
+#[allow(clippy::result_large_err)]
+fn rr_log_file_from_path_impl(
+    stream: CRecordingStream,
+    filepath: CStringView,
+) -> Result<(), CError> {
+    let stream = recording_stream(stream)?;
+
+    let filepath = filepath.as_str("filepath")?;
+    stream.log_file_from_path(filepath).map_err(|err| {
+        CError::new(
+            CErrorCode::RecordingStreamRuntimeFailure,
+            &format!("Couldn't load file {filepath:?}: {err}"),
+        )
+    })?;
+
+    Ok(())
+}
+
+#[allow(unsafe_code)]
+#[no_mangle]
+pub unsafe extern "C" fn rr_recording_stream_log_file_from_path(
+    stream: CRecordingStream,
+    filepath: CStringView,
+    error: *mut CError,
+) {
+    if let Err(err) = rr_log_file_from_path_impl(stream, filepath) {
+        err.write_error(error);
+    }
+}
+
+#[allow(unsafe_code)]
+#[allow(clippy::result_large_err)]
+fn rr_log_file_from_contents_impl(
+    stream: CRecordingStream,
+    filepath: CStringView,
+    contents: CBytesView,
+) -> Result<(), CError> {
+    let stream = recording_stream(stream)?;
+
+    let filepath = filepath.as_str("filepath")?;
+    let contents = contents.as_bytes("contents")?;
+
+    stream
+        .log_file_from_contents(filepath, std::borrow::Cow::Borrowed(contents))
+        .map_err(|err| {
+            CError::new(
+                CErrorCode::RecordingStreamRuntimeFailure,
+                &format!("Couldn't load file {filepath:?}: {err}"),
+            )
+        })?;
+
+    Ok(())
+}
+
+#[allow(unsafe_code)]
+#[no_mangle]
+pub unsafe extern "C" fn rr_recording_stream_log_file_from_contents(
+    stream: CRecordingStream,
+    filepath: CStringView,
+    contents: CBytesView,
+    error: *mut CError,
+) {
+    if let Err(err) = rr_log_file_from_contents_impl(stream, filepath, contents) {
+        err.write_error(error);
+    }
+}
+
 // ----------------------------------------------------------------------------
 // Private functions
 

crates/rerun_c/src/ptr.rs

@@ -2,6 +2,8 @@ use std::ffi::c_char;
 
 use crate::{CError, CErrorCode};
 
+// ---
+
 #[allow(unsafe_code)]
 #[allow(clippy::result_large_err)]
 pub fn try_ptr_as_ref<T>(ptr: *const T, argument_name: &str) -> Result<&T, CError> {
@@ -13,6 +15,17 @@ pub fn try_ptr_as_ref<T>(ptr: *const T, argument_name: &str) -> Result<&T, CErro
     }
 }
 
+#[allow(unsafe_code)]
+#[allow(clippy::result_large_err)]
+pub fn try_ptr_as_slice<T>(
+    ptr: *const T,
+    length: u32,
+    argument_name: &str,
+) -> Result<&[T], CError> {
+    try_ptr_as_ref(ptr, argument_name)?;
+    Ok(unsafe { std::slice::from_raw_parts(ptr.cast::<T>(), length as usize) })
+}
+
 /// Tries to convert a [`c_char`] pointer to a string, raises an error if the pointer is null or it can't be converted to a string.
 #[allow(unsafe_code)]
 #[allow(clippy::result_large_err)]

crates/rerun_c/src/rerun.h

@@ -33,6 +33,17 @@ typedef struct rr_string {
     uint32_t length_in_bytes;
 } rr_string;
 
+/// A byte slice.
+typedef struct rr_bytes {
+    /// Pointer to the bytes.
+    ///
+    /// Rerun is guaranteed to not read beyond bytes[length-1].
+    const uint8_t* bytes;
+
+    /// The length of the data in bytes.
+    uint32_t length;
+} rr_bytes;
+
 #ifndef __cplusplus
 
 #include <string.h> // For strlen
@@ -412,6 +423,30 @@ extern void rr_recording_stream_log(
     rr_recording_stream stream, rr_data_row data_row, bool inject_time, rr_error* error
 );
 
+/// Logs the file at the given `path` using all `DataLoader`s available.
+///
+/// A single `path` might be handled by more than one loader.
+///
+/// This method blocks until either at least one `DataLoader` starts streaming data in
+/// or all of them fail.
+///
+/// See <https://www.rerun.io/docs/howto/open-any-file> for more information.
+extern void rr_recording_stream_log_file_from_path(
+    rr_recording_stream stream, rr_string path, rr_error* error
+);
+
+/// Logs the given `contents` using all `DataLoader`s available.
+///
+/// A single `path` might be handled by more than one loader.
+///
+/// This method blocks until either at least one `DataLoader` starts streaming data in
+/// or all of them fail.
+///
+/// See <https://www.rerun.io/docs/howto/open-any-file> for more information.
+extern void rr_recording_stream_log_file_from_contents(
+    rr_recording_stream stream, rr_string path, rr_bytes contents, rr_error* error
+);
+
 // ----------------------------------------------------------------------------
 // Private functions
 

examples/cpp/CMakeLists.txt

@@ -2,6 +2,7 @@ add_subdirectory(clock)
 add_subdirectory(custom_collection_adapter)
 add_subdirectory(dna)
 add_subdirectory(external_data_loader)
+add_subdirectory(log_file)
 add_subdirectory(minimal)
 add_subdirectory(shared_recording)
 add_subdirectory(spawn_viewer)
@@ -12,6 +13,7 @@ add_custom_target(examples)
 add_dependencies(examples example_clock)
 add_dependencies(examples example_custom_collection_adapter)
 add_dependencies(examples example_dna)
+add_dependencies(examples example_log_file)
 add_dependencies(examples example_minimal)
 add_dependencies(examples example_shared_recording)
 add_dependencies(examples example_spawn_viewer)

examples/cpp/log_file/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required(VERSION 3.16...3.27)
+
+# If you use the example outside of the Rerun SDK you need to specify
+# where the rerun_c build is to be found by setting the `RERUN_CPP_URL` variable.
+# This can be done by passing `-DRERUN_CPP_URL=<path to rerun_sdk_cpp zip>` to cmake.
+if(DEFINED RERUN_REPOSITORY)
+    add_executable(example_log_file main.cpp)
+    rerun_strict_warning_settings(example_log_file)
+else()
+    project(example_log_file LANGUAGES CXX)
+
+    add_executable(example_log_file main.cpp)
+
+    # Set the path to the rerun_c build.
+    set(RERUN_CPP_URL "https://github.com/rerun-io/rerun/releases/latest/download/rerun_cpp_sdk.zip" CACHE STRING "URL to the rerun_cpp zip.")
+    option(RERUN_FIND_PACKAGE "Whether to use find_package to find a preinstalled rerun package (instead of using FetchContent)." OFF)
+
+    if(RERUN_FIND_PACKAGE)
+        find_package(rerun_sdk REQUIRED)
+    else()
+        # Download the rerun_sdk
+        include(FetchContent)
+        FetchContent_Declare(rerun_sdk URL ${RERUN_CPP_URL})
+        FetchContent_MakeAvailable(rerun_sdk)
+    endif()
+
+    # Rerun requires at least C++17, but it should be compatible with newer versions.
+    set_property(TARGET example_log_file PROPERTY CXX_STANDARD 17)
+endif()
+
+# Link against rerun_sdk.
+target_link_libraries(example_log_file PRIVATE rerun_sdk)

examples/cpp/log_file/README.md

@@ -0,0 +1,12 @@
+<!--[metadata]
+title = "Log file example"
+-->
+
+Demonstrates how to log any file from the SDK using the [`DataLoader`](https://www.rerun.io/docs/howto/open-any-file) machinery.
+
+To build it from a checkout of the repository (requires a Rust toolchain):
+```bash
+cmake .
+cmake --build . --target example_log_file
+./examples/cpp/log_file/example_log_file examples/assets/
+```

examples/cpp/log_file/main.cpp

@@ -0,0 +1,79 @@
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include <rerun.hpp>
+#include <rerun/third_party/cxxopts.hpp>
+
+int main(int argc, char** argv) {
+    // Create a new `RecordingStream` which sends data over TCP to the viewer process.
+    const auto rec = rerun::RecordingStream("rerun_example_log_file");
+
+    cxxopts::Options options(
+        "rerun_example_log_file",
+        "Demonstrates how to log any file from the SDK using the `DataLoader` machinery."
+    );
+
+    // clang-format off
+    options.add_options()
+      ("h,help", "Print usage")
+      // Rerun
+      ("spawn", "Start a new Rerun Viewer process and feed it data in real-time")
+      ("connect", "Connects and sends the logged data to a remote Rerun viewer")
+      ("save", "Log data to an rrd file", cxxopts::value<std::string>())
+      ("stdout", "Log data to standard output, to be piped into a Rerun Viewer")
+      // Example
+      ("from-contents", "Log the contents of the file directly (files only -- not supported by external loaders)", cxxopts::value<bool>()->default_value("false"))
+      ("filepaths", "The filepaths to be loaded and logged", cxxopts::value<std::vector<std::string>>())
+    ;
+    // clang-format on
+
+    options.parse_positional({"filepaths"});
+
+    auto args = options.parse(argc, argv);
+
+    if (args.count("help")) {
+        std::cout << options.help() << std::endl;
+        exit(0);
+    }
+
+    // TODO(#4602): need common rerun args helper library
+    if (args["spawn"].as<bool>()) {
+        rec.spawn().exit_on_failure();
+    } else if (args["connect"].as<bool>()) {
+        rec.connect().exit_on_failure();
+    } else if (args["stdout"].as<bool>()) {
+        rec.to_stdout().exit_on_failure();
+    } else if (args.count("save")) {
+        rec.save(args["save"].as<std::string>()).exit_on_failure();
+    } else {
+        rec.spawn().exit_on_failure();
+    }
+
+    const auto from_contents = args["from-contents"].as<bool>();
+    if (args.count("filepaths")) {
+        const auto filepaths = args["filepaths"].as<std::vector<std::string>>();
+        for (const auto& filepath : filepaths) {
+            if (!from_contents) {
+                // Either log the file using its path…
+                rec.log_file_from_path(filepath);
+            } else {
+                // …or using its contents if you already have them loaded for some reason.
+                if (std::filesystem::is_regular_file(filepath)) {
+                    std::ifstream file(filepath);
+                    std::stringstream contents;
+                    contents << file.rdbuf();
+
+                    const auto data = contents.str();
+                    rec.log_file_from_contents(
+                        filepath,
+                        reinterpret_cast<const std::byte*>(data.c_str()),
+                        data.size()
+                    );
+                }
+            }
+        }
+    }
+}