diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9821e24ad139..556f02977008 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -137,18 +137,20 @@ jobs: command: cranky args: --all-targets --all-features -- --deny warnings - - name: Check no default features + # -------------------------------------------------------------------------------- + # Check a few important permutations of the feature flags for our `rerun` library: + - name: Check rerun with `--no-default-features`` uses: actions-rs/cargo@v1 with: - command: check - args: --locked --no-default-features --features __ci --lib + command: cranky + args: --locked -p rerun --no-default-features - # Check a few important permutations of the feature flags for our `rerun` library: - - name: Check rerun with --features sdk + - name: Check rerun with `--features sdk` uses: actions-rs/cargo@v1 with: - command: check - args: --locked --no-default-features --features sdk + command: cranky + args: --locked -p rerun --no-default-features --features sdk + # -------------------------------------------------------------------------------- - name: Test doc-tests uses: actions-rs/cargo@v1 diff --git a/Cargo.lock b/Cargo.lock index 331040f76391..e7986f73779c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3129,8 +3129,6 @@ dependencies = [ "anyhow", "clap 4.1.4", "glam", - "image", - "itertools", "prost", "prost-build", "rerun", @@ -3842,9 +3840,9 @@ dependencies = [ "re_arrow_store", "re_int_histogram", "re_log", + "re_log_encoding", "re_log_types", "re_smart_channel", - "re_string_interner", "serde", "thiserror", ] @@ -3891,17 +3889,40 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "re_log_encoding" +version = "0.4.0" +dependencies = [ + "criterion", + "ehttp", + "instant", + "js-sys", + "mimalloc", + "parking_lot 0.12.1", + "puffin", + "re_build_info", + "re_log", + "re_log_types", + "re_smart_channel", + "rmp-serde", + "ruzstd", + "serde_test", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "zstd", +] + [[package]] name = "re_log_types" version = "0.4.0" dependencies = [ "ahash 0.8.2", - "anyhow", "array-init", "arrow2", "arrow2_convert", "bytemuck", - "criterion", "document-features", "ecolor", "fixed", @@ -3911,29 +3932,24 @@ dependencies = [ "itertools", "lazy_static", "macaw", - "mimalloc", "ndarray", "nohash-hasher", "num-derive", "num-traits", "puffin", "rand", - "re_build_info", "re_format", "re_log", "re_string_interner", "re_tuid", "rmp-serde", - "ruzstd", "serde", "serde_bytes", - "serde_test", "smallvec", "thiserror", "time 0.3.20", "typenum", "uuid", - "zstd", ] [[package]] @@ -3970,7 +3986,6 @@ dependencies = [ "re_arrow_store", "re_data_store", "re_format", - "re_log", "re_log_types", "thiserror", ] @@ -4032,23 +4047,19 @@ dependencies = [ name = "re_sdk" version = "0.4.0" dependencies = [ - "anyhow", "arrow2_convert", "document-features", "ndarray", "ndarray-rand", - "nohash-hasher", "once_cell", "parking_lot 0.12.1", "rand", "re_build_build_info", - "re_build_info", - "re_error", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_sdk_comms", - "re_smart_channel", "thiserror", ] @@ -4143,14 +4154,12 @@ dependencies = [ "egui-wgpu", "egui_dock", "egui_extras", - "ehttp", "enumset", "glam", "half 2.2.1", "image", "instant", "itertools", - "js-sys", "lazy_static", "macaw", "ndarray", @@ -4167,12 +4176,12 @@ dependencies = [ "re_error", "re_format", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_query", "re_renderer", "re_smart_channel", - "re_string_interner", "re_tensor_ops", "re_ui", "re_ws_comms", @@ -4184,9 +4193,7 @@ dependencies = [ "time 0.3.20", "uuid", "vec1", - "wasm-bindgen", "wasm-bindgen-futures", - "web-sys", "wgpu", "winapi", ] @@ -4288,18 +4295,16 @@ dependencies = [ "clap 4.1.4", "ctrlc", "document-features", - "egui", "itertools", "libc", "mimalloc", - "once_cell", "parking_lot 0.12.1", "re_analytics", "re_build_build_info", "re_build_info", - "re_error", "re_format", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_sdk", @@ -4316,14 +4321,9 @@ dependencies = [ name = "rerun_py" version = "0.4.0" dependencies = [ - "ahash 0.8.2", - "anyhow", "arrow2", - "bytemuck", - "crossbeam", "document-features", "glam", - "half 2.2.1", "image", "itertools", "macaw", @@ -4340,7 +4340,6 @@ dependencies = [ "re_log", "re_log_types", "re_memory", - "re_tensor_ops", "rerun", "tokio", "uuid", diff --git a/Cargo.toml b/Cargo.toml index baa3b459875f..76c8577f1c6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ re_error = { path = "crates/re_error", version = "0.4.0" } re_format = { path = "crates/re_format", version = "0.4.0" } re_int_histogram = { path = "crates/re_int_histogram", version = "0.4.0" } re_log = { path = "crates/re_log", version = "0.4.0" } +re_log_encoding = { path = "crates/re_log_encoding", version = "0.4.0" } re_log_types = { path = "crates/re_log_types", version = "0.4.0" } re_memory = { path = "crates/re_memory", version = "0.4.0" } re_query = { path = "crates/re_query", version = "0.4.0" } diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index a77bdcb40096..4b5991b37c0a 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -26,10 +26,10 @@ serde = ["dep:serde", "re_log_types/serde"] [dependencies] re_arrow_store.workspace = true re_int_histogram.workspace = true +re_log_encoding = { workspace = true, optional = true } re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true -re_string_interner.workspace = true ahash.workspace = true document-features = "0.2" @@ -47,7 +47,7 @@ puffin.workspace = true criterion = "0.4" mimalloc.workspace = true rand = "0.8" -re_log_types = { workspace = true, features = ["load", "save"] } +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } [lib] bench = false @@ -55,4 +55,4 @@ bench = false [[example]] name = "memory_usage" path = "examples/memory_usage.rs" -required-features = ["re_log_types/load", "re_log_types/save"] +required-features = ["re_log_encoding/decoder", "re_log_encoding/encoder"] diff --git a/crates/re_data_store/examples/memory_usage.rs b/crates/re_data_store/examples/memory_usage.rs index ff5d1faba8a4..69ca6ef5d6a1 100644 --- a/crates/re_data_store/examples/memory_usage.rs +++ b/crates/re_data_store/examples/memory_usage.rs @@ -65,12 +65,12 @@ fn log_messages() { fn encode_log_msg(log_msg: &LogMsg) -> Vec { let mut bytes = vec![]; - re_log_types::encoding::encode(std::iter::once(log_msg), &mut bytes).unwrap(); + re_log_encoding::encoder::encode(std::iter::once(log_msg), &mut bytes).unwrap(); bytes } fn decode_log_msg(mut bytes: &[u8]) -> LogMsg { - let mut messages = re_log_types::encoding::Decoder::new(&mut bytes) + let mut messages = re_log_encoding::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_log_encoding/Cargo.toml b/crates/re_log_encoding/Cargo.toml new file mode 100644 index 000000000000..b1c5c2943af6 --- /dev/null +++ b/crates/re_log_encoding/Cargo.toml @@ -0,0 +1,69 @@ +[package] +name = "re_log_encoding" +authors.workspace = true +description = "Helpers for encoding and transporting Rerun log messages" +edition.workspace = true +homepage.workspace = true +include.workspace = true +license.workspace = true +publish = true +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = true + + +[features] +default = [] + +## Enable loading data from an .rrd file. +decoder = ["dep:rmp-serde", "dep:zstd", "dep:ruzstd"] + +# Enable encoding of log messages to an .rrd file/stream: +encoder = ["dep:rmp-serde", "dep:zstd"] + + +[dependencies] + +# Rerun: +re_build_info.workspace = true +re_log_types = { workspace = true, features = ["serde"] } +re_log.workspace = true +re_smart_channel.workspace = true + +# External: +ehttp = "0.2" +parking_lot.workspace = true +thiserror.workspace = true + +# Optional external dependencies: +rmp-serde = { version = "1", optional = true } + +# Native dependencies: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +puffin.workspace = true +zstd = { version = "0.11.0", optional = true } # native only + +# Web dependencies: +[target.'cfg(target_arch = "wasm32")'.dependencies] +instant = { version = "0.1", features = ["wasm-bindgen"] } +js-sys = "0.3" +ruzstd = { version = "0.3.0", optional = true } # works on wasm, in contrast to zstd +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3.52", features = ["Window"] } + +[dev-dependencies] +criterion = "0.4" +mimalloc.workspace = true +serde_test = { version = "1" } + +[lib] +bench = false + +[[bench]] +name = "msg_encode_benchmark" +harness = false diff --git a/crates/re_log_encoding/README.md b/crates/re_log_encoding/README.md new file mode 100644 index 000000000000..8b3e7ff4f875 --- /dev/null +++ b/crates/re_log_encoding/README.md @@ -0,0 +1,10 @@ +# re_log_encoding + +Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. + +[![Latest version](https://img.shields.io/crates/v/re_log_encoding.svg)](https://crates.io/crates/re_log_encoding) +[![Documentation](https://docs.rs/re_log_encoding/badge.svg)](https://docs.rs/re_log_encoding) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +Helper library for encoding Rerun log messages. diff --git a/crates/re_log_types/benches/msg_encode_benchmark.rs b/crates/re_log_encoding/benches/msg_encode_benchmark.rs similarity index 96% rename from crates/re_log_types/benches/msg_encode_benchmark.rs rename to crates/re_log_encoding/benches/msg_encode_benchmark.rs index 8faca63d2371..fb36e7bc34d7 100644 --- a/crates/re_log_types/benches/msg_encode_benchmark.rs +++ b/crates/re_log_encoding/benches/msg_encode_benchmark.rs @@ -1,5 +1,5 @@ -#[cfg(not(all(feature = "save", feature = "load")))] -compile_error!("msg_encode_benchmark requires 'save' and 'load' features."); +#[cfg(not(all(feature = "decoder", feature = "encoder")))] +compile_error!("msg_encode_benchmark requires 'decoder' and 'encoder' features."); #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; @@ -28,13 +28,13 @@ criterion_main!(benches); fn encode_log_msgs(messages: &[LogMsg]) -> Vec { let mut bytes = vec![]; - re_log_types::encoding::encode(messages.iter(), &mut bytes).unwrap(); + re_log_encoding::encoder::encode(messages.iter(), &mut bytes).unwrap(); assert!(bytes.len() > messages.len()); bytes } fn decode_log_msgs(mut bytes: &[u8]) -> Vec { - let messages = re_log_types::encoding::Decoder::new(&mut bytes) + let messages = re_log_encoding::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_log_types/src/encoding.rs b/crates/re_log_encoding/src/decoder.rs similarity index 58% rename from crates/re_log_types/src/encoding.rs rename to crates/re_log_encoding/src/decoder.rs index d1e2299c2509..411c580d4439 100644 --- a/crates/re_log_types/src/encoding.rs +++ b/crates/re_log_encoding/src/decoder.rs @@ -1,117 +1,6 @@ -//! Encoding/decoding [`LogMsg`]:es as `.rrd` files. +//! Decoding [`LogMsg`]:es from `.rrd` files/streams. -use crate::LogMsg; - -// ---------------------------------------------------------------------------- -// native encode: - -#[cfg(feature = "save")] -#[cfg(not(target_arch = "wasm32"))] -mod encoder { - use std::io::Write as _; - - use crate::LogMsg; - - /// On failure to encode or serialize a [`LogMsg`]. - #[derive(thiserror::Error, Debug)] - pub enum EncodeError { - #[error("Failed to write: {0}")] - Write(std::io::Error), - - #[error("Zstd error: {0}")] - Zstd(std::io::Error), - - #[error("MsgPack error: {0}")] - MsgPack(#[from] rmp_serde::encode::Error), - - #[error("Called append on already finished encoder")] - AlreadyFinished, - } - - /// Encode a stream of [`LogMsg`] into an `.rrd` file. - pub struct Encoder { - /// Set to None when finished. - zstd_encoder: Option>, - buffer: Vec, - } - - impl Drop for Encoder { - fn drop(&mut self) { - if self.zstd_encoder.is_some() { - re_log::warn!("Encoder dropped without calling finish()!"); - if let Err(err) = self.finish() { - re_log::error!("Failed to finish encoding: {err}"); - } - } - } - } - - impl Encoder { - pub fn new(mut write: W) -> Result { - let rerun_version = re_build_info::CrateVersion::parse(env!("CARGO_PKG_VERSION")); - - write.write_all(b"RRF0").map_err(EncodeError::Write)?; - write - .write_all(&rerun_version.to_bytes()) - .map_err(EncodeError::Write)?; - - let level = 3; - let zstd_encoder = - zstd::stream::Encoder::new(write, level).map_err(EncodeError::Zstd)?; - - Ok(Self { - zstd_encoder: Some(zstd_encoder), - buffer: vec![], - }) - } - - pub fn append(&mut self, message: &LogMsg) -> Result<(), EncodeError> { - let Self { - zstd_encoder, - buffer, - } = self; - - if let Some(zstd_encoder) = zstd_encoder { - buffer.clear(); - rmp_serde::encode::write_named(buffer, message)?; - - zstd_encoder - .write_all(&(buffer.len() as u64).to_le_bytes()) - .map_err(EncodeError::Zstd)?; - zstd_encoder.write_all(buffer).map_err(EncodeError::Zstd)?; - - Ok(()) - } else { - Err(EncodeError::AlreadyFinished) - } - } - - pub fn finish(&mut self) -> Result<(), EncodeError> { - if let Some(zstd_encoder) = self.zstd_encoder.take() { - zstd_encoder.finish().map_err(EncodeError::Zstd)?; - Ok(()) - } else { - re_log::warn!("Encoder::finish called twice"); - Ok(()) - } - } - } - - pub fn encode<'a>( - messages: impl Iterator, - write: impl std::io::Write, - ) -> Result<(), EncodeError> { - let mut encoder = Encoder::new(write)?; - for message in messages { - encoder.append(message)?; - } - encoder.finish() - } -} - -#[cfg(feature = "save")] -#[cfg(not(target_arch = "wasm32"))] -pub use encoder::*; +use re_log_types::LogMsg; // ---------------------------------------------------------------------------- @@ -135,7 +24,6 @@ fn warn_on_version_mismatch(encoded_version: [u8; 4]) { // ---------------------------------------------------------------------------- /// On failure to encode or serialize a [`LogMsg`]. -#[cfg(feature = "load")] #[derive(thiserror::Error, Debug)] pub enum DecodeError { #[error("Not an .rrd file")] @@ -163,14 +51,12 @@ pub enum DecodeError { // ---------------------------------------------------------------------------- // native decode: -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] pub struct Decoder<'r, R: std::io::BufRead> { zdecoder: zstd::stream::Decoder<'r, R>, buffer: Vec, } -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] impl<'r, R: std::io::Read> Decoder<'r, std::io::BufReader> { pub fn new(mut read: R) -> Result { @@ -192,7 +78,6 @@ impl<'r, R: std::io::Read> Decoder<'r, std::io::BufReader> { } } -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] impl<'r, R: std::io::BufRead> Iterator for Decoder<'r, R> { type Item = Result; @@ -225,14 +110,12 @@ impl<'r, R: std::io::BufRead> Iterator for Decoder<'r, R> { // ---------------------------------------------------------------------------- // wasm decode: -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] pub struct Decoder { zdecoder: ruzstd::StreamingDecoder, buffer: Vec, } -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] impl Decoder { pub fn new(mut read: R) -> Result { @@ -254,7 +137,6 @@ impl Decoder { } } -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] impl Iterator for Decoder { type Item = Result; @@ -286,19 +168,22 @@ impl Iterator for Decoder { // ---------------------------------------------------------------------------- -#[cfg(all(feature = "load", feature = "save"))] +#[cfg(all(feature = "decoder", feature = "encoder"))] #[test] fn test_encode_decode() { - use crate::{BeginRecordingMsg, LogMsg, MsgId, Time}; + use re_log_types::{ + ApplicationId, BeginRecordingMsg, LogMsg, MsgId, RecordingId, RecordingInfo, + RecordingSource, Time, + }; let messages = vec![LogMsg::BeginRecordingMsg(BeginRecordingMsg { msg_id: MsgId::random(), - info: crate::RecordingInfo { - application_id: crate::ApplicationId("test".to_owned()), - recording_id: crate::RecordingId::random(), + info: RecordingInfo { + application_id: ApplicationId("test".to_owned()), + recording_id: RecordingId::random(), is_official_example: true, started: Time::now(), - recording_source: crate::RecordingSource::RustSdk { + recording_source: RecordingSource::RustSdk { rustc_version: String::new(), llvm_version: String::new(), }, @@ -306,7 +191,7 @@ fn test_encode_decode() { })]; let mut file = vec![]; - encode(messages.iter(), &mut file).unwrap(); + crate::encoder::encode(messages.iter(), &mut file).unwrap(); let decoded_messages = Decoder::new(&mut file.as_slice()) .unwrap() diff --git a/crates/re_log_encoding/src/encoder.rs b/crates/re_log_encoding/src/encoder.rs new file mode 100644 index 000000000000..13cfbbdd849b --- /dev/null +++ b/crates/re_log_encoding/src/encoder.rs @@ -0,0 +1,100 @@ +//! Encoding of [`LogMsg`]es as a binary stream, e.g. to store in an `.rrd` file, or send over network. + +use std::io::Write as _; + +use re_log_types::LogMsg; + +/// On failure to encode or serialize a [`LogMsg`]. +#[derive(thiserror::Error, Debug)] +pub enum EncodeError { + #[error("Failed to write: {0}")] + Write(std::io::Error), + + #[error("Zstd error: {0}")] + Zstd(std::io::Error), + + #[error("MsgPack error: {0}")] + MsgPack(#[from] rmp_serde::encode::Error), + + #[error("Called append on already finished encoder")] + AlreadyFinished, +} + +/// Encode a stream of [`LogMsg`] into an `.rrd` file. +pub struct Encoder { + /// Set to None when finished. + zstd_encoder: Option>, + buffer: Vec, +} + +impl Drop for Encoder { + fn drop(&mut self) { + if self.zstd_encoder.is_some() { + re_log::warn!("Encoder dropped without calling finish()!"); + if let Err(err) = self.finish() { + re_log::error!("Failed to finish encoding: {err}"); + } + } + } +} + +impl Encoder { + pub fn new(mut write: W) -> Result { + let rerun_version = re_build_info::CrateVersion::parse(env!("CARGO_PKG_VERSION")); + + write.write_all(b"RRF0").map_err(EncodeError::Write)?; + write + .write_all(&rerun_version.to_bytes()) + .map_err(EncodeError::Write)?; + + let level = 3; + let zstd_encoder = zstd::stream::Encoder::new(write, level).map_err(EncodeError::Zstd)?; + + Ok(Self { + zstd_encoder: Some(zstd_encoder), + buffer: vec![], + }) + } + + pub fn append(&mut self, message: &LogMsg) -> Result<(), EncodeError> { + let Self { + zstd_encoder, + buffer, + } = self; + + if let Some(zstd_encoder) = zstd_encoder { + buffer.clear(); + rmp_serde::encode::write_named(buffer, message)?; + + zstd_encoder + .write_all(&(buffer.len() as u64).to_le_bytes()) + .map_err(EncodeError::Zstd)?; + zstd_encoder.write_all(buffer).map_err(EncodeError::Zstd)?; + + Ok(()) + } else { + Err(EncodeError::AlreadyFinished) + } + } + + pub fn finish(&mut self) -> Result<(), EncodeError> { + if let Some(zstd_encoder) = self.zstd_encoder.take() { + zstd_encoder.finish().map_err(EncodeError::Zstd)?; + Ok(()) + } else { + re_log::warn!("Encoder::finish called twice"); + Ok(()) + } + } +} + +pub fn encode<'a>( + messages: impl Iterator, + write: impl std::io::Write, +) -> Result<(), EncodeError> { + let mut encoder = Encoder::new(write)?; + for message in messages { + encoder.append(message)?; + } + encoder.finish() +} diff --git a/crates/re_sdk/src/file_sink.rs b/crates/re_log_encoding/src/file_sink.rs similarity index 89% rename from crates/re_sdk/src/file_sink.rs rename to crates/re_log_encoding/src/file_sink.rs index f5f7e69f6fe1..121383553eb2 100644 --- a/crates/re_sdk/src/file_sink.rs +++ b/crates/re_log_encoding/src/file_sink.rs @@ -17,7 +17,7 @@ pub enum FileSinkError { /// Error encoding a log message. #[error("Failed to encode LogMsg: {0}")] - LogMsgEncode(#[from] re_log_types::encoding::EncodeError), + LogMsgEncode(#[from] crate::encoder::EncodeError), } /// Stream log messages to an `.rrd` file. @@ -47,7 +47,7 @@ impl FileSink { let file = std::fs::File::create(&path) .map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; - let mut encoder = re_log_types::encoding::Encoder::new(file)?; + let mut encoder = crate::encoder::Encoder::new(file)?; let join_handle = std::thread::Builder::new() .name("file_writer".into()) @@ -71,10 +71,8 @@ impl FileSink { join_handle: Some(join_handle), }) } -} -impl crate::sink::LogSink for FileSink { - fn send(&self, msg: LogMsg) { - self.tx.lock().send(Some(msg)).ok(); + pub fn send(&self, log_msg: LogMsg) { + self.tx.lock().send(Some(log_msg)).ok(); } } diff --git a/crates/re_log_encoding/src/lib.rs b/crates/re_log_encoding/src/lib.rs new file mode 100644 index 000000000000..16b883448803 --- /dev/null +++ b/crates/re_log_encoding/src/lib.rs @@ -0,0 +1,42 @@ +//! Crate that handles encoding of rerun log types. + +#[cfg(feature = "decoder")] +pub mod decoder; +#[cfg(feature = "encoder")] +#[cfg(not(target_arch = "wasm32"))] // we do no yet support encoding LogMsgs in the browser +pub mod encoder; + +#[cfg(feature = "encoder")] +#[cfg(not(target_arch = "wasm32"))] +mod file_sink; + +#[cfg(feature = "decoder")] +pub mod stream_rrd_from_http; + +// --------------------------------------------------------------------- + +#[cfg(feature = "encoder")] +#[cfg(not(target_arch = "wasm32"))] +pub use file_sink::{FileSink, FileSinkError}; + +// --------------------------------------------------------------------------- + +/// Profiling macro for feature "puffin" +#[doc(hidden)] +#[macro_export] +macro_rules! profile_function { + ($($arg: tt)*) => { + #[cfg(not(target_arch = "wasm32"))] + puffin::profile_function!($($arg)*); + }; +} + +/// Profiling macro for feature "puffin" +#[doc(hidden)] +#[macro_export] +macro_rules! profile_scope { + ($($arg: tt)*) => { + #[cfg(not(target_arch = "wasm32"))] + puffin::profile_scope!($($arg)*); + }; +} diff --git a/crates/re_viewer/src/stream_rrd_from_http.rs b/crates/re_log_encoding/src/stream_rrd_from_http.rs similarity index 83% rename from crates/re_viewer/src/stream_rrd_from_http.rs rename to crates/re_log_encoding/src/stream_rrd_from_http.rs index b0997d4494a5..007105f37f53 100644 --- a/crates/re_viewer/src/stream_rrd_from_http.rs +++ b/crates/re_log_encoding/src/stream_rrd_from_http.rs @@ -1,6 +1,6 @@ -pub fn stream_rrd_from_http_to_channel( - url: String, -) -> re_smart_channel::Receiver { +use re_log_types::LogMsg; + +pub fn stream_rrd_from_http_to_channel(url: String) -> re_smart_channel::Receiver { let (tx, rx) = re_smart_channel::smart_channel(re_smart_channel::Source::RrdHttpStream { url: url.clone(), }); @@ -13,7 +13,7 @@ pub fn stream_rrd_from_http_to_channel( rx } -pub fn stream_rrd_from_http(url: String, on_msg: Box) { +pub fn stream_rrd_from_http(url: String, on_msg: Box) { re_log::debug!("Downloading .rrd file from {url:?}…"); // TODO(emilk): stream the http request, progressively decoding the .rrd file. @@ -38,8 +38,8 @@ pub fn stream_rrd_from_http(url: String, on_msg: Box, on_msg: Box) { - match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { +fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { + match crate::decoder::Decoder::new(rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { match msg { @@ -60,20 +60,19 @@ fn decode_rrd(rrd_bytes: Vec, on_msg: Box, on_msg: Box) { + use re_log_types::LogMsg; + + pub fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { wasm_bindgen_futures::spawn_local(decode_rrd_async(rrd_bytes, on_msg)); } /// Decodes the file in chunks, with an yield between each chunk. /// /// This is cooperative multi-tasking. - async fn decode_rrd_async( - rrd_bytes: Vec, - on_msg: Box, - ) { + async fn decode_rrd_async(rrd_bytes: Vec, on_msg: Box) { let mut last_yield = instant::Instant::now(); - match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { + match crate::decoder::Decoder::new(rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { match msg { diff --git a/crates/re_log_types/Cargo.toml b/crates/re_log_types/Cargo.toml index 12ae7e9e5463..4c517b9de34f 100644 --- a/crates/re_log_types/Cargo.toml +++ b/crates/re_log_types/Cargo.toml @@ -17,7 +17,7 @@ all-features = true [features] -default = ["arrow_datagen", "anyhow"] +default = ["arrow_datagen"] ## Enables the `datagen` module, which exposes a number of tools for generating random data for ## tests and benchmarks. @@ -32,12 +32,6 @@ glam = ["dep:glam", "dep:macaw"] ## Integration with the [`image`](https://crates.io/crates/image/) crate. image = ["dep:image"] -## Enable loading data from a file. -load = ["anyhow", "rmp-serde", "serde", "zstd", "ruzstd"] - -## Enable saving data to a file. -save = ["anyhow", "rmp-serde", "serde", "zstd"] - ## Enable (de)serialization using serde. serde = [ "dep:serde", @@ -51,11 +45,10 @@ serde = [ [dependencies] # Rerun -re_build_info.workspace = true re_format.workspace = true re_log.workspace = true re_string_interner.workspace = true -re_tuid.workspace = true +re_tuid = { workspace = true, features = ["arrow2_convert"] } # External ahash.workspace = true @@ -75,7 +68,7 @@ lazy_static.workspace = true ndarray.workspace = true nohash-hasher = "0.2" num-derive = "0.3" -num-traits = "0.2" +num-traits = "0.2" # used by num-derive smallvec = "1.10" thiserror.workspace = true time = { workspace = true, default-features = false, features = [ @@ -87,7 +80,6 @@ uuid = { version = "1.1", features = ["serde", "v4", "js"] } # Optional dependencies: -anyhow = { workspace = true, optional = true } ecolor = { workspace = true, optional = true } glam = { workspace = true, optional = true } image = { workspace = true, optional = true, default-features = false, features = [ @@ -95,32 +87,13 @@ image = { workspace = true, optional = true, default-features = false, features ] } macaw = { workspace = true, optional = true } rand = { version = "0.8", optional = true } -rmp-serde = { version = "1", optional = true } serde = { version = "1", optional = true, features = ["derive", "rc"] } serde_bytes = { version = "0.11", optional = true } # Native dependencies: [target.'cfg(not(target_arch = "wasm32"))'.dependencies] puffin.workspace = true -zstd = { version = "0.11.0", optional = true } # native only -# Web dependencies: -[target.'cfg(target_arch = "wasm32")'.dependencies] -ruzstd = { version = "0.3.0", optional = true } # works on wasm [dev-dependencies] -criterion = "0.4" -mimalloc.workspace = true -serde_test = { version = "1" } -arrow2 = { workspace = true, features = [ - "io_ipc", - "io_print", - "compute_concatenate", -] } - -[lib] -bench = false - -[[bench]] -name = "msg_encode_benchmark" -harness = false +rmp-serde = "1.1" diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index aa3c608b7e58..bf258e417ad1 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -4,9 +4,6 @@ #![doc = document_features::document_features!()] //! -#[cfg(any(feature = "save", feature = "load"))] -pub mod encoding; - #[cfg(feature = "arrow_datagen")] pub mod datagen; @@ -162,9 +159,8 @@ impl std::fmt::Display for ApplicationId { /// The most general log message sent from the SDK to the server. #[must_use] -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] // `PartialEq` used for tests in another crate #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -#[cfg_attr(test, derive(PartialEq))] #[allow(clippy::large_enum_variant)] pub enum LogMsg { /// A new recording has begun. diff --git a/crates/re_query/Cargo.toml b/crates/re_query/Cargo.toml index 4a28c1fd0f54..b09471633c7e 100644 --- a/crates/re_query/Cargo.toml +++ b/crates/re_query/Cargo.toml @@ -29,7 +29,6 @@ re_arrow_store.workspace = true re_data_store.workspace = true re_format.workspace = true re_log_types.workspace = true -re_log.workspace = true # External dependencies: arrow2 = { workspace = true, features = [ diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index 1eb913bfcf30..018bc298a43b 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -33,17 +33,13 @@ image = ["re_log_types/image"] [dependencies] -re_build_info.workspace = true -re_error.workspace = true -re_log_types = { workspace = true, features = ["save"] } +re_log_encoding = { workspace = true, features = ["encoder"] } +re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_sdk_comms = { workspace = true, features = ["client"] } -re_smart_channel.workspace = true -anyhow.workspace = true document-features = "0.2" -nohash-hasher = "0.2" parking_lot.workspace = true thiserror.workspace = true diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index 03d329a6435e..f78991944b90 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -9,9 +9,6 @@ // ---------------- // Private modules: -#[cfg(not(target_arch = "wasm32"))] -mod file_sink; - #[cfg(feature = "global_session")] mod global; @@ -34,6 +31,13 @@ pub use re_log_types::{ ApplicationId, Component, ComponentName, EntityPath, RecordingId, SerializableComponent, }; +#[cfg(not(target_arch = "wasm32"))] +impl crate::sink::LogSink for re_log_encoding::FileSink { + fn send(&self, msg: re_log_types::LogMsg) { + re_log_encoding::FileSink::send(self, msg); + } +} + // --------------- // Public modules: @@ -48,7 +52,7 @@ pub mod sink { pub use crate::log_sink::{disabled, BufferedSink, LogSink, TcpSink}; #[cfg(not(target_arch = "wasm32"))] - pub use crate::file_sink::{FileSink, FileSinkError}; + pub use re_log_encoding::{FileSink, FileSinkError}; } /// Things directly related to logging. diff --git a/crates/re_sdk/src/session.rs b/crates/re_sdk/src/session.rs index bbf99ee93451..94d336b30fcc 100644 --- a/crates/re_sdk/src/session.rs +++ b/crates/re_sdk/src/session.rs @@ -133,7 +133,7 @@ impl SessionBuilder { pub fn save( self, path: impl Into, - ) -> Result { + ) -> Result { let (rerun_enabled, recording_info) = self.finalize(); if rerun_enabled { Ok(Session::new( diff --git a/crates/re_tuid/Cargo.toml b/crates/re_tuid/Cargo.toml index 4f8e66c70dd2..71312ce7becd 100644 --- a/crates/re_tuid/Cargo.toml +++ b/crates/re_tuid/Cargo.toml @@ -19,17 +19,20 @@ all-features = true [features] default = [] +## Enable converting Tuid to arrow2 +arrow2_convert = ["dep:arrow2", "dep:arrow2_convert"] + ## Enable (de)serialization using serde. serde = ["dep:serde"] [dependencies] -arrow2_convert.workspace = true -arrow2.workspace = true document-features = "0.2" once_cell = "1.16" # Optional dependencies: +arrow2 = { workspace = true, optional = true } # used by arrow2_convert +arrow2_convert = { workspace = true, optional = true } serde = { version = "1", features = ["derive"], optional = true } # native dependencies: diff --git a/crates/re_tuid/src/lib.rs b/crates/re_tuid/src/lib.rs index 072b65261da5..33b8d85625c2 100644 --- a/crates/re_tuid/src/lib.rs +++ b/crates/re_tuid/src/lib.rs @@ -6,10 +6,11 @@ #![doc = document_features::document_features!()] //! -use arrow2::datatypes::DataType; -use arrow2_convert::{ArrowDeserialize, ArrowSerialize}; - -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, ArrowSerialize, ArrowDeserialize)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + feature = "arrow2_convert", + derive(arrow2_convert::ArrowSerialize, arrow2_convert::ArrowDeserialize) +)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct Tuid { /// Approximate nanoseconds since epoch. @@ -20,9 +21,11 @@ pub struct Tuid { inc: u64, } +#[cfg(feature = "arrow2_convert")] arrow2_convert::arrow_enable_vec_for_type!(Tuid); // TODO(#1774): shouldn't have to write this manually +#[cfg(feature = "arrow2_convert")] impl arrow2_convert::field::ArrowField for Tuid { type Type = Self; @@ -31,7 +34,7 @@ impl arrow2_convert::field::ArrowField for Tuid { ::field("time_ns"), ::field("inc"), ]))); - DataType::Extension("rerun.tuid".into(), Box::new(datatype), None) + arrow2::datatypes::DataType::Extension("rerun.tuid".into(), Box::new(datatype), None) } } diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index a929387a0603..bb849af2fde5 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -41,19 +41,13 @@ re_build_info.workspace = true re_data_store = { workspace = true, features = ["serde"] } re_error.workspace = true re_format.workspace = true +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } +re_log_types = { workspace = true, features = ["ecolor", "glam", "image"] } re_log.workspace = true -re_log_types = { workspace = true, features = [ - "ecolor", - "glam", - "image", - "save", - "load", -] } re_memory.workspace = true re_query.workspace = true re_renderer = { workspace = true, features = ["arrow", "serde"] } re_smart_channel.workspace = true -re_string_interner.workspace = true re_tensor_ops.workspace = true re_ui.workspace = true re_ws_comms = { workspace = true, features = ["client"] } @@ -76,7 +70,6 @@ egui = { workspace = true, features = ["extra_debug_asserts", "tracing"] } egui_dock = { workspace = true, features = ["serde"] } egui_extras = { workspace = true, features = ["tracing"] } egui-wgpu.workspace = true -ehttp = "0.2" enumset.workspace = true glam = { workspace = true, features = [ "mint", @@ -124,12 +117,6 @@ winapi = "0.3.9" [target.'cfg(target_arch = "wasm32")'.dependencies] console_error_panic_hook = "0.1.6" wasm-bindgen-futures = "0.4" -js-sys = "0.3" -wasm-bindgen = "0.2" - -[dependencies.web-sys] -version = "0.3.52" -features = ["Window"] [build-dependencies] diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index 04dd0127d173..ad5459df9dc5 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -1796,7 +1796,7 @@ fn save_database_to_file( let file = std::fs::File::create(path.as_path()) .with_context(|| format!("Failed to create file at {path:?}"))?; - re_log_types::encoding::encode(msgs.iter(), file) + re_log_encoding::encoder::encode(msgs.iter(), file) .map(|_| path) .context("Message encode") } @@ -1806,7 +1806,7 @@ fn save_database_to_file( fn load_rrd_to_log_db(mut read: impl std::io::Read) -> anyhow::Result { crate::profile_function!(); - let decoder = re_log_types::encoding::Decoder::new(read)?; + let decoder = re_log_encoding::decoder::Decoder::new(read)?; let mut log_db = LogDb::default(); for msg in decoder { diff --git a/crates/re_viewer/src/lib.rs b/crates/re_viewer/src/lib.rs index a248d82f521e..d9942fe1928c 100644 --- a/crates/re_viewer/src/lib.rs +++ b/crates/re_viewer/src/lib.rs @@ -8,7 +8,6 @@ pub mod env_vars; pub mod math; mod misc; mod remote_viewer_app; -pub mod stream_rrd_from_http; mod ui; mod viewer_analytics; diff --git a/crates/re_viewer/src/web.rs b/crates/re_viewer/src/web.rs index eb879b32f3f7..4bf72943a05e 100644 --- a/crates/re_viewer/src/web.rs +++ b/crates/re_viewer/src/web.rs @@ -50,7 +50,7 @@ pub async fn start( url: url.clone(), }); let egui_ctx = cc.egui_ctx.clone(); - crate::stream_rrd_from_http::stream_rrd_from_http( + re_log_encoding::stream_rrd_from_http::stream_rrd_from_http( url, Box::new(move |msg| { egui_ctx.request_repaint(); // wake up ui thread diff --git a/crates/rerun/Cargo.toml b/crates/rerun/Cargo.toml index f3a14600869a..643ca8b25deb 100644 --- a/crates/rerun/Cargo.toml +++ b/crates/rerun/Cargo.toml @@ -57,7 +57,6 @@ sdk = ["dep:re_sdk"] # You also need to install some additional tools, which you can do by running # [`scripts/setup_web.sh`](https://github.com/rerun-io/rerun/blob/main/scripts/setup_web.sh). web_viewer = [ - "dep:once_cell", "dep:re_web_viewer_server", "dep:webbrowser", "re_ws_comms/server", @@ -65,9 +64,9 @@ web_viewer = [ [dependencies] re_build_info.workspace = true -re_error.workspace = true re_format.workspace = true -re_log_types = { workspace = true, features = ["load"] } +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } +re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_smart_channel.workspace = true @@ -75,7 +74,6 @@ re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true document-features = "0.2" -egui = { workspace = true, default-features = false } itertools = { workspace = true } parking_lot.workspace = true @@ -86,7 +84,6 @@ re_sdk_comms = { workspace = true, optional = true } re_viewer = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } -once_cell = { version = "1.17", optional = true } webbrowser = { version = "0.8", optional = true } # Native dependencies: diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index 1dfa9ee9ff59..495fd680eb7f 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -263,7 +263,7 @@ async fn run_impl( let rx = if let Some(url_or_path) = args.url_or_path.clone() { match categorize_argument(url_or_path) { ArgumentCategory::RrdHttpUrl(url) => { - re_viewer::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) + re_log_encoding::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) } ArgumentCategory::RrdFilePath(path) => { re_log::info!("Loading {path:?}…"); @@ -281,6 +281,7 @@ async fn run_impl( } #[cfg(not(feature = "web_viewer"))] { + _ = (rerun_server_ws_url, shutdown_rx); panic!("Can't host web-viewer - rerun was not compiled with the 'web_viewer' feature"); } } else { @@ -295,7 +296,7 @@ async fn run_impl( #[cfg(not(feature = "native_viewer"))] { - _ = call_source; + _ = (call_source, rerun_server_ws_url); anyhow::bail!("Can't start viewer - rerun was compiled without the 'native_viewer' feature"); } } @@ -452,7 +453,7 @@ fn native_viewer_connect_to_ws_url( fn load_file_to_channel(path: &std::path::Path) -> anyhow::Result> { use anyhow::Context as _; let file = std::fs::File::open(path).context("Failed to open file")?; - let decoder = re_log_types::encoding::Decoder::new(file)?; + let decoder = re_log_encoding::decoder::Decoder::new(file)?; let (tx, rx) = re_smart_channel::smart_channel(re_smart_channel::Source::File { path: path.to_owned(), @@ -482,8 +483,8 @@ fn stream_to_rrd( rx: &re_smart_channel::Receiver, path: &std::path::PathBuf, shutdown_bool: &Arc, -) -> Result<(), re_sdk::sink::FileSinkError> { - use re_sdk::sink::FileSinkError; +) -> Result<(), re_log_encoding::FileSinkError> { + use re_log_encoding::FileSinkError; use re_smart_channel::RecvTimeoutError; if path.exists() { @@ -494,7 +495,7 @@ fn stream_to_rrd( let file = std::fs::File::create(path).map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; - let mut encoder = re_log_types::encoding::Encoder::new(file)?; + let mut encoder = re_log_encoding::encoder::Encoder::new(file)?; while !shutdown_bool.load(std::sync::atomic::Ordering::Relaxed) { // We wake up and poll shutdown_bool every now and then. diff --git a/examples/rust/objectron/Cargo.toml b/examples/rust/objectron/Cargo.toml index ee1fe349f747..e96c1b651a73 100644 --- a/examples/rust/objectron/Cargo.toml +++ b/examples/rust/objectron/Cargo.toml @@ -13,8 +13,6 @@ rerun = { workspace = true, features = ["web_viewer"] } anyhow.workspace = true clap = { workspace = true, features = ["derive"] } glam.workspace = true -image = { workspace = true, default-features = false, features = ["jpeg"] } -itertools = { workspace = true } prost = "0.11" diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 50fa0ada3d14..a7b89a43f450 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -44,21 +44,15 @@ re_error.workspace = true re_log.workspace = true re_log_types.workspace = true re_memory.workspace = true -re_tensor_ops.workspace = true rerun = { workspace = true, default-features = false, features = [ "analytics", "server", "sdk", ] } -ahash.workspace = true -anyhow.workspace = true arrow2 = { workspace = true, features = ["io_ipc", "io_print"] } -bytemuck = { version = "1.11", features = ["extern_crate_alloc"] } -crossbeam = "0.8" document-features = "0.2" glam.workspace = true -half.workspace = true image = { workspace = true, default-features = false, features = ["jpeg"] } itertools = { workspace = true } macaw.workspace = true diff --git a/scripts/publish_crates.sh b/scripts/publish_crates.sh index 6c5029c34a55..1c52ca6cc140 100755 --- a/scripts/publish_crates.sh +++ b/scripts/publish_crates.sh @@ -102,6 +102,7 @@ cargo publish $FLAGS -p re_memory cargo publish $FLAGS -p re_tuid cargo publish $FLAGS -p re_log_types cargo publish $FLAGS -p re_smart_channel +cargo publish $FLAGS -p re_log_encoding cargo publish $FLAGS -p re_tensor_ops cargo publish $FLAGS -p re_ui cargo publish $FLAGS -p re_arrow_store