From 02c0cdc1137fe8055de419297114eadbac019805 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 08:56:49 +0200 Subject: [PATCH 01/15] CI: Check `rerun` with --no-default features and/or with --features sdk --- .github/workflows/rust.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9821e24ad139..556f02977008 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -137,18 +137,20 @@ jobs: command: cranky args: --all-targets --all-features -- --deny warnings - - name: Check no default features + # -------------------------------------------------------------------------------- + # Check a few important permutations of the feature flags for our `rerun` library: + - name: Check rerun with `--no-default-features`` uses: actions-rs/cargo@v1 with: - command: check - args: --locked --no-default-features --features __ci --lib + command: cranky + args: --locked -p rerun --no-default-features - # Check a few important permutations of the feature flags for our `rerun` library: - - name: Check rerun with --features sdk + - name: Check rerun with `--features sdk` uses: actions-rs/cargo@v1 with: - command: check - args: --locked --no-default-features --features sdk + command: cranky + args: --locked -p rerun --no-default-features --features sdk + # -------------------------------------------------------------------------------- - name: Test doc-tests uses: actions-rs/cargo@v1 From f44236d058d881777cd1aa877af5598a8cce4d9d Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:21:14 +0200 Subject: [PATCH 02/15] Create a new helper crate re_transport containing stream_rrd_from_http --- Cargo.lock | 18 +++++++-- Cargo.toml | 1 + crates/re_transport/Cargo.toml | 37 +++++++++++++++++++ crates/re_transport/README.md | 10 +++++ crates/re_transport/src/lib.rs | 1 + .../src/stream_rrd_from_http.rs | 17 ++++----- crates/re_viewer/Cargo.toml | 4 -- crates/re_viewer/src/lib.rs | 1 - crates/rerun/Cargo.toml | 1 + crates/rerun/src/run.rs | 2 +- scripts/publish_crates.sh | 1 + 11 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 crates/re_transport/Cargo.toml create mode 100644 crates/re_transport/README.md create mode 100644 crates/re_transport/src/lib.rs rename crates/{re_viewer => re_transport}/src/stream_rrd_from_http.rs (87%) diff --git a/Cargo.lock b/Cargo.lock index 6fe5c47177ba..6c6fc9205d91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4094,6 +4094,19 @@ dependencies = [ "serde", ] +[[package]] +name = "re_transport" +version = "0.4.0" +dependencies = [ + "ehttp", + "js-sys", + "re_log", + "re_log_types", + "re_smart_channel", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "re_tuid" version = "0.4.0" @@ -4140,14 +4153,12 @@ dependencies = [ "egui-wgpu", "egui_dock", "egui_extras", - "ehttp", "enumset", "glam", "half 2.2.1", "image", "instant", "itertools", - "js-sys", "lazy_static", "macaw", "ndarray", @@ -4181,8 +4192,6 @@ dependencies = [ "time 0.3.20", "uuid", "vec1", - "wasm-bindgen", - "wasm-bindgen-futures", "web-sys", "wgpu", "winapi", @@ -4302,6 +4311,7 @@ dependencies = [ "re_sdk", "re_sdk_comms", "re_smart_channel", + "re_transport", "re_viewer", "re_web_viewer_server", "re_ws_comms", diff --git a/Cargo.toml b/Cargo.toml index baa3b459875f..d729b16d0fb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ re_sdk_comms = { path = "crates/re_sdk_comms", version = "0.4.0" } re_smart_channel = { path = "crates/re_smart_channel", version = "0.4.0" } re_string_interner = { path = "crates/re_string_interner", version = "0.4.0" } re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.4.0" } +re_transport = { path = "crates/re_transport", version = "0.4.0" } re_tuid = { path = "crates/re_tuid", version = "0.4.0" } re_ui = { path = "crates/re_ui", version = "0.4.0" } re_viewer = { path = "crates/re_viewer", version = "0.4.0" } diff --git a/crates/re_transport/Cargo.toml b/crates/re_transport/Cargo.toml new file mode 100644 index 000000000000..bd80f0879b80 --- /dev/null +++ b/crates/re_transport/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "re_transport" +authors.workspace = true +description = "Helpers for encoding and transporting Rerun log messages" +edition.workspace = true +homepage.workspace = true +include.workspace = true +license.workspace = true +publish = true +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = true + + +[features] + + +[dependencies] + +# Rerun: +re_log.workspace = true +re_log_types.workspace = true +re_smart_channel.workspace = true + +# External: +ehttp = "0.2" + + +# Web dependencies: +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" diff --git a/crates/re_transport/README.md b/crates/re_transport/README.md new file mode 100644 index 000000000000..77a0702501b6 --- /dev/null +++ b/crates/re_transport/README.md @@ -0,0 +1,10 @@ +# re_transport + +Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. + +[![Latest version](https://img.shields.io/crates/v/re_transport.svg)](https://crates.io/crates/re_transport) +[![Documentation](https://docs.rs/re_transport/badge.svg)](https://docs.rs/re_transport) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +Helper library for transporting Rerun log messages. diff --git a/crates/re_transport/src/lib.rs b/crates/re_transport/src/lib.rs new file mode 100644 index 000000000000..42e32de43a23 --- /dev/null +++ b/crates/re_transport/src/lib.rs @@ -0,0 +1 @@ +pub mod stream_rrd_from_http; diff --git a/crates/re_viewer/src/stream_rrd_from_http.rs b/crates/re_transport/src/stream_rrd_from_http.rs similarity index 87% rename from crates/re_viewer/src/stream_rrd_from_http.rs rename to crates/re_transport/src/stream_rrd_from_http.rs index b0997d4494a5..cf86f0cefcb4 100644 --- a/crates/re_viewer/src/stream_rrd_from_http.rs +++ b/crates/re_transport/src/stream_rrd_from_http.rs @@ -1,6 +1,6 @@ -pub fn stream_rrd_from_http_to_channel( - url: String, -) -> re_smart_channel::Receiver { +use re_log_types::LogMsg; + +pub fn stream_rrd_from_http_to_channel(url: String) -> re_smart_channel::Receiver { let (tx, rx) = re_smart_channel::smart_channel(re_smart_channel::Source::RrdHttpStream { url: url.clone(), }); @@ -13,7 +13,7 @@ pub fn stream_rrd_from_http_to_channel( rx } -pub fn stream_rrd_from_http(url: String, on_msg: Box) { +pub fn stream_rrd_from_http(url: String, on_msg: Box) { re_log::debug!("Downloading .rrd file from {url:?}…"); // TODO(emilk): stream the http request, progressively decoding the .rrd file. @@ -38,7 +38,7 @@ pub fn stream_rrd_from_http(url: String, on_msg: Box, on_msg: Box) { +fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { @@ -60,17 +60,14 @@ fn decode_rrd(rrd_bytes: Vec, on_msg: Box, on_msg: Box) { + pub fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { wasm_bindgen_futures::spawn_local(decode_rrd_async(rrd_bytes, on_msg)); } /// Decodes the file in chunks, with an yield between each chunk. /// /// This is cooperative multi-tasking. - async fn decode_rrd_async( - rrd_bytes: Vec, - on_msg: Box, - ) { + async fn decode_rrd_async(rrd_bytes: Vec, on_msg: Box) { let mut last_yield = instant::Instant::now(); match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index a929387a0603..5a863fd82c73 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -76,7 +76,6 @@ egui = { workspace = true, features = ["extra_debug_asserts", "tracing"] } egui_dock = { workspace = true, features = ["serde"] } egui_extras = { workspace = true, features = ["tracing"] } egui-wgpu.workspace = true -ehttp = "0.2" enumset.workspace = true glam = { workspace = true, features = [ "mint", @@ -123,9 +122,6 @@ winapi = "0.3.9" # web dependencies: [target.'cfg(target_arch = "wasm32")'.dependencies] console_error_panic_hook = "0.1.6" -wasm-bindgen-futures = "0.4" -js-sys = "0.3" -wasm-bindgen = "0.2" [dependencies.web-sys] version = "0.3.52" diff --git a/crates/re_viewer/src/lib.rs b/crates/re_viewer/src/lib.rs index a248d82f521e..d9942fe1928c 100644 --- a/crates/re_viewer/src/lib.rs +++ b/crates/re_viewer/src/lib.rs @@ -8,7 +8,6 @@ pub mod env_vars; pub mod math; mod misc; mod remote_viewer_app; -pub mod stream_rrd_from_http; mod ui; mod viewer_analytics; diff --git a/crates/rerun/Cargo.toml b/crates/rerun/Cargo.toml index f3a14600869a..014bcc029023 100644 --- a/crates/rerun/Cargo.toml +++ b/crates/rerun/Cargo.toml @@ -71,6 +71,7 @@ re_log_types = { workspace = true, features = ["load"] } re_log.workspace = true re_memory.workspace = true re_smart_channel.workspace = true +re_transport.workspace = true re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index 06afe4b7f7a0..1e29d6627271 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -263,7 +263,7 @@ async fn run_impl( let rx = if let Some(url_or_path) = args.url_or_path.clone() { match categorize_argument(url_or_path) { ArgumentCategory::RrdHttpUrl(url) => { - re_viewer::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) + re_transport::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) } ArgumentCategory::RrdFilePath(path) => { re_log::info!("Loading {path:?}…"); diff --git a/scripts/publish_crates.sh b/scripts/publish_crates.sh index 6c5029c34a55..a758e9a81047 100755 --- a/scripts/publish_crates.sh +++ b/scripts/publish_crates.sh @@ -102,6 +102,7 @@ cargo publish $FLAGS -p re_memory cargo publish $FLAGS -p re_tuid cargo publish $FLAGS -p re_log_types cargo publish $FLAGS -p re_smart_channel +cargo publish $FLAGS -p re_transport cargo publish $FLAGS -p re_tensor_ops cargo publish $FLAGS -p re_ui cargo publish $FLAGS -p re_arrow_store From 34255239d7aff1160d884a29c1243c0923bb22a2 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:23:19 +0200 Subject: [PATCH 03/15] Fix warnings --- crates/rerun/src/run.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index 1e29d6627271..87e71bb6015f 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -281,6 +281,7 @@ async fn run_impl( } #[cfg(not(feature = "web_viewer"))] { + _ = (rerun_server_ws_url, shutdown_rx); panic!("Can't host web-viewer - rerun was not compiled with the 'web_viewer' feature"); } } else { @@ -295,7 +296,7 @@ async fn run_impl( #[cfg(not(feature = "native_viewer"))] { - _ = call_source; + _ = (call_source, rerun_server_ws_url); anyhow::bail!("Can't start viewer - rerun was compiled without the 'native_viewer' feature"); } } From 72721bf51971119b5c6b3ea15cd520ea808f05bb Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:29:52 +0200 Subject: [PATCH 04/15] Move file sink to re_transport --- Cargo.lock | 3 +++ crates/re_sdk/Cargo.toml | 1 + crates/re_sdk/src/lib.rs | 12 ++++++++---- crates/re_sdk/src/session.rs | 2 +- crates/re_transport/Cargo.toml | 4 +++- crates/{re_sdk => re_transport}/src/file_sink.rs | 6 ++---- crates/re_transport/src/lib.rs | 8 ++++++++ crates/rerun/src/run.rs | 4 ++-- 8 files changed, 28 insertions(+), 12 deletions(-) rename crates/{re_sdk => re_transport}/src/file_sink.rs (95%) diff --git a/Cargo.lock b/Cargo.lock index 6c6fc9205d91..5a3177ef697f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4046,6 +4046,7 @@ dependencies = [ "re_memory", "re_sdk_comms", "re_smart_channel", + "re_transport", "thiserror", ] @@ -4100,9 +4101,11 @@ version = "0.4.0" dependencies = [ "ehttp", "js-sys", + "parking_lot 0.12.1", "re_log", "re_log_types", "re_smart_channel", + "thiserror", "wasm-bindgen", "wasm-bindgen-futures", ] diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index 1eb913bfcf30..d6f94ac624c3 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -40,6 +40,7 @@ re_log.workspace = true re_memory.workspace = true re_sdk_comms = { workspace = true, features = ["client"] } re_smart_channel.workspace = true +re_transport.workspace = true anyhow.workspace = true document-features = "0.2" diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index 03d329a6435e..2c0af4928a9e 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -9,9 +9,6 @@ // ---------------- // Private modules: -#[cfg(not(target_arch = "wasm32"))] -mod file_sink; - #[cfg(feature = "global_session")] mod global; @@ -34,6 +31,13 @@ pub use re_log_types::{ ApplicationId, Component, ComponentName, EntityPath, RecordingId, SerializableComponent, }; +#[cfg(not(target_arch = "wasm32"))] +impl crate::sink::LogSink for re_transport::FileSink { + fn send(&self, msg: re_log_types::LogMsg) { + re_transport::FileSink::send(self, msg); + } +} + // --------------- // Public modules: @@ -48,7 +52,7 @@ pub mod sink { pub use crate::log_sink::{disabled, BufferedSink, LogSink, TcpSink}; #[cfg(not(target_arch = "wasm32"))] - pub use crate::file_sink::{FileSink, FileSinkError}; + pub use re_transport::{FileSink, FileSinkError}; } /// Things directly related to logging. diff --git a/crates/re_sdk/src/session.rs b/crates/re_sdk/src/session.rs index ba2eb9a3b1fa..7b8dbd60b74f 100644 --- a/crates/re_sdk/src/session.rs +++ b/crates/re_sdk/src/session.rs @@ -133,7 +133,7 @@ impl SessionBuilder { pub fn save( self, path: impl Into, - ) -> Result { + ) -> Result { let (rerun_enabled, recording_info) = self.finalize(); if rerun_enabled { Ok(Session::new( diff --git a/crates/re_transport/Cargo.toml b/crates/re_transport/Cargo.toml index bd80f0879b80..45d272a94257 100644 --- a/crates/re_transport/Cargo.toml +++ b/crates/re_transport/Cargo.toml @@ -22,12 +22,14 @@ all-features = true [dependencies] # Rerun: +re_log_types = { workspace = true, features = ["save"] } re_log.workspace = true -re_log_types.workspace = true re_smart_channel.workspace = true # External: ehttp = "0.2" +parking_lot.workspace = true +thiserror.workspace = true # Web dependencies: diff --git a/crates/re_sdk/src/file_sink.rs b/crates/re_transport/src/file_sink.rs similarity index 95% rename from crates/re_sdk/src/file_sink.rs rename to crates/re_transport/src/file_sink.rs index f5f7e69f6fe1..41cc08aef40b 100644 --- a/crates/re_sdk/src/file_sink.rs +++ b/crates/re_transport/src/file_sink.rs @@ -71,10 +71,8 @@ impl FileSink { join_handle: Some(join_handle), }) } -} -impl crate::sink::LogSink for FileSink { - fn send(&self, msg: LogMsg) { - self.tx.lock().send(Some(msg)).ok(); + pub fn send(&self, log_msg: LogMsg) { + self.tx.lock().send(Some(log_msg)).ok(); } } diff --git a/crates/re_transport/src/lib.rs b/crates/re_transport/src/lib.rs index 42e32de43a23..8f80556949cd 100644 --- a/crates/re_transport/src/lib.rs +++ b/crates/re_transport/src/lib.rs @@ -1 +1,9 @@ +//! Crate that handles transporting of rerun log types. + pub mod stream_rrd_from_http; + +#[cfg(not(target_arch = "wasm32"))] +mod file_sink; + +#[cfg(not(target_arch = "wasm32"))] +pub use file_sink::{FileSink, FileSinkError}; diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index 87e71bb6015f..d6275ad4f0ee 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -483,9 +483,9 @@ fn stream_to_rrd( rx: &re_smart_channel::Receiver, path: &std::path::PathBuf, shutdown_bool: &Arc, -) -> Result<(), re_sdk::sink::FileSinkError> { - use re_sdk::sink::FileSinkError; +) -> Result<(), re_transport::FileSinkError> { use re_smart_channel::RecvTimeoutError; + use re_transport::FileSinkError; if path.exists() { re_log::warn!("Overwriting existing file at {path:?}"); From b34552503ba6436032b838e64503bfdfeb4d037d Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:33:12 +0200 Subject: [PATCH 05/15] wasm compilation fix --- crates/re_viewer/src/web.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_viewer/src/web.rs b/crates/re_viewer/src/web.rs index eb879b32f3f7..ab8cd63c9602 100644 --- a/crates/re_viewer/src/web.rs +++ b/crates/re_viewer/src/web.rs @@ -50,7 +50,7 @@ pub async fn start( url: url.clone(), }); let egui_ctx = cc.egui_ctx.clone(); - crate::stream_rrd_from_http::stream_rrd_from_http( + re_transport::stream_rrd_from_http::stream_rrd_from_http( url, Box::new(move |msg| { egui_ctx.request_repaint(); // wake up ui thread From 694d2b77d490e0bd39b2b8a112d6d0d781ea201f Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:54:17 +0200 Subject: [PATCH 06/15] Move LogMsg encoding/decoding into re_transport --- Cargo.lock | 16 ++- crates/re_data_store/Cargo.toml | 2 +- crates/re_data_store/examples/memory_usage.rs | 4 +- crates/re_log_types/Cargo.toml | 30 +---- crates/re_log_types/src/lib.rs | 3 - crates/re_sdk/Cargo.toml | 4 +- crates/re_transport/Cargo.toml | 30 ++++- .../benches/msg_encode_benchmark.rs | 8 +- .../src/decoder.rs} | 122 +----------------- crates/re_transport/src/encoder.rs | 100 ++++++++++++++ crates/re_transport/src/file_sink.rs | 4 +- crates/re_transport/src/lib.rs | 35 ++++- .../re_transport/src/stream_rrd_from_http.rs | 4 +- crates/re_viewer/Cargo.toml | 9 +- crates/re_viewer/src/app.rs | 4 +- crates/rerun/Cargo.toml | 4 +- crates/rerun/src/run.rs | 4 +- 17 files changed, 198 insertions(+), 185 deletions(-) rename crates/{re_log_types => re_transport}/benches/msg_encode_benchmark.rs (96%) rename crates/{re_log_types/src/encoding.rs => re_transport/src/decoder.rs} (62%) create mode 100644 crates/re_transport/src/encoder.rs diff --git a/Cargo.lock b/Cargo.lock index 5a3177ef697f..5baa91aa26c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3845,6 +3845,7 @@ dependencies = [ "re_log_types", "re_smart_channel", "re_string_interner", + "re_transport", "serde", "thiserror", ] @@ -3893,12 +3894,10 @@ name = "re_log_types" version = "0.4.0" dependencies = [ "ahash 0.8.2", - "anyhow", "array-init", "arrow2", "arrow2_convert", "bytemuck", - "criterion", "document-features", "ecolor", "fixed", @@ -3908,7 +3907,6 @@ dependencies = [ "itertools", "lazy_static", "macaw", - "mimalloc", "ndarray", "nohash-hasher", "num-derive", @@ -3921,16 +3919,13 @@ dependencies = [ "re_string_interner", "re_tuid", "rmp-serde", - "ruzstd", "serde", "serde_bytes", - "serde_test", "smallvec", "thiserror", "time 0.3.20", "typenum", "uuid", - "zstd", ] [[package]] @@ -4099,15 +4094,23 @@ dependencies = [ name = "re_transport" version = "0.4.0" dependencies = [ + "criterion", "ehttp", "js-sys", + "mimalloc", "parking_lot 0.12.1", + "puffin", + "re_build_info", "re_log", "re_log_types", "re_smart_channel", + "rmp-serde", + "ruzstd", + "serde_test", "thiserror", "wasm-bindgen", "wasm-bindgen-futures", + "zstd", ] [[package]] @@ -4185,6 +4188,7 @@ dependencies = [ "re_smart_channel", "re_string_interner", "re_tensor_ops", + "re_transport", "re_ui", "re_ws_comms", "rfd", diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index a77bdcb40096..1faecb6b0250 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -47,7 +47,7 @@ puffin.workspace = true criterion = "0.4" mimalloc.workspace = true rand = "0.8" -re_log_types = { workspace = true, features = ["load", "save"] } +re_transport = { workspace = true, features = ["decoder", "encoder"] } [lib] bench = false diff --git a/crates/re_data_store/examples/memory_usage.rs b/crates/re_data_store/examples/memory_usage.rs index 105d7e1a0014..1753ad3258bc 100644 --- a/crates/re_data_store/examples/memory_usage.rs +++ b/crates/re_data_store/examples/memory_usage.rs @@ -65,12 +65,12 @@ fn log_messages() { fn encode_log_msg(log_msg: &LogMsg) -> Vec { let mut bytes = vec![]; - re_log_types::encoding::encode(std::iter::once(log_msg), &mut bytes).unwrap(); + re_transport::encoder::encode(std::iter::once(log_msg), &mut bytes).unwrap(); bytes } fn decode_log_msg(mut bytes: &[u8]) -> LogMsg { - let mut messages = re_log_types::encoding::Decoder::new(&mut bytes) + let mut messages = re_transport::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_log_types/Cargo.toml b/crates/re_log_types/Cargo.toml index 12ae7e9e5463..317aa365932e 100644 --- a/crates/re_log_types/Cargo.toml +++ b/crates/re_log_types/Cargo.toml @@ -17,7 +17,7 @@ all-features = true [features] -default = ["arrow_datagen", "anyhow"] +default = ["arrow_datagen"] ## Enables the `datagen` module, which exposes a number of tools for generating random data for ## tests and benchmarks. @@ -32,12 +32,6 @@ glam = ["dep:glam", "dep:macaw"] ## Integration with the [`image`](https://crates.io/crates/image/) crate. image = ["dep:image"] -## Enable loading data from a file. -load = ["anyhow", "rmp-serde", "serde", "zstd", "ruzstd"] - -## Enable saving data to a file. -save = ["anyhow", "rmp-serde", "serde", "zstd"] - ## Enable (de)serialization using serde. serde = [ "dep:serde", @@ -87,7 +81,6 @@ uuid = { version = "1.1", features = ["serde", "v4", "js"] } # Optional dependencies: -anyhow = { workspace = true, optional = true } ecolor = { workspace = true, optional = true } glam = { workspace = true, optional = true } image = { workspace = true, optional = true, default-features = false, features = [ @@ -95,32 +88,13 @@ image = { workspace = true, optional = true, default-features = false, features ] } macaw = { workspace = true, optional = true } rand = { version = "0.8", optional = true } -rmp-serde = { version = "1", optional = true } serde = { version = "1", optional = true, features = ["derive", "rc"] } serde_bytes = { version = "0.11", optional = true } # Native dependencies: [target.'cfg(not(target_arch = "wasm32"))'.dependencies] puffin.workspace = true -zstd = { version = "0.11.0", optional = true } # native only -# Web dependencies: -[target.'cfg(target_arch = "wasm32")'.dependencies] -ruzstd = { version = "0.3.0", optional = true } # works on wasm [dev-dependencies] -criterion = "0.4" -mimalloc.workspace = true -serde_test = { version = "1" } -arrow2 = { workspace = true, features = [ - "io_ipc", - "io_print", - "compute_concatenate", -] } - -[lib] -bench = false - -[[bench]] -name = "msg_encode_benchmark" -harness = false +rmp-serde = "1.1" diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index 61b0aa817f67..d0cc910783b6 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -4,9 +4,6 @@ #![doc = document_features::document_features!()] //! -#[cfg(any(feature = "save", feature = "load"))] -pub mod encoding; - #[cfg(feature = "arrow_datagen")] pub mod datagen; diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index d6f94ac624c3..7c45db6f81ec 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -35,12 +35,12 @@ image = ["re_log_types/image"] [dependencies] re_build_info.workspace = true re_error.workspace = true -re_log_types = { workspace = true, features = ["save"] } +re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_sdk_comms = { workspace = true, features = ["client"] } re_smart_channel.workspace = true -re_transport.workspace = true +re_transport = { workspace = true, features = ["encoder"] } anyhow.workspace = true document-features = "0.2" diff --git a/crates/re_transport/Cargo.toml b/crates/re_transport/Cargo.toml index 45d272a94257..25474729e961 100644 --- a/crates/re_transport/Cargo.toml +++ b/crates/re_transport/Cargo.toml @@ -17,12 +17,20 @@ all-features = true [features] +default = [] + +## Enable loading data from an .rrd file. +decoder = ["dep:rmp-serde", "dep:zstd", "dep:ruzstd"] + +# Enable encoding of log messages to an .rrd file/stream: +encoder = ["dep:rmp-serde", "dep:zstd"] [dependencies] # Rerun: -re_log_types = { workspace = true, features = ["save"] } +re_build_info.workspace = true +re_log_types = { workspace = true, features = ["serde"] } re_log.workspace = true re_smart_channel.workspace = true @@ -31,9 +39,29 @@ ehttp = "0.2" parking_lot.workspace = true thiserror.workspace = true +# Optional external dependencies: +rmp-serde = { version = "1", optional = true } + +# Native dependencies: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +puffin.workspace = true +zstd = { version = "0.11.0", optional = true } # native only # Web dependencies: [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3" +ruzstd = { version = "0.3.0", optional = true } # works on wasm, in constrast to zstd wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" + +[dev-dependencies] +criterion = "0.4" +mimalloc.workspace = true +serde_test = { version = "1" } + +[lib] +bench = false + +[[bench]] +name = "msg_encode_benchmark" +harness = false diff --git a/crates/re_log_types/benches/msg_encode_benchmark.rs b/crates/re_transport/benches/msg_encode_benchmark.rs similarity index 96% rename from crates/re_log_types/benches/msg_encode_benchmark.rs rename to crates/re_transport/benches/msg_encode_benchmark.rs index d9131ef9f9f9..2c5b01185530 100644 --- a/crates/re_log_types/benches/msg_encode_benchmark.rs +++ b/crates/re_transport/benches/msg_encode_benchmark.rs @@ -1,5 +1,5 @@ -#[cfg(not(all(feature = "save", feature = "load")))] -compile_error!("msg_encode_benchmark requires 'save' and 'load' features."); +#[cfg(not(all(feature = "decoder", feature = "encoder")))] +compile_error!("msg_encode_benchmark requires 'decoder' and 'encoder' features."); #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; @@ -28,13 +28,13 @@ criterion_main!(benches); fn encode_log_msgs(messages: &[LogMsg]) -> Vec { let mut bytes = vec![]; - re_log_types::encoding::encode(messages.iter(), &mut bytes).unwrap(); + re_transport::encoder::encode(messages.iter(), &mut bytes).unwrap(); assert!(bytes.len() > messages.len()); bytes } fn decode_log_msgs(mut bytes: &[u8]) -> Vec { - let messages = re_log_types::encoding::Decoder::new(&mut bytes) + let messages = re_transport::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_log_types/src/encoding.rs b/crates/re_transport/src/decoder.rs similarity index 62% rename from crates/re_log_types/src/encoding.rs rename to crates/re_transport/src/decoder.rs index d1e2299c2509..2c23c55fc0dc 100644 --- a/crates/re_log_types/src/encoding.rs +++ b/crates/re_transport/src/decoder.rs @@ -1,117 +1,6 @@ -//! Encoding/decoding [`LogMsg`]:es as `.rrd` files. +//! Decoding [`LogMsg`]:es from `.rrd` files/streams. -use crate::LogMsg; - -// ---------------------------------------------------------------------------- -// native encode: - -#[cfg(feature = "save")] -#[cfg(not(target_arch = "wasm32"))] -mod encoder { - use std::io::Write as _; - - use crate::LogMsg; - - /// On failure to encode or serialize a [`LogMsg`]. - #[derive(thiserror::Error, Debug)] - pub enum EncodeError { - #[error("Failed to write: {0}")] - Write(std::io::Error), - - #[error("Zstd error: {0}")] - Zstd(std::io::Error), - - #[error("MsgPack error: {0}")] - MsgPack(#[from] rmp_serde::encode::Error), - - #[error("Called append on already finished encoder")] - AlreadyFinished, - } - - /// Encode a stream of [`LogMsg`] into an `.rrd` file. - pub struct Encoder { - /// Set to None when finished. - zstd_encoder: Option>, - buffer: Vec, - } - - impl Drop for Encoder { - fn drop(&mut self) { - if self.zstd_encoder.is_some() { - re_log::warn!("Encoder dropped without calling finish()!"); - if let Err(err) = self.finish() { - re_log::error!("Failed to finish encoding: {err}"); - } - } - } - } - - impl Encoder { - pub fn new(mut write: W) -> Result { - let rerun_version = re_build_info::CrateVersion::parse(env!("CARGO_PKG_VERSION")); - - write.write_all(b"RRF0").map_err(EncodeError::Write)?; - write - .write_all(&rerun_version.to_bytes()) - .map_err(EncodeError::Write)?; - - let level = 3; - let zstd_encoder = - zstd::stream::Encoder::new(write, level).map_err(EncodeError::Zstd)?; - - Ok(Self { - zstd_encoder: Some(zstd_encoder), - buffer: vec![], - }) - } - - pub fn append(&mut self, message: &LogMsg) -> Result<(), EncodeError> { - let Self { - zstd_encoder, - buffer, - } = self; - - if let Some(zstd_encoder) = zstd_encoder { - buffer.clear(); - rmp_serde::encode::write_named(buffer, message)?; - - zstd_encoder - .write_all(&(buffer.len() as u64).to_le_bytes()) - .map_err(EncodeError::Zstd)?; - zstd_encoder.write_all(buffer).map_err(EncodeError::Zstd)?; - - Ok(()) - } else { - Err(EncodeError::AlreadyFinished) - } - } - - pub fn finish(&mut self) -> Result<(), EncodeError> { - if let Some(zstd_encoder) = self.zstd_encoder.take() { - zstd_encoder.finish().map_err(EncodeError::Zstd)?; - Ok(()) - } else { - re_log::warn!("Encoder::finish called twice"); - Ok(()) - } - } - } - - pub fn encode<'a>( - messages: impl Iterator, - write: impl std::io::Write, - ) -> Result<(), EncodeError> { - let mut encoder = Encoder::new(write)?; - for message in messages { - encoder.append(message)?; - } - encoder.finish() - } -} - -#[cfg(feature = "save")] -#[cfg(not(target_arch = "wasm32"))] -pub use encoder::*; +use re_log_types::LogMsg; // ---------------------------------------------------------------------------- @@ -135,7 +24,6 @@ fn warn_on_version_mismatch(encoded_version: [u8; 4]) { // ---------------------------------------------------------------------------- /// On failure to encode or serialize a [`LogMsg`]. -#[cfg(feature = "load")] #[derive(thiserror::Error, Debug)] pub enum DecodeError { #[error("Not an .rrd file")] @@ -163,14 +51,12 @@ pub enum DecodeError { // ---------------------------------------------------------------------------- // native decode: -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] pub struct Decoder<'r, R: std::io::BufRead> { zdecoder: zstd::stream::Decoder<'r, R>, buffer: Vec, } -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] impl<'r, R: std::io::Read> Decoder<'r, std::io::BufReader> { pub fn new(mut read: R) -> Result { @@ -192,7 +78,6 @@ impl<'r, R: std::io::Read> Decoder<'r, std::io::BufReader> { } } -#[cfg(feature = "load")] #[cfg(not(target_arch = "wasm32"))] impl<'r, R: std::io::BufRead> Iterator for Decoder<'r, R> { type Item = Result; @@ -225,14 +110,12 @@ impl<'r, R: std::io::BufRead> Iterator for Decoder<'r, R> { // ---------------------------------------------------------------------------- // wasm decode: -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] pub struct Decoder { zdecoder: ruzstd::StreamingDecoder, buffer: Vec, } -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] impl Decoder { pub fn new(mut read: R) -> Result { @@ -254,7 +137,6 @@ impl Decoder { } } -#[cfg(feature = "load")] #[cfg(target_arch = "wasm32")] impl Iterator for Decoder { type Item = Result; diff --git a/crates/re_transport/src/encoder.rs b/crates/re_transport/src/encoder.rs new file mode 100644 index 000000000000..13cfbbdd849b --- /dev/null +++ b/crates/re_transport/src/encoder.rs @@ -0,0 +1,100 @@ +//! Encoding of [`LogMsg`]es as a binary stream, e.g. to store in an `.rrd` file, or send over network. + +use std::io::Write as _; + +use re_log_types::LogMsg; + +/// On failure to encode or serialize a [`LogMsg`]. +#[derive(thiserror::Error, Debug)] +pub enum EncodeError { + #[error("Failed to write: {0}")] + Write(std::io::Error), + + #[error("Zstd error: {0}")] + Zstd(std::io::Error), + + #[error("MsgPack error: {0}")] + MsgPack(#[from] rmp_serde::encode::Error), + + #[error("Called append on already finished encoder")] + AlreadyFinished, +} + +/// Encode a stream of [`LogMsg`] into an `.rrd` file. +pub struct Encoder { + /// Set to None when finished. + zstd_encoder: Option>, + buffer: Vec, +} + +impl Drop for Encoder { + fn drop(&mut self) { + if self.zstd_encoder.is_some() { + re_log::warn!("Encoder dropped without calling finish()!"); + if let Err(err) = self.finish() { + re_log::error!("Failed to finish encoding: {err}"); + } + } + } +} + +impl Encoder { + pub fn new(mut write: W) -> Result { + let rerun_version = re_build_info::CrateVersion::parse(env!("CARGO_PKG_VERSION")); + + write.write_all(b"RRF0").map_err(EncodeError::Write)?; + write + .write_all(&rerun_version.to_bytes()) + .map_err(EncodeError::Write)?; + + let level = 3; + let zstd_encoder = zstd::stream::Encoder::new(write, level).map_err(EncodeError::Zstd)?; + + Ok(Self { + zstd_encoder: Some(zstd_encoder), + buffer: vec![], + }) + } + + pub fn append(&mut self, message: &LogMsg) -> Result<(), EncodeError> { + let Self { + zstd_encoder, + buffer, + } = self; + + if let Some(zstd_encoder) = zstd_encoder { + buffer.clear(); + rmp_serde::encode::write_named(buffer, message)?; + + zstd_encoder + .write_all(&(buffer.len() as u64).to_le_bytes()) + .map_err(EncodeError::Zstd)?; + zstd_encoder.write_all(buffer).map_err(EncodeError::Zstd)?; + + Ok(()) + } else { + Err(EncodeError::AlreadyFinished) + } + } + + pub fn finish(&mut self) -> Result<(), EncodeError> { + if let Some(zstd_encoder) = self.zstd_encoder.take() { + zstd_encoder.finish().map_err(EncodeError::Zstd)?; + Ok(()) + } else { + re_log::warn!("Encoder::finish called twice"); + Ok(()) + } + } +} + +pub fn encode<'a>( + messages: impl Iterator, + write: impl std::io::Write, +) -> Result<(), EncodeError> { + let mut encoder = Encoder::new(write)?; + for message in messages { + encoder.append(message)?; + } + encoder.finish() +} diff --git a/crates/re_transport/src/file_sink.rs b/crates/re_transport/src/file_sink.rs index 41cc08aef40b..121383553eb2 100644 --- a/crates/re_transport/src/file_sink.rs +++ b/crates/re_transport/src/file_sink.rs @@ -17,7 +17,7 @@ pub enum FileSinkError { /// Error encoding a log message. #[error("Failed to encode LogMsg: {0}")] - LogMsgEncode(#[from] re_log_types::encoding::EncodeError), + LogMsgEncode(#[from] crate::encoder::EncodeError), } /// Stream log messages to an `.rrd` file. @@ -47,7 +47,7 @@ impl FileSink { let file = std::fs::File::create(&path) .map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; - let mut encoder = re_log_types::encoding::Encoder::new(file)?; + let mut encoder = crate::encoder::Encoder::new(file)?; let join_handle = std::thread::Builder::new() .name("file_writer".into()) diff --git a/crates/re_transport/src/lib.rs b/crates/re_transport/src/lib.rs index 8f80556949cd..c82642ca3e98 100644 --- a/crates/re_transport/src/lib.rs +++ b/crates/re_transport/src/lib.rs @@ -1,9 +1,42 @@ //! Crate that handles transporting of rerun log types. -pub mod stream_rrd_from_http; +#[cfg(feature = "decoder")] +pub mod decoder; +#[cfg(feature = "encoder")] +#[cfg(not(target_arch = "wasm32"))] // we do no yet support encoding LogMsgs in the browser +pub mod encoder; +#[cfg(feature = "encoder")] #[cfg(not(target_arch = "wasm32"))] mod file_sink; +#[cfg(feature = "decoder")] +pub mod stream_rrd_from_http; + +// --------------------------------------------------------------------- + +#[cfg(feature = "encoder")] #[cfg(not(target_arch = "wasm32"))] pub use file_sink::{FileSink, FileSinkError}; + +// --------------------------------------------------------------------------- + +/// Profiling macro for feature "puffin" +#[doc(hidden)] +#[macro_export] +macro_rules! profile_function { + ($($arg: tt)*) => { + #[cfg(not(target_arch = "wasm32"))] + puffin::profile_function!($($arg)*); + }; +} + +/// Profiling macro for feature "puffin" +#[doc(hidden)] +#[macro_export] +macro_rules! profile_scope { + ($($arg: tt)*) => { + #[cfg(not(target_arch = "wasm32"))] + puffin::profile_scope!($($arg)*); + }; +} diff --git a/crates/re_transport/src/stream_rrd_from_http.rs b/crates/re_transport/src/stream_rrd_from_http.rs index cf86f0cefcb4..c360a356137e 100644 --- a/crates/re_transport/src/stream_rrd_from_http.rs +++ b/crates/re_transport/src/stream_rrd_from_http.rs @@ -39,7 +39,7 @@ pub fn stream_rrd_from_http(url: String, on_msg: Box) { #[cfg(not(target_arch = "wasm32"))] #[allow(clippy::needless_pass_by_value)] // must match wasm version fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { - match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { + match crate::decoder::Decoder::new(rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { match msg { @@ -70,7 +70,7 @@ mod web_decode { async fn decode_rrd_async(rrd_bytes: Vec, on_msg: Box) { let mut last_yield = instant::Instant::now(); - match re_log_types::encoding::Decoder::new(rrd_bytes.as_slice()) { + match crate::decoder::Decoder::new(rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { match msg { diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index 5a863fd82c73..6ee8f3c65ec4 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -41,20 +41,15 @@ re_build_info.workspace = true re_data_store = { workspace = true, features = ["serde"] } re_error.workspace = true re_format.workspace = true +re_log_types = { workspace = true, features = ["ecolor", "glam", "image"] } re_log.workspace = true -re_log_types = { workspace = true, features = [ - "ecolor", - "glam", - "image", - "save", - "load", -] } re_memory.workspace = true re_query.workspace = true re_renderer = { workspace = true, features = ["arrow", "serde"] } re_smart_channel.workspace = true re_string_interner.workspace = true re_tensor_ops.workspace = true +re_transport = { workspace = true, features = ["decoder", "encoder"] } re_ui.workspace = true re_ws_comms = { workspace = true, features = ["client"] } diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index 56b00df98563..e668804dc1a2 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -1793,7 +1793,7 @@ fn save_database_to_file( let file = std::fs::File::create(path.as_path()) .with_context(|| format!("Failed to create file at {path:?}"))?; - re_log_types::encoding::encode(msgs.iter(), file) + re_transport::encoder::encode(msgs.iter(), file) .map(|_| path) .context("Message encode") } @@ -1803,7 +1803,7 @@ fn save_database_to_file( fn load_rrd_to_log_db(mut read: impl std::io::Read) -> anyhow::Result { crate::profile_function!(); - let decoder = re_log_types::encoding::Decoder::new(read)?; + let decoder = re_transport::decoder::Decoder::new(read)?; let mut log_db = LogDb::default(); for msg in decoder { diff --git a/crates/rerun/Cargo.toml b/crates/rerun/Cargo.toml index 014bcc029023..7e5d7b2eab7e 100644 --- a/crates/rerun/Cargo.toml +++ b/crates/rerun/Cargo.toml @@ -67,11 +67,11 @@ web_viewer = [ re_build_info.workspace = true re_error.workspace = true re_format.workspace = true -re_log_types = { workspace = true, features = ["load"] } +re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_smart_channel.workspace = true -re_transport.workspace = true +re_transport = { workspace = true, features = ["decoder", "encoder"] } re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index d6275ad4f0ee..0c04f4c0c017 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -453,7 +453,7 @@ fn native_viewer_connect_to_ws_url( fn load_file_to_channel(path: &std::path::Path) -> anyhow::Result> { use anyhow::Context as _; let file = std::fs::File::open(path).context("Failed to open file")?; - let decoder = re_log_types::encoding::Decoder::new(file)?; + let decoder = re_transport::decoder::Decoder::new(file)?; let (tx, rx) = re_smart_channel::smart_channel(re_smart_channel::Source::File { path: path.to_owned(), @@ -495,7 +495,7 @@ fn stream_to_rrd( let file = std::fs::File::create(path).map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; - let mut encoder = re_log_types::encoding::Encoder::new(file)?; + let mut encoder = re_transport::encoder::Encoder::new(file)?; while !shutdown_bool.load(std::sync::atomic::Ordering::Relaxed) { // We wake up and poll shutdown_bool every now and then. From e23fcdfedcd834fae1bf3bbc4cd1d7db5bfea9af Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 09:55:35 +0200 Subject: [PATCH 07/15] Fix typo --- crates/re_transport/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_transport/Cargo.toml b/crates/re_transport/Cargo.toml index 25474729e961..2160928f610b 100644 --- a/crates/re_transport/Cargo.toml +++ b/crates/re_transport/Cargo.toml @@ -50,7 +50,7 @@ zstd = { version = "0.11.0", optional = true } # native only # Web dependencies: [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3" -ruzstd = { version = "0.3.0", optional = true } # works on wasm, in constrast to zstd +ruzstd = { version = "0.3.0", optional = true } # works on wasm, in contrast to zstd wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" From 4c77625ae3b4d2220b2d0fa70dae9c9c7c0ed9ca Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 10:09:53 +0200 Subject: [PATCH 08/15] Fix web build --- Cargo.lock | 4 +++- crates/re_transport/Cargo.toml | 4 +++- crates/re_transport/src/stream_rrd_from_http.rs | 2 ++ crates/re_viewer/Cargo.toml | 5 +---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5baa91aa26c5..16136d4509af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4096,6 +4096,7 @@ version = "0.4.0" dependencies = [ "criterion", "ehttp", + "instant", "js-sys", "mimalloc", "parking_lot 0.12.1", @@ -4110,6 +4111,7 @@ dependencies = [ "thiserror", "wasm-bindgen", "wasm-bindgen-futures", + "web-sys", "zstd", ] @@ -4199,7 +4201,7 @@ dependencies = [ "time 0.3.20", "uuid", "vec1", - "web-sys", + "wasm-bindgen-futures", "wgpu", "winapi", ] diff --git a/crates/re_transport/Cargo.toml b/crates/re_transport/Cargo.toml index 2160928f610b..cd8e30f1b8f7 100644 --- a/crates/re_transport/Cargo.toml +++ b/crates/re_transport/Cargo.toml @@ -49,10 +49,12 @@ zstd = { version = "0.11.0", optional = true } # native only # Web dependencies: [target.'cfg(target_arch = "wasm32")'.dependencies] +instant = { version = "0.1", features = ["wasm-bindgen"] } js-sys = "0.3" -ruzstd = { version = "0.3.0", optional = true } # works on wasm, in contrast to zstd +ruzstd = { version = "0.3.0", optional = true } # works on wasm, in contrast to zstd wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3.52", features = ["Window"] } [dev-dependencies] criterion = "0.4" diff --git a/crates/re_transport/src/stream_rrd_from_http.rs b/crates/re_transport/src/stream_rrd_from_http.rs index c360a356137e..007105f37f53 100644 --- a/crates/re_transport/src/stream_rrd_from_http.rs +++ b/crates/re_transport/src/stream_rrd_from_http.rs @@ -60,6 +60,8 @@ fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { #[cfg(target_arch = "wasm32")] mod web_decode { + use re_log_types::LogMsg; + pub fn decode_rrd(rrd_bytes: Vec, on_msg: Box) { wasm_bindgen_futures::spawn_local(decode_rrd_async(rrd_bytes, on_msg)); } diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index 6ee8f3c65ec4..f09f8f8e3107 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -117,10 +117,7 @@ winapi = "0.3.9" # web dependencies: [target.'cfg(target_arch = "wasm32")'.dependencies] console_error_panic_hook = "0.1.6" - -[dependencies.web-sys] -version = "0.3.52" -features = ["Window"] +wasm-bindgen-futures = "0.4" [build-dependencies] From e89694abc67f318b76a1d556277a5948a49a66d3 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 10:10:19 +0200 Subject: [PATCH 09/15] Fix tests --- crates/re_data_store/Cargo.toml | 3 ++- crates/re_log_types/src/lib.rs | 3 +-- crates/re_transport/src/decoder.rs | 17 ++++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index 1faecb6b0250..005ea786ce7f 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -30,6 +30,7 @@ re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true re_string_interner.workspace = true +re_transport = { workspace = true, optional = true } ahash.workspace = true document-features = "0.2" @@ -55,4 +56,4 @@ bench = false [[example]] name = "memory_usage" path = "examples/memory_usage.rs" -required-features = ["re_log_types/load", "re_log_types/save"] +required-features = ["re_transport/decoder", "re_transport/encoder"] diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index d0cc910783b6..8abe5c677543 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -159,9 +159,8 @@ impl std::fmt::Display for ApplicationId { /// The most general log message sent from the SDK to the server. #[must_use] -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] // `PartialEq` used for tests #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -#[cfg_attr(test, derive(PartialEq))] #[allow(clippy::large_enum_variant)] pub enum LogMsg { /// A new recording has begun. diff --git a/crates/re_transport/src/decoder.rs b/crates/re_transport/src/decoder.rs index 2c23c55fc0dc..411c580d4439 100644 --- a/crates/re_transport/src/decoder.rs +++ b/crates/re_transport/src/decoder.rs @@ -168,19 +168,22 @@ impl Iterator for Decoder { // ---------------------------------------------------------------------------- -#[cfg(all(feature = "load", feature = "save"))] +#[cfg(all(feature = "decoder", feature = "encoder"))] #[test] fn test_encode_decode() { - use crate::{BeginRecordingMsg, LogMsg, MsgId, Time}; + use re_log_types::{ + ApplicationId, BeginRecordingMsg, LogMsg, MsgId, RecordingId, RecordingInfo, + RecordingSource, Time, + }; let messages = vec![LogMsg::BeginRecordingMsg(BeginRecordingMsg { msg_id: MsgId::random(), - info: crate::RecordingInfo { - application_id: crate::ApplicationId("test".to_owned()), - recording_id: crate::RecordingId::random(), + info: RecordingInfo { + application_id: ApplicationId("test".to_owned()), + recording_id: RecordingId::random(), is_official_example: true, started: Time::now(), - recording_source: crate::RecordingSource::RustSdk { + recording_source: RecordingSource::RustSdk { rustc_version: String::new(), llvm_version: String::new(), }, @@ -188,7 +191,7 @@ fn test_encode_decode() { })]; let mut file = vec![]; - encode(messages.iter(), &mut file).unwrap(); + crate::encoder::encode(messages.iter(), &mut file).unwrap(); let decoded_messages = Decoder::new(&mut file.as_slice()) .unwrap() From 11469b334b93e858464b9c8dea502335a17d8d4a Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 10:19:31 +0200 Subject: [PATCH 10/15] Remove a lot of unused dependencies with `cargo machete` --- Cargo.lock | 21 --------------------- crates/re_data_store/Cargo.toml | 1 - crates/re_log_types/Cargo.toml | 3 +-- crates/re_query/Cargo.toml | 1 - crates/re_sdk/Cargo.toml | 9 --------- crates/re_sdk/build.rs | 3 --- crates/re_tuid/Cargo.toml | 7 +++++-- crates/re_viewer/Cargo.toml | 1 - crates/rerun/Cargo.toml | 4 ---- examples/rust/objectron/Cargo.toml | 2 -- rerun_py/Cargo.toml | 6 ------ 11 files changed, 6 insertions(+), 52 deletions(-) delete mode 100644 crates/re_sdk/build.rs diff --git a/Cargo.lock b/Cargo.lock index 16136d4509af..b919124cc315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3129,8 +3129,6 @@ dependencies = [ "anyhow", "clap 4.1.4", "glam", - "image", - "itertools", "prost", "prost-build", "rerun", @@ -3844,7 +3842,6 @@ dependencies = [ "re_log", "re_log_types", "re_smart_channel", - "re_string_interner", "re_transport", "serde", "thiserror", @@ -3913,7 +3910,6 @@ dependencies = [ "num-traits", "puffin", "rand", - "re_build_info", "re_format", "re_log", "re_string_interner", @@ -3962,7 +3958,6 @@ dependencies = [ "re_arrow_store", "re_data_store", "re_format", - "re_log", "re_log_types", "thiserror", ] @@ -4024,23 +4019,17 @@ dependencies = [ name = "re_sdk" version = "0.4.0" dependencies = [ - "anyhow", "arrow2_convert", "document-features", "ndarray", "ndarray-rand", - "nohash-hasher", "once_cell", "parking_lot 0.12.1", "rand", - "re_build_build_info", - "re_build_info", - "re_error", "re_log", "re_log_types", "re_memory", "re_sdk_comms", - "re_smart_channel", "re_transport", "thiserror", ] @@ -4188,7 +4177,6 @@ dependencies = [ "re_query", "re_renderer", "re_smart_channel", - "re_string_interner", "re_tensor_ops", "re_transport", "re_ui", @@ -4303,16 +4291,13 @@ dependencies = [ "clap 4.1.4", "ctrlc", "document-features", - "egui", "itertools", "libc", "mimalloc", - "once_cell", "parking_lot 0.12.1", "re_analytics", "re_build_build_info", "re_build_info", - "re_error", "re_format", "re_log", "re_log_types", @@ -4332,14 +4317,9 @@ dependencies = [ name = "rerun_py" version = "0.4.0" dependencies = [ - "ahash 0.8.2", - "anyhow", "arrow2", - "bytemuck", - "crossbeam", "document-features", "glam", - "half 2.2.1", "image", "itertools", "macaw", @@ -4356,7 +4336,6 @@ dependencies = [ "re_log", "re_log_types", "re_memory", - "re_tensor_ops", "rerun", "tokio", "uuid", diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index 005ea786ce7f..451ee8104639 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -29,7 +29,6 @@ re_int_histogram.workspace = true re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true -re_string_interner.workspace = true re_transport = { workspace = true, optional = true } ahash.workspace = true diff --git a/crates/re_log_types/Cargo.toml b/crates/re_log_types/Cargo.toml index 317aa365932e..64205fa749e5 100644 --- a/crates/re_log_types/Cargo.toml +++ b/crates/re_log_types/Cargo.toml @@ -45,7 +45,6 @@ serde = [ [dependencies] # Rerun -re_build_info.workspace = true re_format.workspace = true re_log.workspace = true re_string_interner.workspace = true @@ -69,7 +68,7 @@ lazy_static.workspace = true ndarray.workspace = true nohash-hasher = "0.2" num-derive = "0.3" -num-traits = "0.2" +num-traits = "0.2" # used by num-derive smallvec = "1.10" thiserror.workspace = true time = { workspace = true, default-features = false, features = [ diff --git a/crates/re_query/Cargo.toml b/crates/re_query/Cargo.toml index 4a28c1fd0f54..b09471633c7e 100644 --- a/crates/re_query/Cargo.toml +++ b/crates/re_query/Cargo.toml @@ -29,7 +29,6 @@ re_arrow_store.workspace = true re_data_store.workspace = true re_format.workspace = true re_log_types.workspace = true -re_log.workspace = true # External dependencies: arrow2 = { workspace = true, features = [ diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index 7c45db6f81ec..33ac184cc3ec 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -33,18 +33,13 @@ image = ["re_log_types/image"] [dependencies] -re_build_info.workspace = true -re_error.workspace = true re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_sdk_comms = { workspace = true, features = ["client"] } -re_smart_channel.workspace = true re_transport = { workspace = true, features = ["encoder"] } -anyhow.workspace = true document-features = "0.2" -nohash-hasher = "0.2" parking_lot.workspace = true thiserror.workspace = true @@ -57,7 +52,3 @@ arrow2_convert.workspace = true ndarray.workspace = true ndarray-rand = "0.14" rand = "0.8" - - -[build-dependencies] -re_build_build_info.workspace = true diff --git a/crates/re_sdk/build.rs b/crates/re_sdk/build.rs deleted file mode 100644 index 3c87fda25b01..000000000000 --- a/crates/re_sdk/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - re_build_build_info::export_env_vars(); -} diff --git a/crates/re_tuid/Cargo.toml b/crates/re_tuid/Cargo.toml index 4f8e66c70dd2..6fe8973fa4cb 100644 --- a/crates/re_tuid/Cargo.toml +++ b/crates/re_tuid/Cargo.toml @@ -19,17 +19,20 @@ all-features = true [features] default = [] +## Enable converting Tuid to arrow2 +arrow_convert = ["dep:arrow2", "dep:arrow2_convert"] + ## Enable (de)serialization using serde. serde = ["dep:serde"] [dependencies] -arrow2_convert.workspace = true -arrow2.workspace = true document-features = "0.2" once_cell = "1.16" # Optional dependencies: +arrow2 = { workspace = true, optional = true } # used by arrow2_convert +arrow2_convert = { workspace = true, optional = true } serde = { version = "1", features = ["derive"], optional = true } # native dependencies: diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index f09f8f8e3107..9b23df12d511 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -47,7 +47,6 @@ re_memory.workspace = true re_query.workspace = true re_renderer = { workspace = true, features = ["arrow", "serde"] } re_smart_channel.workspace = true -re_string_interner.workspace = true re_tensor_ops.workspace = true re_transport = { workspace = true, features = ["decoder", "encoder"] } re_ui.workspace = true diff --git a/crates/rerun/Cargo.toml b/crates/rerun/Cargo.toml index 7e5d7b2eab7e..54e8f40a65eb 100644 --- a/crates/rerun/Cargo.toml +++ b/crates/rerun/Cargo.toml @@ -57,7 +57,6 @@ sdk = ["dep:re_sdk"] # You also need to install some additional tools, which you can do by running # [`scripts/setup_web.sh`](https://github.com/rerun-io/rerun/blob/main/scripts/setup_web.sh). web_viewer = [ - "dep:once_cell", "dep:re_web_viewer_server", "dep:webbrowser", "re_ws_comms/server", @@ -65,7 +64,6 @@ web_viewer = [ [dependencies] re_build_info.workspace = true -re_error.workspace = true re_format.workspace = true re_log_types.workspace = true re_log.workspace = true @@ -76,7 +74,6 @@ re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true document-features = "0.2" -egui = { workspace = true, default-features = false } itertools = { workspace = true } parking_lot.workspace = true @@ -87,7 +84,6 @@ re_sdk_comms = { workspace = true, optional = true } re_viewer = { workspace = true, optional = true } re_web_viewer_server = { workspace = true, optional = true } -once_cell = { version = "1.17", optional = true } webbrowser = { version = "0.8", optional = true } # Native dependencies: diff --git a/examples/rust/objectron/Cargo.toml b/examples/rust/objectron/Cargo.toml index ee1fe349f747..e96c1b651a73 100644 --- a/examples/rust/objectron/Cargo.toml +++ b/examples/rust/objectron/Cargo.toml @@ -13,8 +13,6 @@ rerun = { workspace = true, features = ["web_viewer"] } anyhow.workspace = true clap = { workspace = true, features = ["derive"] } glam.workspace = true -image = { workspace = true, default-features = false, features = ["jpeg"] } -itertools = { workspace = true } prost = "0.11" diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index 50fa0ada3d14..a7b89a43f450 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -44,21 +44,15 @@ re_error.workspace = true re_log.workspace = true re_log_types.workspace = true re_memory.workspace = true -re_tensor_ops.workspace = true rerun = { workspace = true, default-features = false, features = [ "analytics", "server", "sdk", ] } -ahash.workspace = true -anyhow.workspace = true arrow2 = { workspace = true, features = ["io_ipc", "io_print"] } -bytemuck = { version = "1.11", features = ["extern_crate_alloc"] } -crossbeam = "0.8" document-features = "0.2" glam.workspace = true -half.workspace = true image = { workspace = true, default-features = false, features = ["jpeg"] } itertools = { workspace = true } macaw.workspace = true From b6cf65ca4086cc21ed74a6f1868e9a18b9c33c1b Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 10:23:39 +0200 Subject: [PATCH 11/15] Build fix --- Cargo.lock | 1 + crates/re_log_types/Cargo.toml | 2 +- crates/re_sdk/Cargo.toml | 4 ++++ crates/re_sdk/build.rs | 3 +++ crates/re_tuid/Cargo.toml | 2 +- crates/re_tuid/src/lib.rs | 12 ++++++++---- 6 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 crates/re_sdk/build.rs diff --git a/Cargo.lock b/Cargo.lock index b919124cc315..a95645bee699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4026,6 +4026,7 @@ dependencies = [ "once_cell", "parking_lot 0.12.1", "rand", + "re_build_build_info", "re_log", "re_log_types", "re_memory", diff --git a/crates/re_log_types/Cargo.toml b/crates/re_log_types/Cargo.toml index 64205fa749e5..4c517b9de34f 100644 --- a/crates/re_log_types/Cargo.toml +++ b/crates/re_log_types/Cargo.toml @@ -48,7 +48,7 @@ serde = [ re_format.workspace = true re_log.workspace = true re_string_interner.workspace = true -re_tuid.workspace = true +re_tuid = { workspace = true, features = ["arrow2_convert"] } # External ahash.workspace = true diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index 33ac184cc3ec..b83182c7b99d 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -52,3 +52,7 @@ arrow2_convert.workspace = true ndarray.workspace = true ndarray-rand = "0.14" rand = "0.8" + + +[build-dependencies] +re_build_build_info.workspace = true diff --git a/crates/re_sdk/build.rs b/crates/re_sdk/build.rs new file mode 100644 index 000000000000..3c87fda25b01 --- /dev/null +++ b/crates/re_sdk/build.rs @@ -0,0 +1,3 @@ +fn main() { + re_build_build_info::export_env_vars(); +} diff --git a/crates/re_tuid/Cargo.toml b/crates/re_tuid/Cargo.toml index 6fe8973fa4cb..71312ce7becd 100644 --- a/crates/re_tuid/Cargo.toml +++ b/crates/re_tuid/Cargo.toml @@ -20,7 +20,7 @@ all-features = true default = [] ## Enable converting Tuid to arrow2 -arrow_convert = ["dep:arrow2", "dep:arrow2_convert"] +arrow2_convert = ["dep:arrow2", "dep:arrow2_convert"] ## Enable (de)serialization using serde. serde = ["dep:serde"] diff --git a/crates/re_tuid/src/lib.rs b/crates/re_tuid/src/lib.rs index 49c0840dad3d..e00275420033 100644 --- a/crates/re_tuid/src/lib.rs +++ b/crates/re_tuid/src/lib.rs @@ -6,10 +6,14 @@ #![doc = document_features::document_features!()] //! -use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; - -#[derive( - Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, ArrowField, ArrowSerialize, ArrowDeserialize, +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + feature = "arrow2_convert", + derive( + arrow2_convert::ArrowField, + arrow2_convert::ArrowSerialize, + arrow2_convert::ArrowDeserialize + ) )] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct Tuid { From 031e2f63f2a0288fe2e2f6dcb688be6d8225e158 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Wed, 5 Apr 2023 13:02:10 +0200 Subject: [PATCH 12/15] Clarify --- crates/re_log_types/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index 8abe5c677543..e7c45feba34c 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -159,7 +159,7 @@ impl std::fmt::Display for ApplicationId { /// The most general log message sent from the SDK to the server. #[must_use] -#[derive(Clone, Debug, PartialEq)] // `PartialEq` used for tests +#[derive(Clone, Debug, PartialEq)] // `PartialEq` used for tests in another crate #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] #[allow(clippy::large_enum_variant)] pub enum LogMsg { From c08b6e77a99b692db06134e969259b10fbd41f6e Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 6 Apr 2023 13:12:01 +0200 Subject: [PATCH 13/15] Rename the crate to re_log_encoding --- Cargo.lock | 58 +++++++++---------- Cargo.toml | 2 +- crates/re_data_store/Cargo.toml | 6 +- crates/re_data_store/examples/memory_usage.rs | 4 +- .../Cargo.toml | 2 +- .../README.md | 6 +- .../benches/msg_encode_benchmark.rs | 4 +- .../src/decoder.rs | 0 .../src/encoder.rs | 0 .../src/file_sink.rs | 0 .../src/lib.rs | 0 .../src/stream_rrd_from_http.rs | 0 crates/re_sdk/Cargo.toml | 2 +- crates/re_sdk/src/lib.rs | 6 +- crates/re_viewer/Cargo.toml | 2 +- crates/re_viewer/src/app.rs | 4 +- crates/re_viewer/src/web.rs | 2 +- crates/rerun/Cargo.toml | 2 +- crates/rerun/src/run.rs | 10 ++-- scripts/publish_crates.sh | 2 +- 20 files changed, 56 insertions(+), 56 deletions(-) rename crates/{re_transport => re_log_encoding}/Cargo.toml (98%) rename crates/{re_transport => re_log_encoding}/README.md (52%) rename crates/{re_transport => re_log_encoding}/benches/msg_encode_benchmark.rs (98%) rename crates/{re_transport => re_log_encoding}/src/decoder.rs (100%) rename crates/{re_transport => re_log_encoding}/src/encoder.rs (100%) rename crates/{re_transport => re_log_encoding}/src/file_sink.rs (100%) rename crates/{re_transport => re_log_encoding}/src/lib.rs (100%) rename crates/{re_transport => re_log_encoding}/src/stream_rrd_from_http.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index a95645bee699..4b1dd5917427 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3840,9 +3840,9 @@ dependencies = [ "re_arrow_store", "re_int_histogram", "re_log", + "re_log_encoding", "re_log_types", "re_smart_channel", - "re_transport", "serde", "thiserror", ] @@ -3886,6 +3886,31 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "re_log_encoding" +version = "0.4.0" +dependencies = [ + "criterion", + "ehttp", + "instant", + "js-sys", + "mimalloc", + "parking_lot 0.12.1", + "puffin", + "re_build_info", + "re_log", + "re_log_types", + "re_smart_channel", + "rmp-serde", + "ruzstd", + "serde_test", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "zstd", +] + [[package]] name = "re_log_types" version = "0.4.0" @@ -4028,10 +4053,10 @@ dependencies = [ "rand", "re_build_build_info", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_sdk_comms", - "re_transport", "thiserror", ] @@ -4080,31 +4105,6 @@ dependencies = [ "serde", ] -[[package]] -name = "re_transport" -version = "0.4.0" -dependencies = [ - "criterion", - "ehttp", - "instant", - "js-sys", - "mimalloc", - "parking_lot 0.12.1", - "puffin", - "re_build_info", - "re_log", - "re_log_types", - "re_smart_channel", - "rmp-serde", - "ruzstd", - "serde_test", - "thiserror", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "zstd", -] - [[package]] name = "re_tuid" version = "0.4.0" @@ -4173,13 +4173,13 @@ dependencies = [ "re_error", "re_format", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_query", "re_renderer", "re_smart_channel", "re_tensor_ops", - "re_transport", "re_ui", "re_ws_comms", "rfd", @@ -4301,12 +4301,12 @@ dependencies = [ "re_build_info", "re_format", "re_log", + "re_log_encoding", "re_log_types", "re_memory", "re_sdk", "re_sdk_comms", "re_smart_channel", - "re_transport", "re_viewer", "re_web_viewer_server", "re_ws_comms", diff --git a/Cargo.toml b/Cargo.toml index d729b16d0fb9..76c8577f1c6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ re_error = { path = "crates/re_error", version = "0.4.0" } re_format = { path = "crates/re_format", version = "0.4.0" } re_int_histogram = { path = "crates/re_int_histogram", version = "0.4.0" } re_log = { path = "crates/re_log", version = "0.4.0" } +re_log_encoding = { path = "crates/re_log_encoding", version = "0.4.0" } re_log_types = { path = "crates/re_log_types", version = "0.4.0" } re_memory = { path = "crates/re_memory", version = "0.4.0" } re_query = { path = "crates/re_query", version = "0.4.0" } @@ -42,7 +43,6 @@ re_sdk_comms = { path = "crates/re_sdk_comms", version = "0.4.0" } re_smart_channel = { path = "crates/re_smart_channel", version = "0.4.0" } re_string_interner = { path = "crates/re_string_interner", version = "0.4.0" } re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.4.0" } -re_transport = { path = "crates/re_transport", version = "0.4.0" } re_tuid = { path = "crates/re_tuid", version = "0.4.0" } re_ui = { path = "crates/re_ui", version = "0.4.0" } re_viewer = { path = "crates/re_viewer", version = "0.4.0" } diff --git a/crates/re_data_store/Cargo.toml b/crates/re_data_store/Cargo.toml index 451ee8104639..4b5991b37c0a 100644 --- a/crates/re_data_store/Cargo.toml +++ b/crates/re_data_store/Cargo.toml @@ -26,10 +26,10 @@ serde = ["dep:serde", "re_log_types/serde"] [dependencies] re_arrow_store.workspace = true re_int_histogram.workspace = true +re_log_encoding = { workspace = true, optional = true } re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true -re_transport = { workspace = true, optional = true } ahash.workspace = true document-features = "0.2" @@ -47,7 +47,7 @@ puffin.workspace = true criterion = "0.4" mimalloc.workspace = true rand = "0.8" -re_transport = { workspace = true, features = ["decoder", "encoder"] } +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } [lib] bench = false @@ -55,4 +55,4 @@ bench = false [[example]] name = "memory_usage" path = "examples/memory_usage.rs" -required-features = ["re_transport/decoder", "re_transport/encoder"] +required-features = ["re_log_encoding/decoder", "re_log_encoding/encoder"] diff --git a/crates/re_data_store/examples/memory_usage.rs b/crates/re_data_store/examples/memory_usage.rs index 1753ad3258bc..601b9152cdec 100644 --- a/crates/re_data_store/examples/memory_usage.rs +++ b/crates/re_data_store/examples/memory_usage.rs @@ -65,12 +65,12 @@ fn log_messages() { fn encode_log_msg(log_msg: &LogMsg) -> Vec { let mut bytes = vec![]; - re_transport::encoder::encode(std::iter::once(log_msg), &mut bytes).unwrap(); + re_log_encoding::encoder::encode(std::iter::once(log_msg), &mut bytes).unwrap(); bytes } fn decode_log_msg(mut bytes: &[u8]) -> LogMsg { - let mut messages = re_transport::decoder::Decoder::new(&mut bytes) + let mut messages = re_log_encoding::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_transport/Cargo.toml b/crates/re_log_encoding/Cargo.toml similarity index 98% rename from crates/re_transport/Cargo.toml rename to crates/re_log_encoding/Cargo.toml index cd8e30f1b8f7..b1c5c2943af6 100644 --- a/crates/re_transport/Cargo.toml +++ b/crates/re_log_encoding/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "re_transport" +name = "re_log_encoding" authors.workspace = true description = "Helpers for encoding and transporting Rerun log messages" edition.workspace = true diff --git a/crates/re_transport/README.md b/crates/re_log_encoding/README.md similarity index 52% rename from crates/re_transport/README.md rename to crates/re_log_encoding/README.md index 77a0702501b6..3f765f37456c 100644 --- a/crates/re_transport/README.md +++ b/crates/re_log_encoding/README.md @@ -1,9 +1,9 @@ -# re_transport +# re_log_encoding Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. -[![Latest version](https://img.shields.io/crates/v/re_transport.svg)](https://crates.io/crates/re_transport) -[![Documentation](https://docs.rs/re_transport/badge.svg)](https://docs.rs/re_transport) +[![Latest version](https://img.shields.io/crates/v/re_log_encoding.svg)](https://crates.io/crates/re_log_encoding) +[![Documentation](https://docs.rs/re_log_encoding/badge.svg)](https://docs.rs/re_log_encoding) ![MIT](https://img.shields.io/badge/license-MIT-blue.svg) ![Apache](https://img.shields.io/badge/license-Apache-blue.svg) diff --git a/crates/re_transport/benches/msg_encode_benchmark.rs b/crates/re_log_encoding/benches/msg_encode_benchmark.rs similarity index 98% rename from crates/re_transport/benches/msg_encode_benchmark.rs rename to crates/re_log_encoding/benches/msg_encode_benchmark.rs index 2c5b01185530..90a4b323feb2 100644 --- a/crates/re_transport/benches/msg_encode_benchmark.rs +++ b/crates/re_log_encoding/benches/msg_encode_benchmark.rs @@ -28,13 +28,13 @@ criterion_main!(benches); fn encode_log_msgs(messages: &[LogMsg]) -> Vec { let mut bytes = vec![]; - re_transport::encoder::encode(messages.iter(), &mut bytes).unwrap(); + re_log_encoding::encoder::encode(messages.iter(), &mut bytes).unwrap(); assert!(bytes.len() > messages.len()); bytes } fn decode_log_msgs(mut bytes: &[u8]) -> Vec { - let messages = re_transport::decoder::Decoder::new(&mut bytes) + let messages = re_log_encoding::decoder::Decoder::new(&mut bytes) .unwrap() .collect::, _>>() .unwrap(); diff --git a/crates/re_transport/src/decoder.rs b/crates/re_log_encoding/src/decoder.rs similarity index 100% rename from crates/re_transport/src/decoder.rs rename to crates/re_log_encoding/src/decoder.rs diff --git a/crates/re_transport/src/encoder.rs b/crates/re_log_encoding/src/encoder.rs similarity index 100% rename from crates/re_transport/src/encoder.rs rename to crates/re_log_encoding/src/encoder.rs diff --git a/crates/re_transport/src/file_sink.rs b/crates/re_log_encoding/src/file_sink.rs similarity index 100% rename from crates/re_transport/src/file_sink.rs rename to crates/re_log_encoding/src/file_sink.rs diff --git a/crates/re_transport/src/lib.rs b/crates/re_log_encoding/src/lib.rs similarity index 100% rename from crates/re_transport/src/lib.rs rename to crates/re_log_encoding/src/lib.rs diff --git a/crates/re_transport/src/stream_rrd_from_http.rs b/crates/re_log_encoding/src/stream_rrd_from_http.rs similarity index 100% rename from crates/re_transport/src/stream_rrd_from_http.rs rename to crates/re_log_encoding/src/stream_rrd_from_http.rs diff --git a/crates/re_sdk/Cargo.toml b/crates/re_sdk/Cargo.toml index b83182c7b99d..018bc298a43b 100644 --- a/crates/re_sdk/Cargo.toml +++ b/crates/re_sdk/Cargo.toml @@ -33,11 +33,11 @@ image = ["re_log_types/image"] [dependencies] +re_log_encoding = { workspace = true, features = ["encoder"] } re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_sdk_comms = { workspace = true, features = ["client"] } -re_transport = { workspace = true, features = ["encoder"] } document-features = "0.2" parking_lot.workspace = true diff --git a/crates/re_sdk/src/lib.rs b/crates/re_sdk/src/lib.rs index 2c0af4928a9e..f78991944b90 100644 --- a/crates/re_sdk/src/lib.rs +++ b/crates/re_sdk/src/lib.rs @@ -32,9 +32,9 @@ pub use re_log_types::{ }; #[cfg(not(target_arch = "wasm32"))] -impl crate::sink::LogSink for re_transport::FileSink { +impl crate::sink::LogSink for re_log_encoding::FileSink { fn send(&self, msg: re_log_types::LogMsg) { - re_transport::FileSink::send(self, msg); + re_log_encoding::FileSink::send(self, msg); } } @@ -52,7 +52,7 @@ pub mod sink { pub use crate::log_sink::{disabled, BufferedSink, LogSink, TcpSink}; #[cfg(not(target_arch = "wasm32"))] - pub use re_transport::{FileSink, FileSinkError}; + pub use re_log_encoding::{FileSink, FileSinkError}; } /// Things directly related to logging. diff --git a/crates/re_viewer/Cargo.toml b/crates/re_viewer/Cargo.toml index 9b23df12d511..bb849af2fde5 100644 --- a/crates/re_viewer/Cargo.toml +++ b/crates/re_viewer/Cargo.toml @@ -41,6 +41,7 @@ re_build_info.workspace = true re_data_store = { workspace = true, features = ["serde"] } re_error.workspace = true re_format.workspace = true +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } re_log_types = { workspace = true, features = ["ecolor", "glam", "image"] } re_log.workspace = true re_memory.workspace = true @@ -48,7 +49,6 @@ re_query.workspace = true re_renderer = { workspace = true, features = ["arrow", "serde"] } re_smart_channel.workspace = true re_tensor_ops.workspace = true -re_transport = { workspace = true, features = ["decoder", "encoder"] } re_ui.workspace = true re_ws_comms = { workspace = true, features = ["client"] } diff --git a/crates/re_viewer/src/app.rs b/crates/re_viewer/src/app.rs index e668804dc1a2..7f868f1d1692 100644 --- a/crates/re_viewer/src/app.rs +++ b/crates/re_viewer/src/app.rs @@ -1793,7 +1793,7 @@ fn save_database_to_file( let file = std::fs::File::create(path.as_path()) .with_context(|| format!("Failed to create file at {path:?}"))?; - re_transport::encoder::encode(msgs.iter(), file) + re_log_encoding::encoder::encode(msgs.iter(), file) .map(|_| path) .context("Message encode") } @@ -1803,7 +1803,7 @@ fn save_database_to_file( fn load_rrd_to_log_db(mut read: impl std::io::Read) -> anyhow::Result { crate::profile_function!(); - let decoder = re_transport::decoder::Decoder::new(read)?; + let decoder = re_log_encoding::decoder::Decoder::new(read)?; let mut log_db = LogDb::default(); for msg in decoder { diff --git a/crates/re_viewer/src/web.rs b/crates/re_viewer/src/web.rs index ab8cd63c9602..4bf72943a05e 100644 --- a/crates/re_viewer/src/web.rs +++ b/crates/re_viewer/src/web.rs @@ -50,7 +50,7 @@ pub async fn start( url: url.clone(), }); let egui_ctx = cc.egui_ctx.clone(); - re_transport::stream_rrd_from_http::stream_rrd_from_http( + re_log_encoding::stream_rrd_from_http::stream_rrd_from_http( url, Box::new(move |msg| { egui_ctx.request_repaint(); // wake up ui thread diff --git a/crates/rerun/Cargo.toml b/crates/rerun/Cargo.toml index 54e8f40a65eb..643ca8b25deb 100644 --- a/crates/rerun/Cargo.toml +++ b/crates/rerun/Cargo.toml @@ -65,11 +65,11 @@ web_viewer = [ [dependencies] re_build_info.workspace = true re_format.workspace = true +re_log_encoding = { workspace = true, features = ["decoder", "encoder"] } re_log_types.workspace = true re_log.workspace = true re_memory.workspace = true re_smart_channel.workspace = true -re_transport = { workspace = true, features = ["decoder", "encoder"] } re_ws_comms = { workspace = true, features = ["client"] } anyhow.workspace = true diff --git a/crates/rerun/src/run.rs b/crates/rerun/src/run.rs index 5aaf42cd57bb..495fd680eb7f 100644 --- a/crates/rerun/src/run.rs +++ b/crates/rerun/src/run.rs @@ -263,7 +263,7 @@ async fn run_impl( let rx = if let Some(url_or_path) = args.url_or_path.clone() { match categorize_argument(url_or_path) { ArgumentCategory::RrdHttpUrl(url) => { - re_transport::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) + re_log_encoding::stream_rrd_from_http::stream_rrd_from_http_to_channel(url) } ArgumentCategory::RrdFilePath(path) => { re_log::info!("Loading {path:?}…"); @@ -453,7 +453,7 @@ fn native_viewer_connect_to_ws_url( fn load_file_to_channel(path: &std::path::Path) -> anyhow::Result> { use anyhow::Context as _; let file = std::fs::File::open(path).context("Failed to open file")?; - let decoder = re_transport::decoder::Decoder::new(file)?; + let decoder = re_log_encoding::decoder::Decoder::new(file)?; let (tx, rx) = re_smart_channel::smart_channel(re_smart_channel::Source::File { path: path.to_owned(), @@ -483,9 +483,9 @@ fn stream_to_rrd( rx: &re_smart_channel::Receiver, path: &std::path::PathBuf, shutdown_bool: &Arc, -) -> Result<(), re_transport::FileSinkError> { +) -> Result<(), re_log_encoding::FileSinkError> { + use re_log_encoding::FileSinkError; use re_smart_channel::RecvTimeoutError; - use re_transport::FileSinkError; if path.exists() { re_log::warn!("Overwriting existing file at {path:?}"); @@ -495,7 +495,7 @@ fn stream_to_rrd( let file = std::fs::File::create(path).map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; - let mut encoder = re_transport::encoder::Encoder::new(file)?; + let mut encoder = re_log_encoding::encoder::Encoder::new(file)?; while !shutdown_bool.load(std::sync::atomic::Ordering::Relaxed) { // We wake up and poll shutdown_bool every now and then. diff --git a/scripts/publish_crates.sh b/scripts/publish_crates.sh index a758e9a81047..1c52ca6cc140 100755 --- a/scripts/publish_crates.sh +++ b/scripts/publish_crates.sh @@ -102,7 +102,7 @@ cargo publish $FLAGS -p re_memory cargo publish $FLAGS -p re_tuid cargo publish $FLAGS -p re_log_types cargo publish $FLAGS -p re_smart_channel -cargo publish $FLAGS -p re_transport +cargo publish $FLAGS -p re_log_encoding cargo publish $FLAGS -p re_tensor_ops cargo publish $FLAGS -p re_ui cargo publish $FLAGS -p re_arrow_store From e724b3fe30ad78440172fb18ba423e65090e464a Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 6 Apr 2023 17:44:27 +0200 Subject: [PATCH 14/15] better docstring Co-authored-by: Jeremy Leibs --- crates/re_log_encoding/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_log_encoding/src/lib.rs b/crates/re_log_encoding/src/lib.rs index c82642ca3e98..16b883448803 100644 --- a/crates/re_log_encoding/src/lib.rs +++ b/crates/re_log_encoding/src/lib.rs @@ -1,4 +1,4 @@ -//! Crate that handles transporting of rerun log types. +//! Crate that handles encoding of rerun log types. #[cfg(feature = "decoder")] pub mod decoder; From 854e21ca4ad2476d4b2c3f882a91c3798837f6ac Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Thu, 6 Apr 2023 17:44:35 +0200 Subject: [PATCH 15/15] better readme Co-authored-by: Jeremy Leibs --- crates/re_log_encoding/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_log_encoding/README.md b/crates/re_log_encoding/README.md index 3f765f37456c..8b3e7ff4f875 100644 --- a/crates/re_log_encoding/README.md +++ b/crates/re_log_encoding/README.md @@ -7,4 +7,4 @@ Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. ![MIT](https://img.shields.io/badge/license-MIT-blue.svg) ![Apache](https://img.shields.io/badge/license-Apache-blue.svg) -Helper library for transporting Rerun log messages. +Helper library for encoding Rerun log messages.