From 0786fdab0ad8043e8bc43205050ff9387eb10b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= Date: Thu, 12 Dec 2024 13:01:38 +0100 Subject: [PATCH] Encode `LogMsg` using protobuf (#8347) --- Cargo.lock | 2 + Cargo.toml | 1 + crates/build/re_protos_builder/Cargo.toml | 1 + .../src/bin/build_re_remote_store_types.rs | 12 +- crates/build/re_protos_builder/src/lib.rs | 5 +- crates/store/re_chunk_store/src/lib.rs | 4 +- .../src/protobuf_conversions.rs | 34 +- crates/store/re_chunk_store/src/store.rs | 4 +- .../re_data_loader/src/loader_external.rs | 2 +- crates/store/re_data_loader/src/loader_rrd.rs | 10 +- crates/store/re_data_source/src/load_stdin.rs | 2 +- crates/store/re_dataframe/Cargo.toml | 3 +- crates/store/re_dataframe/examples/query.rs | 3 +- crates/store/re_dataframe/src/engine.rs | 3 +- crates/store/re_dataframe/src/lib.rs | 3 +- .../re_entity_db/examples/memory_usage.rs | 4 +- crates/store/re_entity_db/src/store_bundle.rs | 2 +- crates/store/re_grpc_client/src/lib.rs | 3 +- .../benches/msg_encode_benchmark.rs | 219 +++++-- .../store/re_log_encoding/src/codec/arrow.rs | 96 +++ .../re_log_encoding/src/codec/file/decoder.rs | 58 ++ .../re_log_encoding/src/codec/file/encoder.rs | 59 ++ .../re_log_encoding/src/codec/file/mod.rs | 70 +++ crates/store/re_log_encoding/src/codec/mod.rs | 14 +- .../store/re_log_encoding/src/codec/wire.rs | 261 -------- .../re_log_encoding/src/codec/wire/decoder.rs | 55 ++ .../re_log_encoding/src/codec/wire/encoder.rs | 58 ++ .../re_log_encoding/src/codec/wire/mod.rs | 121 ++++ .../store/re_log_encoding/src/decoder/mod.rs | 329 +++++++---- .../re_log_encoding/src/decoder/stream.rs | 16 +- crates/store/re_log_encoding/src/encoder.rs | 117 ++-- crates/store/re_log_encoding/src/file_sink.rs | 4 +- crates/store/re_log_encoding/src/lib.rs | 27 +- .../src/protobuf_conversions.rs | 17 + .../src/stream_rrd_from_http.rs | 4 +- crates/store/re_log_types/src/lib.rs | 115 ++++ .../re_log_types/src/protobuf_conversions.rs | 559 +++++++++++++++++- crates/store/re_protos/.gitattributes | 2 +- .../re_protos/proto/rerun/v0/log_msg.proto | 190 ++++++ crates/store/re_protos/src/lib.rs | 46 +- .../store/re_protos/src/v0/rerun.common.v0.rs | 240 ++++++++ .../re_protos/src/v0/rerun.log_msg.v0.rs | 388 ++++++++++++ .../re_protos/src/v0/rerun.remote_store.v0.rs | 120 ++++ .../store/re_sdk_comms/src/buffered_client.rs | 2 +- crates/store/re_sdk_comms/src/server.rs | 2 +- crates/top/re_sdk/src/binary_stream_sink.rs | 2 +- crates/top/rerun/Cargo.toml | 8 +- crates/top/rerun/src/commands/entrypoint.rs | 2 +- crates/top/rerun/src/commands/rrd/compare.rs | 2 +- crates/top/rerun/src/commands/rrd/filter.rs | 4 +- .../rerun/src/commands/rrd/merge_compact.rs | 4 +- crates/top/rerun/src/commands/rrd/print.rs | 2 +- crates/top/rerun/src/commands/stdio.rs | 2 +- crates/top/rerun/src/lib.rs | 4 +- .../utils/re_tuid/src/protobuf_conversions.rs | 11 + crates/viewer/re_viewer/src/app.rs | 2 +- crates/viewer/re_viewer/src/loading.rs | 2 +- crates/viewer/re_viewer/src/saving.rs | 2 +- rerun_py/src/dataframe.rs | 3 +- 59 files changed, 2740 insertions(+), 597 deletions(-) create mode 100644 crates/store/re_log_encoding/src/codec/arrow.rs create mode 100644 crates/store/re_log_encoding/src/codec/file/decoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/file/encoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/file/mod.rs delete mode 100644 crates/store/re_log_encoding/src/codec/wire.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/decoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/encoder.rs create mode 100644 crates/store/re_log_encoding/src/codec/wire/mod.rs create mode 100644 crates/store/re_log_encoding/src/protobuf_conversions.rs create mode 100644 crates/store/re_protos/proto/rerun/v0/log_msg.proto create mode 100644 crates/store/re_protos/src/v0/rerun.log_msg.v0.rs diff --git a/Cargo.lock b/Cargo.lock index 98a0e10c66c5..fef05a09e3c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5816,6 +5816,7 @@ dependencies = [ "re_chunk", "re_chunk_store", "re_log", + "re_log_encoding", "re_log_types", "re_query", "re_tracing", @@ -6083,6 +6084,7 @@ name = "re_protos_builder" version = "0.21.0-alpha.1+dev" dependencies = [ "camino", + "prost-build", "re_log", "tonic-build", ] diff --git a/Cargo.toml b/Cargo.toml index 73d17d1b009f..f9093aa23cf5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -241,6 +241,7 @@ prettyplease = "0.2" proc-macro2 = { version = "1.0", default-features = false } profiling = { version = "1.0.12", default-features = false } prost = "0.13.3" +prost-build = "0.13.3" puffin = "0.19.1" puffin_http = "0.16" pyo3 = "0.22.5" diff --git a/crates/build/re_protos_builder/Cargo.toml b/crates/build/re_protos_builder/Cargo.toml index 87ab72d8efb8..6f100bcd9d49 100644 --- a/crates/build/re_protos_builder/Cargo.toml +++ b/crates/build/re_protos_builder/Cargo.toml @@ -29,3 +29,4 @@ camino.workspace = true tonic-build = { workspace = true, default-features = false, features = [ "prost", ] } +prost-build = { workspace = true } diff --git a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs index cbb6b8fea943..9ed06af18e62 100644 --- a/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs +++ b/crates/build/re_protos_builder/src/bin/build_re_remote_store_types.rs @@ -7,9 +7,9 @@ use camino::Utf8Path; -const PROTOBUF_DEFINITIONS_DIR_PATH: &str = "crates/store/re_protos/proto"; -const PROTOBUF_REMOTE_STORE_V0_RELATIVE_PATH: &str = "rerun/v0/remote_store.proto"; -const RUST_V0_OUTPUT_DIR_PATH: &str = "crates/store/re_protos/src/v0"; +const PROTOS_DIR: &str = "crates/store/re_protos/proto"; +const INPUT_V0: &[&str] = &["rerun/v0/remote_store.proto", "rerun/v0/log_msg.proto"]; +const OUTPUT_V0_RUST_DIR: &str = "crates/store/re_protos/src/v0"; fn main() { re_log::setup_logging(); @@ -26,8 +26,8 @@ fn main() { "failed to find workspace root" ); - let definitions_dir_path = workspace_dir.join(PROTOBUF_DEFINITIONS_DIR_PATH); - let rust_generated_output_dir_path = workspace_dir.join(RUST_V0_OUTPUT_DIR_PATH); + let definitions_dir_path = workspace_dir.join(PROTOS_DIR); + let rust_generated_output_dir_path = workspace_dir.join(OUTPUT_V0_RUST_DIR); re_log::info!( definitions=?definitions_dir_path, @@ -37,7 +37,7 @@ fn main() { re_protos_builder::generate_rust_code( definitions_dir_path, - &[PROTOBUF_REMOTE_STORE_V0_RELATIVE_PATH], + INPUT_V0, rust_generated_output_dir_path, ); } diff --git a/crates/build/re_protos_builder/src/lib.rs b/crates/build/re_protos_builder/src/lib.rs index a1d2f744e1a0..6f382e89fb48 100644 --- a/crates/build/re_protos_builder/src/lib.rs +++ b/crates/build/re_protos_builder/src/lib.rs @@ -16,12 +16,15 @@ pub fn generate_rust_code( proto_paths: &[impl AsRef], output_dir: impl AsRef, ) { + let mut prost_config = prost_build::Config::new(); + prost_config.enable_type_names(); // tonic doesn't expose this option + if let Err(err) = tonic_build::configure() .out_dir(output_dir.as_ref()) .build_client(true) .build_server(true) .build_transport(false) // Small convenience, but doesn't work on web - .compile_protos(proto_paths, &[definitions_dir]) + .compile_protos_with_config(prost_config, proto_paths, &[definitions_dir]) { match err.kind() { std::io::ErrorKind::Other => { diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs index 14921f45157c..645d6c7e9e84 100644 --- a/crates/store/re_chunk_store/src/lib.rs +++ b/crates/store/re_chunk_store/src/lib.rs @@ -51,8 +51,7 @@ pub use re_chunk::{ Chunk, ChunkId, ChunkShared, LatestAtQuery, RangeQuery, RangeQueryOptions, RowId, UnitChunkShared, }; -#[doc(no_inline)] -pub use re_log_encoding::decoder::VersionPolicy; + #[doc(no_inline)] pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline}; @@ -60,7 +59,6 @@ pub mod external { pub use arrow2; pub use re_chunk; - pub use re_log_encoding; } // --- diff --git a/crates/store/re_chunk_store/src/protobuf_conversions.rs b/crates/store/re_chunk_store/src/protobuf_conversions.rs index 67ef038284e0..4f37e7b3bf5a 100644 --- a/crates/store/re_chunk_store/src/protobuf_conversions.rs +++ b/crates/store/re_chunk_store/src/protobuf_conversions.rs @@ -1,8 +1,8 @@ +use re_protos::missing_field; +use re_protos::TypeConversionError; use std::collections::BTreeMap; use std::collections::BTreeSet; -use re_protos::TypeConversionError; - impl TryFrom for crate::ComponentColumnSelector { type Error = TypeConversionError; @@ -11,16 +11,16 @@ impl TryFrom for crate::Componen ) -> Result { let entity_path = value .entity_path - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ComponentColumnSelector", + .ok_or(missing_field!( + re_protos::common::v0::ComponentColumnSelector, "entity_path", ))? .try_into()?; let component_name = value .component - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ComponentColumnSelector", + .ok_or(missing_field!( + re_protos::common::v0::ComponentColumnSelector, "component", ))? .name; @@ -36,8 +36,8 @@ impl TryFrom for crate::TimeColumnSel type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::TimeColumnSelector) -> Result { - let timeline = value.timeline.ok_or(TypeConversionError::missing_field( - "rerun.common.v0.TimeColumnSelector", + let timeline = value.timeline.ok_or(missing_field!( + re_protos::common::v0::TimeColumnSelector, "timeline", ))?; @@ -51,12 +51,10 @@ impl TryFrom for crate::ColumnSelector { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::ColumnSelector) -> Result { - match value - .selector_type - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ColumnSelector", - "selector_type", - ))? { + match value.selector_type.ok_or(missing_field!( + re_protos::common::v0::ColumnSelector, + "selector_type", + ))? { re_protos::common::v0::column_selector::SelectorType::ComponentColumn( component_column_selector, ) => { @@ -115,8 +113,8 @@ impl TryFrom for crate::ViewContentsSelecto .map(|part| { let entity_path: re_log_types::EntityPath = part .path - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.ViewContentsPart", + .ok_or(missing_field!( + re_protos::common::v0::ViewContentsPart, "path", ))? .try_into()?; @@ -139,8 +137,8 @@ impl TryFrom for crate::QueryExpression { fn try_from(value: re_protos::common::v0::Query) -> Result { let filtered_index = value .filtered_index - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.Query", + .ok_or(missing_field!( + re_protos::common::v0::Query, "filtered_index", ))? .try_into()?; diff --git a/crates/store/re_chunk_store/src/store.rs b/crates/store/re_chunk_store/src/store.rs index 454e3fb5c3ab..4658dd03812d 100644 --- a/crates/store/re_chunk_store/src/store.rs +++ b/crates/store/re_chunk_store/src/store.rs @@ -691,7 +691,7 @@ impl ChunkStore { pub fn from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: crate::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { let path_to_rrd = path_to_rrd.as_ref(); @@ -808,7 +808,7 @@ impl ChunkStore { pub fn handle_from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: crate::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { Ok( Self::from_rrd_filepath(store_config, path_to_rrd, version_policy)? diff --git a/crates/store/re_data_loader/src/loader_external.rs b/crates/store/re_data_loader/src/loader_external.rs index be4dacea5e61..51e4ac10f1d9 100644 --- a/crates/store/re_data_loader/src/loader_external.rs +++ b/crates/store/re_data_loader/src/loader_external.rs @@ -182,7 +182,7 @@ impl crate::DataLoader for ExternalLoader { // streaming data to stdout. let is_sending_data = Arc::new(AtomicBool::new(false)); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let stdout = std::io::BufReader::new(stdout); match re_log_encoding::decoder::Decoder::new(version_policy, stdout) { Ok(decoder) => { diff --git a/crates/store/re_data_loader/src/loader_rrd.rs b/crates/store/re_data_loader/src/loader_rrd.rs index a7df3b51acb7..b421c6aa8d06 100644 --- a/crates/store/re_data_loader/src/loader_rrd.rs +++ b/crates/store/re_data_loader/src/loader_rrd.rs @@ -40,7 +40,7 @@ impl crate::DataLoader for RrdLoader { "Loading rrd data from filesystem…", ); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; match extension.as_str() { "rbl" => { @@ -118,7 +118,7 @@ impl crate::DataLoader for RrdLoader { return Err(crate::DataLoaderError::Incompatible(filepath)); } - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let contents = std::io::Cursor::new(contents); let decoder = match re_log_encoding::decoder::Decoder::new(version_policy, contents) { Ok(decoder) => decoder, @@ -308,7 +308,7 @@ impl RetryableFileReader { mod tests { use re_build_info::CrateVersion; use re_chunk::RowId; - use re_log_encoding::{decoder, encoder::DroppableEncoder}; + use re_log_encoding::{encoder::DroppableEncoder, VersionPolicy}; use re_log_types::{ ApplicationId, LogMsg, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; @@ -341,7 +341,7 @@ mod tests { let mut encoder = DroppableEncoder::new( re_build_info::CrateVersion::LOCAL, - re_log_encoding::EncodingOptions::UNCOMPRESSED, + re_log_encoding::EncodingOptions::MSGPACK_UNCOMPRESSED, rrd_file, ) .unwrap(); @@ -372,7 +372,7 @@ mod tests { encoder.flush_blocking().expect("failed to flush messages"); let reader = RetryableFileReader::new(&rrd_file_path).unwrap(); - let mut decoder = Decoder::new(decoder::VersionPolicy::Warn, reader).unwrap(); + let mut decoder = Decoder::new(VersionPolicy::Warn, reader).unwrap(); // we should be able to read 5 messages that we wrote let decoded_messages = (0..5) diff --git a/crates/store/re_data_source/src/load_stdin.rs b/crates/store/re_data_source/src/load_stdin.rs index 0573c20750ae..1c4f5ac437f5 100644 --- a/crates/store/re_data_source/src/load_stdin.rs +++ b/crates/store/re_data_source/src/load_stdin.rs @@ -6,7 +6,7 @@ use re_smart_channel::Sender; /// This fails synchronously iff the standard input stream could not be opened, otherwise errors /// are handled asynchronously (as in: they're logged). pub fn load_stdin(tx: Sender) -> anyhow::Result<()> { - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let stdin = std::io::BufReader::new(std::io::stdin()); let decoder = re_log_encoding::decoder::Decoder::new_concatenated(version_policy, stdin)?; diff --git a/crates/store/re_dataframe/Cargo.toml b/crates/store/re_dataframe/Cargo.toml index eb15df0a9b11..eaaca1a0a58d 100644 --- a/crates/store/re_dataframe/Cargo.toml +++ b/crates/store/re_dataframe/Cargo.toml @@ -24,16 +24,17 @@ all-features = true [features] default = [] - [dependencies] # Rerun dependencies: re_chunk.workspace = true re_chunk_store.workspace = true re_log.workspace = true +re_log_encoding.workspace = true re_log_types.workspace = true re_query.workspace = true re_tracing.workspace = true re_types_core.workspace = true + # External dependencies: anyhow.workspace = true arrow2.workspace = true diff --git a/crates/store/re_dataframe/examples/query.rs b/crates/store/re_dataframe/examples/query.rs index 3cc999177439..f4ffc1af746d 100644 --- a/crates/store/re_dataframe/examples/query.rs +++ b/crates/store/re_dataframe/examples/query.rs @@ -4,8 +4,9 @@ use itertools::Itertools; use re_dataframe::{ ChunkStoreConfig, EntityPathFilter, QueryEngine, QueryExpression, ResolvedTimeRange, - SparseFillStrategy, StoreKind, TimeInt, Timeline, VersionPolicy, + SparseFillStrategy, StoreKind, TimeInt, Timeline, }; +use re_log_encoding::VersionPolicy; fn main() -> anyhow::Result<()> { let args = std::env::args().collect_vec(); diff --git a/crates/store/re_dataframe/src/engine.rs b/crates/store/re_dataframe/src/engine.rs index 8a22a4cdaec0..3fc2bb875944 100644 --- a/crates/store/re_dataframe/src/engine.rs +++ b/crates/store/re_dataframe/src/engine.rs @@ -3,7 +3,6 @@ use std::collections::BTreeMap; use re_chunk::{EntityPath, TransportChunk}; use re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreHandle, ColumnDescriptor, QueryExpression, - VersionPolicy, }; use re_log_types::{EntityPathFilter, StoreId}; use re_query::{QueryCache, QueryCacheHandle, StorageEngine, StorageEngineLike}; @@ -59,7 +58,7 @@ impl QueryEngine { pub fn from_rrd_filepath( store_config: &ChunkStoreConfig, path_to_rrd: impl AsRef, - version_policy: VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, ) -> anyhow::Result> { Ok( ChunkStore::handle_from_rrd_filepath(store_config, path_to_rrd, version_policy)? diff --git a/crates/store/re_dataframe/src/lib.rs b/crates/store/re_dataframe/src/lib.rs index 40702bc590ac..064727f9b612 100644 --- a/crates/store/re_dataframe/src/lib.rs +++ b/crates/store/re_dataframe/src/lib.rs @@ -13,8 +13,7 @@ pub use self::external::re_chunk::{util::concatenate_record_batches, TransportCh #[doc(no_inline)] pub use self::external::re_chunk_store::{ ChunkStoreConfig, ChunkStoreHandle, ColumnSelector, ComponentColumnSelector, Index, IndexRange, - IndexValue, QueryExpression, SparseFillStrategy, TimeColumnSelector, VersionPolicy, - ViewContentsSelector, + IndexValue, QueryExpression, SparseFillStrategy, TimeColumnSelector, ViewContentsSelector, }; #[doc(no_inline)] pub use self::external::re_log_types::{ diff --git a/crates/store/re_entity_db/examples/memory_usage.rs b/crates/store/re_entity_db/examples/memory_usage.rs index 7c4c524dc10e..870e296bcf72 100644 --- a/crates/store/re_entity_db/examples/memory_usage.rs +++ b/crates/store/re_entity_db/examples/memory_usage.rs @@ -66,7 +66,7 @@ fn log_messages() { fn encode_log_msg(log_msg: &LogMsg) -> Vec { let mut bytes = vec![]; - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::encode_ref( re_build_info::CrateVersion::LOCAL, encoding_options, @@ -78,7 +78,7 @@ fn log_messages() { } fn decode_log_msg(mut bytes: &[u8]) -> LogMsg { - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let mut messages = re_log_encoding::decoder::Decoder::new(version_policy, &mut bytes) .unwrap() .collect::, _>>() diff --git a/crates/store/re_entity_db/src/store_bundle.rs b/crates/store/re_entity_db/src/store_bundle.rs index 45fa833e02ee..a6645d606c45 100644 --- a/crates/store/re_entity_db/src/store_bundle.rs +++ b/crates/store/re_entity_db/src/store_bundle.rs @@ -1,7 +1,7 @@ use itertools::Itertools as _; use crate::EntityDb; -use re_log_encoding::decoder::VersionPolicy; +use re_log_encoding::VersionPolicy; use re_log_types::{StoreId, StoreKind}; #[derive(thiserror::Error, Debug)] diff --git a/crates/store/re_grpc_client/src/lib.rs b/crates/store/re_grpc_client/src/lib.rs index d6e7b3605e07..b37c07f20137 100644 --- a/crates/store/re_grpc_client/src/lib.rs +++ b/crates/store/re_grpc_client/src/lib.rs @@ -160,7 +160,8 @@ async fn stream_recording_async( .connect() .await?; - StorageNodeClient::new(tonic_client) + // TODO(#8411): figure out the right size for this + StorageNodeClient::new(tonic_client).max_decoding_message_size(usize::MAX) }; re_log::debug!("Fetching {recording_id}…"); diff --git a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs index d2455b765530..40c72e5a906b 100644 --- a/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs +++ b/crates/store/re_log_encoding/benches/msg_encode_benchmark.rs @@ -14,6 +14,10 @@ use re_log_types::{ LogMsg, StoreId, StoreKind, TimeInt, TimeType, Timeline, }; +use re_log_encoding::EncodingOptions; +const MSGPACK_COMPRESSED: EncodingOptions = EncodingOptions::MSGPACK_COMPRESSED; +const PROTOBUF_COMPRESSED: EncodingOptions = EncodingOptions::PROTOBUF_COMPRESSED; + use criterion::{criterion_group, criterion_main, Criterion}; #[cfg(not(debug_assertions))] @@ -31,8 +35,10 @@ criterion_group!( ); criterion_main!(benches); -fn encode_log_msgs(messages: &[LogMsg]) -> Vec { - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; +fn encode_log_msgs( + messages: &[LogMsg], + encoding_options: re_log_encoding::EncodingOptions, +) -> Vec { let mut bytes = vec![]; re_log_encoding::encoder::encode_ref( re_build_info::CrateVersion::LOCAL, @@ -46,7 +52,7 @@ fn encode_log_msgs(messages: &[LogMsg]) -> Vec { } fn decode_log_msgs(mut bytes: &[u8]) -> Vec { - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let messages = re_log_encoding::decoder::Decoder::new(version_policy, &mut bytes) .unwrap() .collect::, _>>() @@ -111,30 +117,67 @@ fn mono_points_arrow(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &generate_chunks()))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let chunks = decode_chunks(&messages); - assert_eq!(chunks.len(), messages.len()); - chunks + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } @@ -167,30 +210,67 @@ fn mono_points_arrow_batched(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &[generate_chunk()]))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &[generate_chunk()]), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let bundles = decode_chunks(&messages); - assert_eq!(bundles.len(), messages.len()); - bundles + encode_log_msgs( + &generate_messages(&store_id, &[generate_chunk()]), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let bundles = decode_chunks(&messages); + assert_eq!(bundles.len(), messages.len()); + bundles + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let bundles = decode_chunks(&messages); + assert_eq!(bundles.len(), messages.len()); + bundles + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } @@ -222,30 +302,67 @@ fn batch_points_arrow(c: &mut Criterion) { }); let messages = generate_messages(&store_id, &chunks); group.bench_function("encode_log_msg", |b| { - b.iter(|| encode_log_msgs(&messages)); + b.iter(|| encode_log_msgs(&messages, MSGPACK_COMPRESSED)); }); - group.bench_function("encode_total", |b| { - b.iter(|| encode_log_msgs(&generate_messages(&store_id, &generate_chunks()))); + group.bench_function("encode_log_msg(protobuf)", |b| { + b.iter(|| encode_log_msgs(&messages, PROTOBUF_COMPRESSED)); }); - - let encoded = encode_log_msgs(&messages); - group.bench_function("decode_log_msg", |b| { + group.bench_function("encode_total", |b| { b.iter(|| { - let decoded = decode_log_msgs(&encoded); - assert_eq!(decoded.len(), messages.len()); - decoded + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + MSGPACK_COMPRESSED, + ) }); }); - group.bench_function("decode_message_bundles", |b| { + group.bench_function("encode_total(protobuf)", |b| { b.iter(|| { - let chunks = decode_chunks(&messages); - assert_eq!(chunks.len(), messages.len()); - chunks + encode_log_msgs( + &generate_messages(&store_id, &generate_chunks()), + PROTOBUF_COMPRESSED, + ) }); }); - group.bench_function("decode_total", |b| { - b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); - }); + + { + let encoded = encode_log_msgs(&messages, MSGPACK_COMPRESSED); + group.bench_function("decode_log_msg", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + + let encoded = encode_log_msgs(&messages, PROTOBUF_COMPRESSED); + group.bench_function("decode_log_msg(protobuf)", |b| { + b.iter(|| { + let decoded = decode_log_msgs(&encoded); + assert_eq!(decoded.len(), messages.len()); + decoded + }); + }); + group.bench_function("decode_message_bundles(protobuf)", |b| { + b.iter(|| { + let chunks = decode_chunks(&messages); + assert_eq!(chunks.len(), messages.len()); + chunks + }); + }); + group.bench_function("decode_total(protobuf)", |b| { + b.iter(|| decode_chunks(&decode_log_msgs(&encoded))); + }); + } } } diff --git a/crates/store/re_log_encoding/src/codec/arrow.rs b/crates/store/re_log_encoding/src/codec/arrow.rs new file mode 100644 index 000000000000..ae7cddabe27e --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/arrow.rs @@ -0,0 +1,96 @@ +use super::CodecError; + +use arrow2::array::Array as Arrow2Array; +use arrow2::datatypes::Schema as Arrow2Schema; +use arrow2::io::ipc; + +type Arrow2Chunk = arrow2::chunk::Chunk>; + +/// Helper function that serializes given arrow schema and record batch into bytes +/// using Arrow IPC format. +pub(crate) fn write_arrow_to_bytes( + writer: &mut W, + schema: &Arrow2Schema, + data: &Arrow2Chunk, +) -> Result<(), CodecError> { + let options = ipc::write::WriteOptions { compression: None }; + let mut sw = ipc::write::StreamWriter::new(writer, options); + + sw.start(schema, None) + .map_err(CodecError::ArrowSerialization)?; + sw.write(data, None) + .map_err(CodecError::ArrowSerialization)?; + sw.finish().map_err(CodecError::ArrowSerialization)?; + + Ok(()) +} + +/// Helper function that deserializes raw bytes into arrow schema and record batch +/// using Arrow IPC format. +pub(crate) fn read_arrow_from_bytes( + reader: &mut R, +) -> Result<(Arrow2Schema, Arrow2Chunk), CodecError> { + use arrow2::io::ipc; + + let metadata = + ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; + let mut stream = ipc::read::StreamReader::new(reader, metadata, None); + + let schema = stream.schema().clone(); + // there should be at least one record batch in the stream + let stream_state = stream + .next() + .ok_or(CodecError::MissingRecordBatch)? + .map_err(CodecError::ArrowSerialization)?; + + match stream_state { + ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), + ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), + } +} + +#[cfg(feature = "encoder")] +pub(crate) struct Payload { + pub uncompressed_size: usize, + pub data: Vec, +} + +#[cfg(feature = "encoder")] +pub(crate) fn encode_arrow( + schema: &Arrow2Schema, + chunk: &Arrow2Chunk, + compression: crate::Compression, +) -> Result { + let mut uncompressed = Vec::new(); + write_arrow_to_bytes(&mut uncompressed, schema, chunk)?; + let uncompressed_size = uncompressed.len(); + + let data = match compression { + crate::Compression::Off => uncompressed, + crate::Compression::LZ4 => lz4_flex::block::compress(&uncompressed), + }; + + Ok(Payload { + uncompressed_size, + data, + }) +} + +#[cfg(feature = "decoder")] +pub(crate) fn decode_arrow( + data: &[u8], + uncompressed_size: usize, + compression: crate::Compression, +) -> Result<(Arrow2Schema, Arrow2Chunk), crate::decoder::DecodeError> { + let mut uncompressed = Vec::new(); + let data = match compression { + crate::Compression::Off => data, + crate::Compression::LZ4 => { + uncompressed.resize(uncompressed_size, 0); + lz4_flex::block::decompress_into(data, &mut uncompressed)?; + uncompressed.as_slice() + } + }; + + Ok(read_arrow_from_bytes(&mut &data[..])?) +} diff --git a/crates/store/re_log_encoding/src/codec/file/decoder.rs b/crates/store/re_log_encoding/src/codec/file/decoder.rs new file mode 100644 index 000000000000..16b6f0ba4728 --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/file/decoder.rs @@ -0,0 +1,58 @@ +use super::{MessageHeader, MessageKind}; +use crate::codec::arrow::decode_arrow; +use crate::codec::CodecError; +use crate::decoder::DecodeError; +use re_log_types::LogMsg; +use re_protos::missing_field; + +pub(crate) fn decode(data: &mut impl std::io::Read) -> Result<(u64, Option), DecodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo}; + + let mut read_bytes = 0u64; + let header = MessageHeader::decode(data)?; + read_bytes += std::mem::size_of::() as u64 + header.len; + + let mut buf = vec![0; header.len as usize]; + data.read_exact(&mut buf[..])?; + + let msg = match header.kind { + MessageKind::SetStoreInfo => { + let set_store_info = SetStoreInfo::decode(&buf[..])?; + Some(LogMsg::SetStoreInfo(set_store_info.try_into()?)) + } + MessageKind::ArrowMsg => { + let arrow_msg = ArrowMsg::decode(&buf[..])?; + if arrow_msg.encoding() != Encoding::ArrowIpc { + return Err(DecodeError::Codec(CodecError::UnsupportedEncoding)); + } + + let (schema, chunk) = decode_arrow( + &arrow_msg.payload, + arrow_msg.uncompressed_size as usize, + arrow_msg.compression().into(), + )?; + + let store_id: re_log_types::StoreId = arrow_msg + .store_id + .ok_or_else(|| missing_field!(re_protos::log_msg::v0::ArrowMsg, "store_id"))? + .into(); + + let chunk = re_chunk::Chunk::from_transport(&re_chunk::TransportChunk { + schema, + data: chunk, + })?; + + Some(LogMsg::ArrowMsg(store_id, chunk.to_arrow_msg()?)) + } + MessageKind::BlueprintActivationCommand => { + let blueprint_activation_command = BlueprintActivationCommand::decode(&buf[..])?; + Some(LogMsg::BlueprintActivationCommand( + blueprint_activation_command.try_into()?, + )) + } + MessageKind::End => None, + }; + + Ok((read_bytes, msg)) +} diff --git a/crates/store/re_log_encoding/src/codec/file/encoder.rs b/crates/store/re_log_encoding/src/codec/file/encoder.rs new file mode 100644 index 000000000000..3456df76b37b --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/file/encoder.rs @@ -0,0 +1,59 @@ +use super::{MessageHeader, MessageKind}; +use crate::codec::arrow::encode_arrow; +use crate::encoder::EncodeError; +use crate::Compression; +use re_log_types::LogMsg; + +pub(crate) fn encode( + buf: &mut Vec, + message: &LogMsg, + compression: Compression, +) -> Result<(), EncodeError> { + use re_protos::external::prost::Message; + use re_protos::log_msg::v0::{ + self as proto, ArrowMsg, BlueprintActivationCommand, Encoding, SetStoreInfo, + }; + + match message { + LogMsg::SetStoreInfo(set_store_info) => { + let set_store_info: SetStoreInfo = set_store_info.clone().into(); + let header = MessageHeader { + kind: MessageKind::SetStoreInfo, + len: set_store_info.encoded_len() as u64, + }; + header.encode(buf)?; + set_store_info.encode(buf)?; + } + LogMsg::ArrowMsg(store_id, arrow_msg) => { + let payload = encode_arrow(&arrow_msg.schema, &arrow_msg.chunk, compression)?; + let arrow_msg = ArrowMsg { + store_id: Some(store_id.clone().into()), + compression: match compression { + Compression::Off => proto::Compression::None as i32, + Compression::LZ4 => proto::Compression::Lz4 as i32, + }, + uncompressed_size: payload.uncompressed_size as i32, + encoding: Encoding::ArrowIpc as i32, + payload: payload.data, + }; + let header = MessageHeader { + kind: MessageKind::ArrowMsg, + len: arrow_msg.encoded_len() as u64, + }; + header.encode(buf)?; + arrow_msg.encode(buf)?; + } + LogMsg::BlueprintActivationCommand(blueprint_activation_command) => { + let blueprint_activation_command: BlueprintActivationCommand = + blueprint_activation_command.clone().into(); + let header = MessageHeader { + kind: MessageKind::BlueprintActivationCommand, + len: blueprint_activation_command.encoded_len() as u64, + }; + header.encode(buf)?; + blueprint_activation_command.encode(buf)?; + } + } + + Ok(()) +} diff --git a/crates/store/re_log_encoding/src/codec/file/mod.rs b/crates/store/re_log_encoding/src/codec/file/mod.rs new file mode 100644 index 000000000000..e491777409ca --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/file/mod.rs @@ -0,0 +1,70 @@ +#[cfg(feature = "decoder")] +pub(crate) mod decoder; +#[cfg(feature = "encoder")] +pub(crate) mod encoder; + +#[allow(dead_code)] // used behind feature flag +#[derive(Default, Debug, Clone, Copy)] +#[repr(u64)] +pub(crate) enum MessageKind { + #[default] + End = Self::END, + SetStoreInfo = Self::SET_STORE_INFO, + ArrowMsg = Self::ARROW_MSG, + BlueprintActivationCommand = Self::BLUEPRINT_ACTIVATION_COMMAND, +} + +#[allow(dead_code)] // used behind feature flag +impl MessageKind { + const END: u64 = 0; + const SET_STORE_INFO: u64 = 1; + const ARROW_MSG: u64 = 2; + const BLUEPRINT_ACTIVATION_COMMAND: u64 = 3; +} + +#[allow(dead_code)] // used behind feature flag +#[derive(Debug, Clone, Copy)] +pub(crate) struct MessageHeader { + pub(crate) kind: MessageKind, + pub(crate) len: u64, +} + +impl MessageHeader { + #[cfg(feature = "encoder")] + pub(crate) fn encode( + &self, + buf: &mut impl std::io::Write, + ) -> Result<(), crate::encoder::EncodeError> { + let Self { kind, len } = *self; + buf.write_all(&(kind as u64).to_le_bytes())?; + buf.write_all(&len.to_le_bytes())?; + Ok(()) + } + + #[cfg(feature = "decoder")] + pub(crate) fn decode( + data: &mut impl std::io::Read, + ) -> Result { + let mut buf = [0; std::mem::size_of::()]; + data.read_exact(&mut buf)?; + + #[allow(clippy::unwrap_used)] // cannot fail + let kind = u64::from_le_bytes(buf[0..8].try_into().unwrap()); + let kind = match kind { + MessageKind::END => MessageKind::End, + MessageKind::SET_STORE_INFO => MessageKind::SetStoreInfo, + MessageKind::ARROW_MSG => MessageKind::ArrowMsg, + MessageKind::BLUEPRINT_ACTIVATION_COMMAND => MessageKind::BlueprintActivationCommand, + _ => { + return Err(crate::decoder::DecodeError::Codec( + crate::codec::CodecError::UnknownMessageHeader, + )) + } + }; + + #[allow(clippy::unwrap_used)] // cannot fail + let len = u64::from_le_bytes(buf[8..16].try_into().unwrap()); + + Ok(Self { kind, len }) + } +} diff --git a/crates/store/re_log_encoding/src/codec/mod.rs b/crates/store/re_log_encoding/src/codec/mod.rs index c332217e0e04..7c51b4b690e0 100644 --- a/crates/store/re_log_encoding/src/codec/mod.rs +++ b/crates/store/re_log_encoding/src/codec/mod.rs @@ -1,3 +1,5 @@ +mod arrow; +pub mod file; pub mod wire; #[derive(Debug, thiserror::Error)] @@ -20,18 +22,6 @@ pub enum CodecError { #[error("Unsupported encoding, expected Arrow IPC")] UnsupportedEncoding, - #[error("Invalid file header")] - InvalidFileHeader, - #[error("Unknown message header")] UnknownMessageHeader, - - #[error("Invalid message header")] - InvalidMessageHeader, - - #[error("Unknown message kind {0}")] - UnknownMessageKind(u8), - - #[error("Invalid argument: {0}")] - InvalidArgument(String), } diff --git a/crates/store/re_log_encoding/src/codec/wire.rs b/crates/store/re_log_encoding/src/codec/wire.rs deleted file mode 100644 index e0b0daabd1d1..000000000000 --- a/crates/store/re_log_encoding/src/codec/wire.rs +++ /dev/null @@ -1,261 +0,0 @@ -use arrow2::chunk::Chunk as Arrow2Chunk; -use arrow2::datatypes::Schema as Arrow2Schema; -use arrow2::io::ipc; -use re_chunk::Arrow2Array; -use re_chunk::TransportChunk; - -use super::CodecError; - -#[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] -pub struct MessageHeader(pub u8); - -impl MessageHeader { - pub const NO_DATA: Self = Self(1); - pub const RECORD_BATCH: Self = Self(2); - - pub const SIZE_BYTES: usize = 1; -} - -impl MessageHeader { - fn decode(read: &mut impl std::io::Read) -> Result { - let mut buffer = [0_u8; Self::SIZE_BYTES]; - read.read_exact(&mut buffer) - .map_err(CodecError::HeaderDecoding)?; - - let header = u8::from_le(buffer[0]); - - Ok(Self(header)) - } - - fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { - write - .write_all(&[self.0]) - .map_err(CodecError::HeaderEncoding)?; - - Ok(()) - } -} - -#[derive(Debug)] -pub enum TransportMessageV0 { - NoData, - RecordBatch(TransportChunk), -} - -impl TransportMessageV0 { - fn to_bytes(&self) -> Result, CodecError> { - match self { - Self::NoData => { - let mut data: Vec = Vec::new(); - MessageHeader::NO_DATA.encode(&mut data)?; - Ok(data) - } - Self::RecordBatch(chunk) => { - let mut data: Vec = Vec::new(); - MessageHeader::RECORD_BATCH.encode(&mut data)?; - - write_arrow_to_bytes(&mut data, &chunk.schema, &chunk.data)?; - - Ok(data) - } - } - } - - fn from_bytes(data: &[u8]) -> Result { - let mut reader = std::io::Cursor::new(data); - let header = MessageHeader::decode(&mut reader)?; - - match header { - MessageHeader::NO_DATA => Ok(Self::NoData), - MessageHeader::RECORD_BATCH => { - let (schema, data) = read_arrow_from_bytes(&mut reader)?; - - let tc = TransportChunk { - schema: schema.clone(), - data, - }; - - Ok(Self::RecordBatch(tc)) - } - _ => Err(CodecError::UnknownMessageHeader), - } - } -} - -// TODO(zehiko) add support for separately encoding schema from the record batch to get rid of overhead -// of sending schema in each transport message for the same stream of batches. This will require codec -// to become stateful and keep track if schema was sent / received. -/// Encode a transport chunk into a byte stream. -pub fn encode( - version: re_protos::common::v0::EncoderVersion, - chunk: TransportChunk, -) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => { - TransportMessageV0::RecordBatch(chunk).to_bytes() - } - } -} - -/// Encode a `NoData` message into a byte stream. This can be used by the remote store -/// (i.e. data producer) to signal back to the client that there's no data available. -pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), - } -} - -/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. -pub fn decode( - version: re_protos::common::v0::EncoderVersion, - data: &[u8], -) -> Result, CodecError> { - match version { - re_protos::common::v0::EncoderVersion::V0 => { - let msg = TransportMessageV0::from_bytes(data)?; - match msg { - TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), - TransportMessageV0::NoData => Ok(None), - } - } - } -} - -/// Helper function that serializes given arrow schema and record batch into bytes -/// using Arrow IPC format. -pub fn write_arrow_to_bytes( - writer: &mut W, - schema: &Arrow2Schema, - data: &Arrow2Chunk>, -) -> Result<(), CodecError> { - let options = ipc::write::WriteOptions { compression: None }; - let mut sw = ipc::write::StreamWriter::new(writer, options); - - sw.start(schema, None) - .map_err(CodecError::ArrowSerialization)?; - sw.write(data, None) - .map_err(CodecError::ArrowSerialization)?; - sw.finish().map_err(CodecError::ArrowSerialization)?; - - Ok(()) -} - -/// Helper function that deserializes raw bytes into arrow schema and record batch -/// using Arrow IPC format. -pub fn read_arrow_from_bytes( - reader: &mut R, -) -> Result<(Arrow2Schema, Arrow2Chunk>), CodecError> { - let metadata = - ipc::read::read_stream_metadata(reader).map_err(CodecError::ArrowSerialization)?; - let mut stream = ipc::read::StreamReader::new(reader, metadata, None); - - let schema = stream.schema().clone(); - // there should be at least one record batch in the stream - let stream_state = stream - .next() - .ok_or(CodecError::MissingRecordBatch)? - .map_err(CodecError::ArrowSerialization)?; - - match stream_state { - ipc::read::StreamState::Waiting => Err(CodecError::UnexpectedStreamState), - ipc::read::StreamState::Some(chunk) => Ok((schema, chunk)), - } -} - -#[cfg(test)] -mod tests { - use crate::{ - codec::wire::{decode, encode, TransportMessageV0}, - codec::CodecError, - }; - use re_chunk::{Chunk, RowId}; - use re_log_types::{example_components::MyPoint, Timeline}; - use re_protos::common::v0::EncoderVersion; - - fn get_test_chunk() -> Chunk { - let row_id1 = RowId::new(); - let row_id2 = RowId::new(); - - let timepoint1 = [ - (Timeline::log_time(), 100), - (Timeline::new_sequence("frame"), 1), - ]; - let timepoint2 = [ - (Timeline::log_time(), 104), - (Timeline::new_sequence("frame"), 1), - ]; - - let points1 = &[MyPoint::new(1.0, 1.0)]; - let points2 = &[MyPoint::new(2.0, 2.0)]; - - Chunk::builder("mypoints".into()) - .with_component_batches(row_id1, timepoint1, [points1 as _]) - .with_component_batches(row_id2, timepoint2, [points2 as _]) - .build() - .unwrap() - } - - #[test] - fn test_message_v0_no_data() { - let msg = TransportMessageV0::NoData; - let data = msg.to_bytes().unwrap(); - let decoded = TransportMessageV0::from_bytes(&data).unwrap(); - assert!(matches!(decoded, TransportMessageV0::NoData)); - } - - #[test] - fn test_message_v0_record_batch() { - let expected_chunk = get_test_chunk(); - - let msg = TransportMessageV0::RecordBatch(expected_chunk.clone().to_transport().unwrap()); - let data = msg.to_bytes().unwrap(); - let decoded = TransportMessageV0::from_bytes(&data).unwrap(); - - #[allow(clippy::match_wildcard_for_single_variants)] - match decoded { - TransportMessageV0::RecordBatch(transport) => { - let decoded_chunk = Chunk::from_transport(&transport).unwrap(); - assert_eq!(expected_chunk, decoded_chunk); - } - _ => panic!("unexpected message type"), - } - } - - #[test] - fn test_invalid_batch_data() { - let data = vec![2, 3, 4]; // '1' is NO_DATA message header - let decoded = TransportMessageV0::from_bytes(&data); - - assert!(matches!( - decoded.err().unwrap(), - CodecError::ArrowSerialization(_) - )); - } - - #[test] - fn test_unknown_header() { - let data = vec![3]; - let decoded = TransportMessageV0::from_bytes(&data); - assert!(decoded.is_err()); - - assert!(matches!( - decoded.err().unwrap(), - CodecError::UnknownMessageHeader - )); - } - - #[test] - fn test_v0_codec() { - let expected_chunk = get_test_chunk(); - - let encoded = encode( - EncoderVersion::V0, - expected_chunk.clone().to_transport().unwrap(), - ) - .unwrap(); - let decoded = decode(EncoderVersion::V0, &encoded).unwrap().unwrap(); - let decoded_chunk = Chunk::from_transport(&decoded).unwrap(); - - assert_eq!(expected_chunk, decoded_chunk); - } -} diff --git a/crates/store/re_log_encoding/src/codec/wire/decoder.rs b/crates/store/re_log_encoding/src/codec/wire/decoder.rs new file mode 100644 index 000000000000..06b9bccdbc23 --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/decoder.rs @@ -0,0 +1,55 @@ +use super::MessageHeader; +use super::TransportMessageV0; +use crate::codec::arrow::read_arrow_from_bytes; +use crate::codec::CodecError; +use re_chunk::TransportChunk; + +impl MessageHeader { + pub(crate) fn decode(read: &mut impl std::io::Read) -> Result { + let mut buffer = [0_u8; Self::SIZE_BYTES]; + read.read_exact(&mut buffer) + .map_err(CodecError::HeaderDecoding)?; + + let header = u8::from_le(buffer[0]); + + Ok(Self(header)) + } +} + +impl TransportMessageV0 { + pub(crate) fn from_bytes(data: &[u8]) -> Result { + let mut reader = std::io::Cursor::new(data); + let header = MessageHeader::decode(&mut reader)?; + + match header { + MessageHeader::NO_DATA => Ok(Self::NoData), + MessageHeader::RECORD_BATCH => { + let (schema, data) = read_arrow_from_bytes(&mut reader)?; + + let tc = TransportChunk { + schema: schema.clone(), + data, + }; + + Ok(Self::RecordBatch(tc)) + } + _ => Err(CodecError::UnknownMessageHeader), + } + } +} + +/// Decode transport data from a byte stream - if there's a record batch present, return it, otherwise return `None`. +pub fn decode( + version: re_protos::common::v0::EncoderVersion, + data: &[u8], +) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => { + let msg = TransportMessageV0::from_bytes(data)?; + match msg { + TransportMessageV0::RecordBatch(chunk) => Ok(Some(chunk)), + TransportMessageV0::NoData => Ok(None), + } + } + } +} diff --git a/crates/store/re_log_encoding/src/codec/wire/encoder.rs b/crates/store/re_log_encoding/src/codec/wire/encoder.rs new file mode 100644 index 000000000000..e6ae62c1e1b7 --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/encoder.rs @@ -0,0 +1,58 @@ +use super::MessageHeader; +use super::TransportMessageV0; +use crate::codec::arrow::write_arrow_to_bytes; +use crate::codec::CodecError; +use re_chunk::TransportChunk; + +impl MessageHeader { + pub(crate) fn encode(&self, write: &mut impl std::io::Write) -> Result<(), CodecError> { + write + .write_all(&[self.0]) + .map_err(CodecError::HeaderEncoding)?; + + Ok(()) + } +} + +impl TransportMessageV0 { + pub(crate) fn to_bytes(&self) -> Result, CodecError> { + match self { + Self::NoData => { + let mut data: Vec = Vec::new(); + MessageHeader::NO_DATA.encode(&mut data)?; + Ok(data) + } + Self::RecordBatch(chunk) => { + let mut data: Vec = Vec::new(); + MessageHeader::RECORD_BATCH.encode(&mut data)?; + + write_arrow_to_bytes(&mut data, &chunk.schema, &chunk.data)?; + + Ok(data) + } + } + } +} + +/// Encode a `NoData` message into a byte stream. This can be used by the remote store +/// (i.e. data producer) to signal back to the client that there's no data available. +pub fn no_data(version: re_protos::common::v0::EncoderVersion) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => TransportMessageV0::NoData.to_bytes(), + } +} + +// TODO(zehiko) add support for separately encoding schema from the record batch to get rid of overhead +// of sending schema in each transport message for the same stream of batches. This will require codec +// to become stateful and keep track if schema was sent / received. +/// Encode a transport chunk into a byte stream. +pub fn encode( + version: re_protos::common::v0::EncoderVersion, + chunk: TransportChunk, +) -> Result, CodecError> { + match version { + re_protos::common::v0::EncoderVersion::V0 => { + TransportMessageV0::RecordBatch(chunk).to_bytes() + } + } +} diff --git a/crates/store/re_log_encoding/src/codec/wire/mod.rs b/crates/store/re_log_encoding/src/codec/wire/mod.rs new file mode 100644 index 000000000000..587e9e31e2ce --- /dev/null +++ b/crates/store/re_log_encoding/src/codec/wire/mod.rs @@ -0,0 +1,121 @@ +pub mod decoder; +pub mod encoder; + +pub use decoder::decode; +pub use encoder::encode; + +use re_chunk::TransportChunk; + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct MessageHeader(pub u8); + +impl MessageHeader { + pub const NO_DATA: Self = Self(1); + pub const RECORD_BATCH: Self = Self(2); + + pub const SIZE_BYTES: usize = 1; +} + +#[derive(Debug)] +pub enum TransportMessageV0 { + NoData, + RecordBatch(TransportChunk), +} + +#[cfg(test)] +mod tests { + use crate::{ + codec::wire::{decode, encode, TransportMessageV0}, + codec::CodecError, + }; + use re_chunk::{Chunk, RowId}; + use re_log_types::{example_components::MyPoint, Timeline}; + use re_protos::common::v0::EncoderVersion; + + fn get_test_chunk() -> Chunk { + let row_id1 = RowId::new(); + let row_id2 = RowId::new(); + + let timepoint1 = [ + (Timeline::log_time(), 100), + (Timeline::new_sequence("frame"), 1), + ]; + let timepoint2 = [ + (Timeline::log_time(), 104), + (Timeline::new_sequence("frame"), 1), + ]; + + let points1 = &[MyPoint::new(1.0, 1.0)]; + let points2 = &[MyPoint::new(2.0, 2.0)]; + + Chunk::builder("mypoints".into()) + .with_component_batches(row_id1, timepoint1, [points1 as _]) + .with_component_batches(row_id2, timepoint2, [points2 as _]) + .build() + .unwrap() + } + + #[test] + fn test_message_v0_no_data() { + let msg = TransportMessageV0::NoData; + let data = msg.to_bytes().unwrap(); + let decoded = TransportMessageV0::from_bytes(&data).unwrap(); + assert!(matches!(decoded, TransportMessageV0::NoData)); + } + + #[test] + fn test_message_v0_record_batch() { + let expected_chunk = get_test_chunk(); + + let msg = TransportMessageV0::RecordBatch(expected_chunk.clone().to_transport().unwrap()); + let data = msg.to_bytes().unwrap(); + let decoded = TransportMessageV0::from_bytes(&data).unwrap(); + + #[allow(clippy::match_wildcard_for_single_variants)] + match decoded { + TransportMessageV0::RecordBatch(transport) => { + let decoded_chunk = Chunk::from_transport(&transport).unwrap(); + assert_eq!(expected_chunk, decoded_chunk); + } + _ => panic!("unexpected message type"), + } + } + + #[test] + fn test_invalid_batch_data() { + let data = vec![2, 3, 4]; // '1' is NO_DATA message header + let decoded = TransportMessageV0::from_bytes(&data); + + assert!(matches!( + decoded.err().unwrap(), + CodecError::ArrowSerialization(_) + )); + } + + #[test] + fn test_unknown_header() { + let data = vec![3]; + let decoded = TransportMessageV0::from_bytes(&data); + assert!(decoded.is_err()); + + assert!(matches!( + decoded.err().unwrap(), + CodecError::UnknownMessageHeader + )); + } + + #[test] + fn test_v0_codec() { + let expected_chunk = get_test_chunk(); + + let encoded = encode( + EncoderVersion::V0, + expected_chunk.clone().to_transport().unwrap(), + ) + .unwrap(); + let decoded = decode(EncoderVersion::V0, &encoded).unwrap().unwrap(); + let decoded_chunk = Chunk::from_transport(&decoded).unwrap(); + + assert_eq!(expected_chunk, decoded_chunk); + } +} diff --git a/crates/store/re_log_encoding/src/decoder/mod.rs b/crates/store/re_log_encoding/src/decoder/mod.rs index 3483bd4b39fb..224bc03ceef2 100644 --- a/crates/store/re_log_encoding/src/decoder/mod.rs +++ b/crates/store/re_log_encoding/src/decoder/mod.rs @@ -8,27 +8,16 @@ use std::io::Read; use re_build_info::CrateVersion; use re_log_types::LogMsg; +use crate::codec; +use crate::codec::file::decoder; use crate::FileHeader; use crate::MessageHeader; +use crate::VersionPolicy; use crate::OLD_RRD_HEADERS; use crate::{Compression, EncodingOptions, Serializer}; // ---------------------------------------------------------------------------- -/// How to handle version mismatches during decoding. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum VersionPolicy { - /// Warn if the versions don't match, but continue loading. - /// - /// We usually use this for loading `.rrd` recordings. - Warn, - - /// Return [`DecodeError::IncompatibleRerunVersion`] if the versions aren't compatible. - /// - /// We usually use this for tests, and for loading `.rbl` blueprint files. - Error, -} - fn warn_on_version_mismatch( version_policy: VersionPolicy, encoded_version: [u8; 4], @@ -82,13 +71,28 @@ pub enum DecodeError { Options(#[from] crate::OptionsError), #[error("Failed to read: {0}")] - Read(std::io::Error), + Read(#[from] std::io::Error), #[error("lz4 error: {0}")] - Lz4(lz4_flex::block::DecompressError), + Lz4(#[from] lz4_flex::block::DecompressError), + + #[error("Protobuf error: {0}")] + Protobuf(#[from] re_protos::external::prost::DecodeError), + + #[error("Could not convert type from protobuf: {0}")] + TypeConversion(#[from] re_protos::TypeConversionError), + + #[error("Failed to read chunk: {0}")] + Chunk(#[from] re_chunk::ChunkError), + + #[error("Arrow error: {0}")] + Arrow(#[from] arrow2::error::Error), #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::decode::Error), + + #[error("Codec error: {0}")] + Codec(#[from] codec::CodecError), } // ---------------------------------------------------------------------------- @@ -129,7 +133,7 @@ pub fn read_options( warn_on_version_mismatch(version_policy, version)?; match options.serializer { - Serializer::MsgPack => {} + Serializer::MsgPack | Serializer::Protobuf => {} } Ok((CrateVersion::from_bytes(version), options)) @@ -152,7 +156,7 @@ impl std::io::Read for Reader { pub struct Decoder { version: CrateVersion, - compression: Compression, + options: EncodingOptions, read: Reader, uncompressed: Vec, // scratch space compressed: Vec, // scratch space @@ -179,11 +183,10 @@ impl Decoder { read.read_exact(&mut data).map_err(DecodeError::Read)?; let (version, options) = read_options(version_policy, &data)?; - let compression = options.compression; Ok(Self { version, - compression, + options, read: Reader::Raw(read), uncompressed: vec![], compressed: vec![], @@ -217,11 +220,10 @@ impl Decoder { read.read_exact(&mut data).map_err(DecodeError::Read)?; let (version, options) = read_options(version_policy, &data)?; - let compression = options.compression; Ok(Self { version, - compression, + options, read: Reader::Buffered(read), uncompressed: vec![], compressed: vec![], @@ -235,6 +237,7 @@ impl Decoder { self.version } + // TODO(jan): stop returning number of read bytes, use cursors wrapping readers instead. /// Returns the size in bytes of the data that has been decoded up to now. #[inline] pub fn size_bytes(&self) -> u64 { @@ -284,90 +287,114 @@ impl Iterator for Decoder { Ok(opts) => opts, Err(err) => return Some(Err(err)), }; - let compression = options.compression; self.version = CrateVersion::max(self.version, version); - self.compression = compression; + self.options = options; self.size_bytes += FileHeader::SIZE as u64; } - let header = match MessageHeader::decode(&mut self.read) { - Ok(header) => header, - Err(err) => match err { - DecodeError::Read(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - return None; + let msg = match self.options.serializer { + Serializer::Protobuf => match decoder::decode(&mut self.read) { + Ok((read_bytes, msg)) => { + self.size_bytes += read_bytes; + msg } - other => return Some(Err(other)), + Err(err) => return Some(Err(err)), }, - }; - self.size_bytes += MessageHeader::SIZE as u64; - - let (uncompressed_len, compressed_len) = match header { - MessageHeader::Data { - compressed_len, - uncompressed_len, - } => (uncompressed_len as usize, compressed_len as usize), - MessageHeader::EndOfStream => { - // we might have a concatenated stream, so we peek beyond end of file marker to see - if self.peek_file_header() { - re_log::debug!("Reached end of stream, but it seems we have a concatenated file, continuing"); - return self.next(); + Serializer::MsgPack => { + let header = match MessageHeader::decode(&mut self.read) { + Ok(header) => header, + Err(err) => match err { + DecodeError::Read(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + return None; + } + other => return Some(Err(other)), + }, + }; + self.size_bytes += MessageHeader::SIZE as u64; + + match header { + MessageHeader::Data { + compressed_len, + uncompressed_len, + } => { + let uncompressed_len = uncompressed_len as usize; + let compressed_len = compressed_len as usize; + + self.uncompressed + .resize(self.uncompressed.len().max(uncompressed_len), 0); + + match self.options.compression { + Compression::Off => { + re_tracing::profile_scope!("read uncompressed"); + if let Err(err) = self + .read + .read_exact(&mut self.uncompressed[..uncompressed_len]) + { + return Some(Err(DecodeError::Read(err))); + } + self.size_bytes += uncompressed_len as u64; + } + + Compression::LZ4 => { + self.compressed + .resize(self.compressed.len().max(compressed_len), 0); + + { + re_tracing::profile_scope!("read compressed"); + if let Err(err) = + self.read.read_exact(&mut self.compressed[..compressed_len]) + { + return Some(Err(DecodeError::Read(err))); + } + } + + re_tracing::profile_scope!("lz4"); + if let Err(err) = lz4_flex::block::decompress_into( + &self.compressed[..compressed_len], + &mut self.uncompressed[..uncompressed_len], + ) { + return Some(Err(DecodeError::Lz4(err))); + } + + self.size_bytes += compressed_len as u64; + } + } + + let data = &self.uncompressed[..uncompressed_len]; + { + re_tracing::profile_scope!("MsgPack deser"); + match rmp_serde::from_slice::(data) { + Ok(msg) => Some(msg), + Err(err) => return Some(Err(err.into())), + } + } + } + MessageHeader::EndOfStream => None, } - - re_log::debug!("Reached end of stream, iterator complete"); - return None; } }; - self.uncompressed - .resize(self.uncompressed.len().max(uncompressed_len), 0); - - match self.compression { - Compression::Off => { - re_tracing::profile_scope!("read uncompressed"); - if let Err(err) = self - .read - .read_exact(&mut self.uncompressed[..uncompressed_len]) - { - return Some(Err(DecodeError::Read(err))); - } - self.size_bytes += uncompressed_len as u64; + let Some(mut msg) = msg else { + // we might have a concatenated stream, so we peek beyond end of file marker to see + if self.peek_file_header() { + re_log::debug!( + "Reached end of stream, but it seems we have a concatenated file, continuing" + ); + return self.next(); } - Compression::LZ4 => { - self.compressed - .resize(self.compressed.len().max(compressed_len), 0); - - { - re_tracing::profile_scope!("read compressed"); - if let Err(err) = self.read.read_exact(&mut self.compressed[..compressed_len]) { - return Some(Err(DecodeError::Read(err))); - } - } - - re_tracing::profile_scope!("lz4"); - if let Err(err) = lz4_flex::block::decompress_into( - &self.compressed[..compressed_len], - &mut self.uncompressed[..uncompressed_len], - ) { - return Some(Err(DecodeError::Lz4(err))); - } + re_log::debug!("Reached end of stream, iterator complete"); + return None; + }; - self.size_bytes += compressed_len as u64; - } + if let LogMsg::SetStoreInfo(msg) = &mut msg { + // Propagate the protocol version from the header into the `StoreInfo` so that all + // parts of the app can easily access it. + msg.info.store_version = Some(self.version()); } - re_tracing::profile_scope!("MsgPack deser"); - match rmp_serde::from_slice(&self.uncompressed[..uncompressed_len]) { - Ok(re_log_types::LogMsg::SetStoreInfo(mut msg)) => { - // Propagate the protocol version from the header into the `StoreInfo` so that all - // parts of the app can easily access it. - msg.info.store_version = Some(self.version()); - Some(Ok(re_log_types::LogMsg::SetStoreInfo(msg))) - } - Ok(msg) => Some(Ok(msg)), - Err(err) => Some(Err(err.into())), - } + Some(Ok(msg)) } } @@ -375,6 +402,8 @@ impl Iterator for Decoder { #[cfg(all(test, feature = "decoder", feature = "encoder"))] mod tests { + #![allow(clippy::unwrap_used)] // acceptable for tests + use super::*; use re_build_info::CrateVersion; use re_chunk::RowId; @@ -382,29 +411,68 @@ mod tests { ApplicationId, SetStoreInfo, StoreId, StoreInfo, StoreKind, StoreSource, Time, }; - fn fake_log_message() -> LogMsg { - LogMsg::SetStoreInfo(SetStoreInfo { - row_id: *RowId::new(), - info: StoreInfo { - application_id: ApplicationId("test".to_owned()), - store_id: StoreId::random(StoreKind::Recording), - cloned_from: None, - is_official_example: true, - started: Time::now(), - store_source: StoreSource::RustSdk { - rustc_version: String::new(), - llvm_version: String::new(), + fn fake_log_messages() -> Vec { + let store_id = StoreId::random(StoreKind::Blueprint); + vec![ + LogMsg::SetStoreInfo(SetStoreInfo { + row_id: *RowId::new(), + info: StoreInfo { + application_id: ApplicationId("test".to_owned()), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: true, + started: Time::now(), + store_source: StoreSource::RustSdk { + rustc_version: String::new(), + llvm_version: String::new(), + }, + store_version: Some(CrateVersion::LOCAL), }, - store_version: Some(CrateVersion::LOCAL), - }, - }) + }), + LogMsg::ArrowMsg( + store_id.clone(), + re_chunk::Chunk::builder("test_entity".into()) + .with_archetype( + re_chunk::RowId::new(), + re_log_types::TimePoint::default().with( + re_log_types::Timeline::new_sequence("blueprint"), + re_log_types::TimeInt::from_milliseconds(re_log_types::NonMinI64::MIN), + ), + &re_types::blueprint::archetypes::Background::new( + re_types::blueprint::components::BackgroundKind::SolidColor, + ) + .with_color([255, 0, 0]), + ) + .build() + .unwrap() + .to_arrow_msg() + .unwrap(), + ), + LogMsg::BlueprintActivationCommand(re_log_types::BlueprintActivationCommand { + blueprint_id: store_id, + make_active: true, + make_default: true, + }), + ] + } + + fn clear_arrow_extension_metadata(messages: &mut Vec) { + for msg in messages { + if let LogMsg::ArrowMsg(_, arrow_msg) = msg { + for field in &mut arrow_msg.schema.fields { + field + .metadata + .retain(|k, _| !k.starts_with("ARROW:extension")); + } + } + } } #[test] fn test_encode_decode() { let rrd_version = CrateVersion::LOCAL; - let messages = vec![fake_log_message()]; + let messages = fake_log_messages(); let options = [ EncodingOptions { @@ -415,6 +483,14 @@ mod tests { compression: Compression::LZ4, serializer: Serializer::MsgPack, }, + EncodingOptions { + compression: Compression::Off, + serializer: Serializer::Protobuf, + }, + EncodingOptions { + compression: Compression::LZ4, + serializer: Serializer::Protobuf, + }, ]; for options in options { @@ -422,11 +498,13 @@ mod tests { crate::encoder::encode_ref(rrd_version, options, messages.iter().map(Ok), &mut file) .unwrap(); - let decoded_messages = Decoder::new(VersionPolicy::Error, &mut file.as_slice()) + let mut decoded_messages = Decoder::new(VersionPolicy::Error, &mut file.as_slice()) .unwrap() .collect::, DecodeError>>() .unwrap(); + clear_arrow_extension_metadata(&mut decoded_messages); + assert_eq!(messages, decoded_messages); } } @@ -442,25 +520,31 @@ mod tests { compression: Compression::LZ4, serializer: Serializer::MsgPack, }, + EncodingOptions { + compression: Compression::Off, + serializer: Serializer::Protobuf, + }, + EncodingOptions { + compression: Compression::LZ4, + serializer: Serializer::Protobuf, + }, ]; for options in options { + println!("{options:?}"); + let mut data = vec![]; // write "2 files" i.e. 2 streams that end with end-of-stream marker - let messages = vec![ - fake_log_message(), - fake_log_message(), - fake_log_message(), - fake_log_message(), - ]; + let messages = fake_log_messages(); // (2 encoders as each encoder writes a file header) let writer = std::io::Cursor::new(&mut data); let mut encoder1 = crate::encoder::Encoder::new(CrateVersion::LOCAL, options, writer).unwrap(); - encoder1.append(&messages[0]).unwrap(); - encoder1.append(&messages[1]).unwrap(); + for message in &messages { + encoder1.append(message).unwrap(); + } encoder1.finish().unwrap(); let written = data.len() as u64; @@ -468,9 +552,9 @@ mod tests { writer.set_position(written); let mut encoder2 = crate::encoder::Encoder::new(CrateVersion::LOCAL, options, writer).unwrap(); - - encoder2.append(&messages[2]).unwrap(); - encoder2.append(&messages[3]).unwrap(); + for message in &messages { + encoder2.append(message).unwrap(); + } encoder2.finish().unwrap(); let decoder = Decoder::new_concatenated( @@ -479,12 +563,11 @@ mod tests { ) .unwrap(); - let mut decoded_messages = vec![]; - for msg in decoder { - decoded_messages.push(msg.unwrap()); - } + let mut decoded_messages = decoder.into_iter().collect::, _>>().unwrap(); - assert_eq!(messages, decoded_messages); + clear_arrow_extension_metadata(&mut decoded_messages); + + assert_eq!([messages.clone(), messages].concat(), decoded_messages); } } } diff --git a/crates/store/re_log_encoding/src/decoder/stream.rs b/crates/store/re_log_encoding/src/decoder/stream.rs index ff85e734581b..0682c57a0609 100644 --- a/crates/store/re_log_encoding/src/decoder/stream.rs +++ b/crates/store/re_log_encoding/src/decoder/stream.rs @@ -317,7 +317,7 @@ mod tests { #[test] fn stream_whole_chunks_uncompressed() { - let (input, data) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -334,7 +334,7 @@ mod tests { #[test] fn stream_byte_chunks_uncompressed() { - let (input, data) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -353,8 +353,8 @@ mod tests { #[test] fn two_concatenated_streams() { - let (input1, data1) = test_data(EncodingOptions::UNCOMPRESSED, 16); - let (input2, data2) = test_data(EncodingOptions::UNCOMPRESSED, 16); + let (input1, data1) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); + let (input2, data2) = test_data(EncodingOptions::MSGPACK_UNCOMPRESSED, 16); let input = input1.into_iter().chain(input2).collect::>(); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -373,7 +373,7 @@ mod tests { #[test] fn stream_whole_chunks_compressed() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -390,7 +390,7 @@ mod tests { #[test] fn stream_byte_chunks_compressed() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); @@ -409,7 +409,7 @@ mod tests { #[test] fn stream_3x16_chunks() { - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut decoder = StreamDecoder::new(VersionPolicy::Error); let mut decoded_messages = vec![]; @@ -438,7 +438,7 @@ mod tests { fn stream_irregular_chunks() { // this attempts to stress-test `try_read` with chunks of various sizes - let (input, data) = test_data(EncodingOptions::COMPRESSED, 16); + let (input, data) = test_data(EncodingOptions::MSGPACK_COMPRESSED, 16); let mut data = Cursor::new(data); let mut decoder = StreamDecoder::new(VersionPolicy::Error); diff --git a/crates/store/re_log_encoding/src/encoder.rs b/crates/store/re_log_encoding/src/encoder.rs index edc966ce0d19..47fd4bdffa2e 100644 --- a/crates/store/re_log_encoding/src/encoder.rs +++ b/crates/store/re_log_encoding/src/encoder.rs @@ -1,12 +1,14 @@ //! Encoding of [`LogMsg`]es as a binary stream, e.g. to store in an `.rrd` file, or send over network. -use re_build_info::CrateVersion; -use re_chunk::{ChunkError, ChunkResult}; -use re_log_types::LogMsg; - +use crate::codec; +use crate::codec::file::{self, encoder}; use crate::FileHeader; use crate::MessageHeader; +use crate::Serializer; use crate::{Compression, EncodingOptions}; +use re_build_info::CrateVersion; +use re_chunk::{ChunkError, ChunkResult}; +use re_log_types::LogMsg; // ---------------------------------------------------------------------------- @@ -14,14 +16,23 @@ use crate::{Compression, EncodingOptions}; #[derive(thiserror::Error, Debug)] pub enum EncodeError { #[error("Failed to write: {0}")] - Write(std::io::Error), + Write(#[from] std::io::Error), #[error("lz4 error: {0}")] - Lz4(lz4_flex::block::CompressError), + Lz4(#[from] lz4_flex::block::CompressError), #[error("MsgPack error: {0}")] MsgPack(#[from] rmp_serde::encode::Error), + #[error("Protobuf error: {0}")] + Protobuf(#[from] re_protos::external::prost::EncodeError), + + #[error("Arrow error: {0}")] + Arrow(#[from] arrow2::error::Error), + + #[error("{0}")] + Codec(#[from] codec::CodecError), + #[error("Chunk error: {0}")] Chunk(#[from] ChunkError), @@ -109,6 +120,7 @@ impl std::ops::Drop for DroppableEncoder { /// Prefer [`DroppableEncoder`] if possible, make sure to call [`Encoder::finish`] when appropriate /// otherwise. pub struct Encoder { + serializer: Serializer, compression: Compression, write: W, uncompressed: Vec, @@ -128,15 +140,12 @@ impl Encoder { } .encode(&mut write)?; - match options.serializer { - crate::Serializer::MsgPack => {} - } - Ok(Self { + serializer: options.serializer, compression: options.compression, write, - uncompressed: vec![], - compressed: vec![], + uncompressed: Vec::new(), + compressed: Vec::new(), }) } @@ -145,36 +154,51 @@ impl Encoder { re_tracing::profile_function!(); self.uncompressed.clear(); - rmp_serde::encode::write_named(&mut self.uncompressed, message)?; + match self.serializer { + Serializer::Protobuf => { + encoder::encode(&mut self.uncompressed, message, self.compression)?; - match self.compression { - Compression::Off => { - MessageHeader::Data { - uncompressed_len: self.uncompressed.len() as u32, - compressed_len: self.uncompressed.len() as u32, - } - .encode(&mut self.write)?; self.write .write_all(&self.uncompressed) .map(|_| self.uncompressed.len() as _) .map_err(EncodeError::Write) } - - Compression::LZ4 => { - let max_len = lz4_flex::block::get_maximum_output_size(self.uncompressed.len()); - self.compressed.resize(max_len, 0); - let compressed_len = - lz4_flex::block::compress_into(&self.uncompressed, &mut self.compressed) + Serializer::MsgPack => { + rmp_serde::encode::write_named(&mut self.uncompressed, message)?; + + match self.compression { + Compression::Off => { + MessageHeader::Data { + uncompressed_len: self.uncompressed.len() as u32, + compressed_len: self.uncompressed.len() as u32, + } + .encode(&mut self.write)?; + self.write + .write_all(&self.uncompressed) + .map(|_| self.uncompressed.len() as _) + .map_err(EncodeError::Write) + } + + Compression::LZ4 => { + let max_len = + lz4_flex::block::get_maximum_output_size(self.uncompressed.len()); + self.compressed.resize(max_len, 0); + let compressed_len = lz4_flex::block::compress_into( + &self.uncompressed, + &mut self.compressed, + ) .map_err(EncodeError::Lz4)?; - MessageHeader::Data { - uncompressed_len: self.uncompressed.len() as u32, - compressed_len: compressed_len as u32, + MessageHeader::Data { + uncompressed_len: self.uncompressed.len() as u32, + compressed_len: compressed_len as u32, + } + .encode(&mut self.write)?; + self.write + .write_all(&self.compressed[..compressed_len]) + .map(|_| compressed_len as _) + .map_err(EncodeError::Write) + } } - .encode(&mut self.write)?; - self.write - .write_all(&self.compressed[..compressed_len]) - .map(|_| compressed_len as _) - .map_err(EncodeError::Write) } } } @@ -183,7 +207,18 @@ impl Encoder { // does a partial move. #[inline] pub fn finish(&mut self) -> Result<(), EncodeError> { - MessageHeader::EndOfStream.encode(&mut self.write)?; + match self.serializer { + Serializer::MsgPack => { + MessageHeader::EndOfStream.encode(&mut self.write)?; + } + Serializer::Protobuf => { + file::MessageHeader { + kind: file::MessageKind::End, + len: 0, + } + .encode(&mut self.write)?; + } + } Ok(()) } @@ -247,12 +282,20 @@ pub fn encode_as_bytes( #[inline] pub fn local_encoder() -> Result>, EncodeError> { - DroppableEncoder::new(CrateVersion::LOCAL, EncodingOptions::COMPRESSED, Vec::new()) + DroppableEncoder::new( + CrateVersion::LOCAL, + EncodingOptions::MSGPACK_COMPRESSED, + Vec::new(), + ) } #[inline] pub fn local_raw_encoder() -> Result>, EncodeError> { - Encoder::new(CrateVersion::LOCAL, EncodingOptions::COMPRESSED, Vec::new()) + Encoder::new( + CrateVersion::LOCAL, + EncodingOptions::MSGPACK_COMPRESSED, + Vec::new(), + ) } #[inline] diff --git a/crates/store/re_log_encoding/src/file_sink.rs b/crates/store/re_log_encoding/src/file_sink.rs index cd5e7d2d953f..1a80625987f3 100644 --- a/crates/store/re_log_encoding/src/file_sink.rs +++ b/crates/store/re_log_encoding/src/file_sink.rs @@ -61,7 +61,7 @@ impl FileSink { /// Start writing log messages to a file at the given path. pub fn new(path: impl Into) -> Result { // We always compress on disk - let encoding_options = crate::EncodingOptions::COMPRESSED; + let encoding_options = crate::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); @@ -91,7 +91,7 @@ impl FileSink { /// Start writing log messages to standard output. pub fn stdout() -> Result { - let encoding_options = crate::EncodingOptions::COMPRESSED; + let encoding_options = crate::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); diff --git a/crates/store/re_log_encoding/src/lib.rs b/crates/store/re_log_encoding/src/lib.rs index fb0cbbbfe5b7..d35b8e331342 100644 --- a/crates/store/re_log_encoding/src/lib.rs +++ b/crates/store/re_log_encoding/src/lib.rs @@ -2,11 +2,14 @@ #[cfg(feature = "decoder")] pub mod decoder; + #[cfg(feature = "encoder")] pub mod encoder; pub mod codec; +mod protobuf_conversions; + #[cfg(feature = "encoder")] #[cfg(not(target_arch = "wasm32"))] mod file_sink; @@ -14,6 +17,20 @@ mod file_sink; #[cfg(feature = "stream_from_http")] pub mod stream_rrd_from_http; +/// How to handle version mismatches during decoding. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum VersionPolicy { + /// Warn if the versions don't match, but continue loading. + /// + /// We usually use this for loading `.rrd` recordings. + Warn, + + /// Return an error if the versions aren't compatible. + /// + /// We usually use this for tests, and for loading `.rbl` blueprint files. + Error, +} + // --------------------------------------------------------------------- #[cfg(feature = "encoder")] @@ -45,6 +62,7 @@ pub enum Compression { #[repr(u8)] pub enum Serializer { MsgPack = 1, + Protobuf = 2, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -54,14 +72,18 @@ pub struct EncodingOptions { } impl EncodingOptions { - pub const UNCOMPRESSED: Self = Self { + pub const MSGPACK_UNCOMPRESSED: Self = Self { compression: Compression::Off, serializer: Serializer::MsgPack, }; - pub const COMPRESSED: Self = Self { + pub const MSGPACK_COMPRESSED: Self = Self { compression: Compression::LZ4, serializer: Serializer::MsgPack, }; + pub const PROTOBUF_COMPRESSED: Self = Self { + compression: Compression::LZ4, + serializer: Serializer::Protobuf, + }; pub fn from_bytes(bytes: [u8; 4]) -> Result { match bytes { @@ -73,6 +95,7 @@ impl EncodingOptions { }; let serializer = match serializer { 1 => Serializer::MsgPack, + 2 => Serializer::Protobuf, _ => return Err(OptionsError::UnknownSerializer(serializer)), }; Ok(Self { diff --git a/crates/store/re_log_encoding/src/protobuf_conversions.rs b/crates/store/re_log_encoding/src/protobuf_conversions.rs new file mode 100644 index 000000000000..9b613a1e3937 --- /dev/null +++ b/crates/store/re_log_encoding/src/protobuf_conversions.rs @@ -0,0 +1,17 @@ +impl From for crate::Compression { + fn from(value: re_protos::log_msg::v0::Compression) -> Self { + match value { + re_protos::log_msg::v0::Compression::None => Self::Off, + re_protos::log_msg::v0::Compression::Lz4 => Self::LZ4, + } + } +} + +impl From for re_protos::log_msg::v0::Compression { + fn from(value: crate::Compression) -> Self { + match value { + crate::Compression::Off => Self::None, + crate::Compression::LZ4 => Self::Lz4, + } + } +} diff --git a/crates/store/re_log_encoding/src/stream_rrd_from_http.rs b/crates/store/re_log_encoding/src/stream_rrd_from_http.rs index 718a8f8bb479..a52283c4fb05 100644 --- a/crates/store/re_log_encoding/src/stream_rrd_from_http.rs +++ b/crates/store/re_log_encoding/src/stream_rrd_from_http.rs @@ -71,7 +71,7 @@ pub fn stream_rrd_from_http(url: String, on_msg: Arc) { re_log::debug!("Downloading .rrd file from {url:?}…"); ehttp::streaming::fetch(ehttp::Request::get(&url), { - let version_policy = crate::decoder::VersionPolicy::Warn; + let version_policy = crate::VersionPolicy::Warn; let decoder = RefCell::new(StreamDecoder::new(version_policy)); move |part| match part { Ok(part) => match part { @@ -184,7 +184,7 @@ pub mod web_decode { async fn decode_rrd_async(rrd_bytes: Vec, on_msg: Arc) { let mut last_yield = web_time::Instant::now(); - let version_policy = crate::decoder::VersionPolicy::Warn; + let version_policy = crate::VersionPolicy::Warn; match crate::decoder::Decoder::new(version_policy, rrd_bytes.as_slice()) { Ok(decoder) => { for msg in decoder { diff --git a/crates/store/re_log_types/src/lib.rs b/crates/store/re_log_types/src/lib.rs index 3b708c10a2f1..3174228b7750 100644 --- a/crates/store/re_log_types/src/lib.rs +++ b/crates/store/re_log_types/src/lib.rs @@ -407,6 +407,67 @@ impl std::fmt::Display for PythonVersion { } } +impl std::str::FromStr for PythonVersion { + type Err = PythonVersionParseError; + + fn from_str(s: &str) -> Result { + if s.is_empty() { + return Err(PythonVersionParseError::MissingMajor); + } + let (major, rest) = s + .split_once('.') + .ok_or(PythonVersionParseError::MissingMinor)?; + if rest.is_empty() { + return Err(PythonVersionParseError::MissingMinor); + } + let (minor, rest) = rest + .split_once('.') + .ok_or(PythonVersionParseError::MissingPatch)?; + if rest.is_empty() { + return Err(PythonVersionParseError::MissingPatch); + } + let pos = rest.bytes().position(|v| !v.is_ascii_digit()); + let (patch, suffix) = match pos { + Some(pos) => rest.split_at(pos), + None => (rest, ""), + }; + + Ok(Self { + major: major + .parse() + .map_err(PythonVersionParseError::InvalidMajor)?, + minor: minor + .parse() + .map_err(PythonVersionParseError::InvalidMinor)?, + patch: patch + .parse() + .map_err(PythonVersionParseError::InvalidPatch)?, + suffix: suffix.into(), + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum PythonVersionParseError { + #[error("missing major version")] + MissingMajor, + + #[error("missing minor version")] + MissingMinor, + + #[error("missing patch version")] + MissingPatch, + + #[error("invalid major version: {0}")] + InvalidMajor(std::num::ParseIntError), + + #[error("invalid minor version: {0}")] + InvalidMinor(std::num::ParseIntError), + + #[error("invalid patch version: {0}")] + InvalidPatch(std::num::ParseIntError), +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub enum FileSource { @@ -573,3 +634,57 @@ pub fn build_frame_nr(frame_nr: impl TryInto) -> (Timeline, TimeInt) { frame_nr.try_into().unwrap_or(TimeInt::MIN), ) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_version() { + macro_rules! assert_parse_err { + ($input:literal, $expected:pat) => { + let actual = $input.parse::(); + + assert!( + matches!(actual, Err($expected)), + "actual: {actual:?}, expected: {}", + stringify!($expected) + ); + }; + } + + macro_rules! assert_parse_ok { + ($input:literal, $expected:expr) => { + let actual = $input.parse::().expect("failed to parse"); + assert_eq!(actual, $expected); + }; + } + + assert_parse_err!("", PythonVersionParseError::MissingMajor); + assert_parse_err!("3", PythonVersionParseError::MissingMinor); + assert_parse_err!("3.", PythonVersionParseError::MissingMinor); + assert_parse_err!("3.11", PythonVersionParseError::MissingPatch); + assert_parse_err!("3.11.", PythonVersionParseError::MissingPatch); + assert_parse_err!("a.11.0", PythonVersionParseError::InvalidMajor(_)); + assert_parse_err!("3.b.0", PythonVersionParseError::InvalidMinor(_)); + assert_parse_err!("3.11.c", PythonVersionParseError::InvalidPatch(_)); + assert_parse_ok!( + "3.11.0", + PythonVersion { + major: 3, + minor: 11, + patch: 0, + suffix: String::new(), + } + ); + assert_parse_ok!( + "3.11.0a1", + PythonVersion { + major: 3, + minor: 11, + patch: 0, + suffix: "a1".to_owned(), + } + ); + } +} diff --git a/crates/store/re_log_types/src/protobuf_conversions.rs b/crates/store/re_log_types/src/protobuf_conversions.rs index 80993323e529..aab629292512 100644 --- a/crates/store/re_log_types/src/protobuf_conversions.rs +++ b/crates/store/re_log_types/src/protobuf_conversions.rs @@ -1,6 +1,6 @@ -use std::sync::Arc; - use re_protos::TypeConversionError; +use re_protos::{invalid_field, missing_field}; +use std::sync::Arc; impl From for re_protos::common::v0::EntityPath { fn from(value: crate::EntityPath) -> Self { @@ -14,11 +14,8 @@ impl TryFrom for crate::EntityPath { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::EntityPath) -> Result { - Self::parse_strict(&value.path).map_err(|err| TypeConversionError::InvalidField { - type_name: "rerun.common.v0.EntityPath", - field_name: "path", - reason: err.to_string(), - }) + Self::parse_strict(&value.path) + .map_err(|err| invalid_field!(re_protos::common::v0::EntityPath, "path", err)) } } @@ -82,35 +79,47 @@ impl TryFrom for crate::ResolvedTimeRange { fn try_from(value: re_protos::common::v0::IndexRange) -> Result { value .time_range - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.IndexRange", - "time_range", + .ok_or(missing_field!( + re_protos::common::v0::IndexRange, + "time_range" )) .map(|time_range| Self::new(time_range.start, time_range.end)) } } +impl From for crate::Timeline { + fn from(value: re_protos::common::v0::Timeline) -> Self { + // TODO(cmc): QueryExpression::filtered_index gotta be a selector + #[allow(clippy::match_same_arms)] + match value.name.as_str() { + "log_time" => Self::new_temporal(value.name), + "log_tick" => Self::new_sequence(value.name), + "frame" => Self::new_sequence(value.name), + "frame_nr" => Self::new_sequence(value.name), + _ => Self::new_temporal(value.name), + } + } +} + +impl From for re_protos::common::v0::Timeline { + fn from(value: crate::Timeline) -> Self { + Self { + name: value.name().to_string(), + } + } +} + impl TryFrom for crate::Timeline { type Error = TypeConversionError; fn try_from(value: re_protos::common::v0::IndexColumnSelector) -> Result { - let timeline_name = value + let timeline = value .timeline - .ok_or(TypeConversionError::missing_field( - "rerun.common.v0.IndexColumnSelector", - "timeline", + .ok_or(missing_field!( + re_protos::common::v0::IndexColumnSelector, + "timeline" ))? - .name; - - // TODO(cmc): QueryExpression::filtered_index gotta be a selector - #[allow(clippy::match_same_arms)] - let timeline = match timeline_name.as_str() { - "log_time" => Self::new_temporal(timeline_name), - "log_tick" => Self::new_sequence(timeline_name), - "frame" => Self::new_sequence(timeline_name), - "frame_nr" => Self::new_sequence(timeline_name), - _ => Self::new_temporal(timeline_name), - }; + .into(); Ok(timeline) } @@ -154,7 +163,7 @@ impl From for crate::StoreId { #[inline] fn from(value: re_protos::common::v0::StoreId) -> Self { Self { - kind: crate::StoreKind::Recording, + kind: value.kind().into(), id: Arc::new(value.id), } } @@ -189,3 +198,501 @@ impl From for re_protos::common::v0::RecordingId { } } } + +impl From for re_protos::log_msg::v0::StoreSource { + #[inline] + fn from(value: crate::StoreSource) -> Self { + use re_protos::external::prost::Message as _; + + let (kind, payload) = match value { + crate::StoreSource::Unknown => ( + re_protos::log_msg::v0::StoreSourceKind::UnknownKind as i32, + Vec::new(), + ), + crate::StoreSource::CSdk => ( + re_protos::log_msg::v0::StoreSourceKind::CSdk as i32, + Vec::new(), + ), + crate::StoreSource::PythonSdk(python_version) => ( + re_protos::log_msg::v0::StoreSourceKind::PythonSdk as i32, + re_protos::log_msg::v0::PythonVersion::from(python_version).encode_to_vec(), + ), + crate::StoreSource::RustSdk { + rustc_version, + llvm_version, + } => ( + re_protos::log_msg::v0::StoreSourceKind::RustSdk as i32, + re_protos::log_msg::v0::CrateInfo { + rustc_version, + llvm_version, + } + .encode_to_vec(), + ), + crate::StoreSource::File { file_source } => ( + re_protos::log_msg::v0::StoreSourceKind::File as i32, + re_protos::log_msg::v0::FileSource::from(file_source).encode_to_vec(), + ), + crate::StoreSource::Viewer => ( + re_protos::log_msg::v0::StoreSourceKind::Viewer as i32, + Vec::new(), + ), + crate::StoreSource::Other(description) => ( + re_protos::log_msg::v0::StoreSourceKind::Other as i32, + description.into_bytes(), + ), + }; + + Self { + kind, + extra: Some(re_protos::log_msg::v0::StoreSourceExtra { payload }), + } + } +} + +impl TryFrom for crate::StoreSource { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::StoreSource) -> Result { + use re_protos::external::prost::Message as _; + use re_protos::log_msg::v0::StoreSourceKind; + + match value.kind() { + StoreSourceKind::UnknownKind => Ok(Self::Unknown), + StoreSourceKind::CSdk => Ok(Self::CSdk), + StoreSourceKind::PythonSdk => { + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; + let python_version = + re_protos::log_msg::v0::PythonVersion::decode(&mut &extra.payload[..])?; + Ok(Self::PythonSdk(crate::PythonVersion::try_from( + python_version, + )?)) + } + StoreSourceKind::RustSdk => { + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; + let crate_info = + re_protos::log_msg::v0::CrateInfo::decode(&mut &extra.payload[..])?; + Ok(Self::RustSdk { + rustc_version: crate_info.rustc_version, + llvm_version: crate_info.llvm_version, + }) + } + StoreSourceKind::File => { + let extra = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; + let file_source = + re_protos::log_msg::v0::FileSource::decode(&mut &extra.payload[..])?; + Ok(Self::File { + file_source: crate::FileSource::try_from(file_source)?, + }) + } + StoreSourceKind::Viewer => Ok(Self::Viewer), + StoreSourceKind::Other => { + let description = value + .extra + .ok_or(missing_field!(re_protos::log_msg::v0::StoreSource, "extra"))?; + let description = String::from_utf8(description.payload).map_err(|err| { + invalid_field!(re_protos::log_msg::v0::StoreSource, "extra", err) + })?; + Ok(Self::Other(description)) + } + } + } +} + +impl From for re_protos::log_msg::v0::PythonVersion { + #[inline] + fn from(value: crate::PythonVersion) -> Self { + Self { + major: value.major as i32, + minor: value.minor as i32, + patch: value.patch as i32, + suffix: value.suffix, + } + } +} + +impl TryFrom for crate::PythonVersion { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::PythonVersion) -> Result { + Ok(Self { + major: value.major as u8, + minor: value.minor as u8, + patch: value.patch as u8, + suffix: value.suffix, + }) + } +} + +impl From for re_protos::log_msg::v0::FileSource { + #[inline] + fn from(value: crate::FileSource) -> Self { + let kind = match value { + crate::FileSource::Cli => re_protos::log_msg::v0::FileSourceKind::Cli as i32, + crate::FileSource::Uri => re_protos::log_msg::v0::FileSourceKind::Uri as i32, + crate::FileSource::DragAndDrop { .. } => { + re_protos::log_msg::v0::FileSourceKind::DragAndDrop as i32 + } + crate::FileSource::FileDialog { .. } => { + re_protos::log_msg::v0::FileSourceKind::FileDialog as i32 + } + crate::FileSource::Sdk => re_protos::log_msg::v0::FileSourceKind::Sdk as i32, + }; + + Self { kind } + } +} + +impl TryFrom for crate::FileSource { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::FileSource) -> Result { + use re_protos::log_msg::v0::FileSourceKind; + + match value.kind() { + FileSourceKind::Cli => Ok(Self::Cli), + FileSourceKind::Uri => Ok(Self::Uri), + FileSourceKind::DragAndDrop => Ok(Self::DragAndDrop { + recommended_application_id: None, + recommended_recording_id: None, + force_store_info: false, + }), + FileSourceKind::FileDialog => Ok(Self::FileDialog { + recommended_application_id: None, + recommended_recording_id: None, + force_store_info: false, + }), + FileSourceKind::Sdk => Ok(Self::Sdk), + FileSourceKind::UnknownSource => Err(invalid_field!( + re_protos::log_msg::v0::FileSource, + "kind", + "unknown kind", + )), + } + } +} + +impl From for re_protos::log_msg::v0::StoreInfo { + #[inline] + fn from(value: crate::StoreInfo) -> Self { + Self { + application_id: Some(value.application_id.into()), + store_id: Some(value.store_id.into()), + is_official_example: value.is_official_example, + started: Some(value.started.into()), + store_source: Some(value.store_source.into()), + store_version: value + .store_version + .map(|v| re_protos::log_msg::v0::StoreVersion { + crate_version_bits: i32::from_le_bytes(v.to_bytes()), + }), + } + } +} + +impl TryFrom for crate::StoreInfo { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::StoreInfo) -> Result { + let application_id: crate::ApplicationId = value + .application_id + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, + "application_id", + ))? + .into(); + let store_id: crate::StoreId = value + .store_id + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, + "store_id", + ))? + .into(); + let is_official_example = value.is_official_example; + let started: crate::Time = value + .started + .ok_or(missing_field!(re_protos::log_msg::v0::StoreInfo, "started"))? + .into(); + let store_source: crate::StoreSource = value + .store_source + .ok_or(missing_field!( + re_protos::log_msg::v0::StoreInfo, + "store_source", + ))? + .try_into()?; + let store_version = value + .store_version + .map(|v| re_build_info::CrateVersion::from_bytes(v.crate_version_bits.to_le_bytes())); + + Ok(Self { + application_id, + store_id, + cloned_from: None, + is_official_example, + started, + store_source, + store_version, + }) + } +} + +impl From for re_protos::log_msg::v0::SetStoreInfo { + #[inline] + fn from(value: crate::SetStoreInfo) -> Self { + Self { + row_id: Some(value.row_id.into()), + info: Some(value.info.into()), + } + } +} + +impl TryFrom for crate::SetStoreInfo { + type Error = TypeConversionError; + + #[inline] + fn try_from(value: re_protos::log_msg::v0::SetStoreInfo) -> Result { + Ok(Self { + row_id: value + .row_id + .ok_or(missing_field!( + re_protos::log_msg::v0::SetStoreInfo, + "row_id", + ))? + .into(), + info: value + .info + .ok_or(missing_field!(re_protos::log_msg::v0::SetStoreInfo, "info"))? + .try_into()?, + }) + } +} + +impl From + for re_protos::log_msg::v0::BlueprintActivationCommand +{ + #[inline] + fn from(value: crate::BlueprintActivationCommand) -> Self { + Self { + blueprint_id: Some(value.blueprint_id.into()), + make_active: value.make_active, + make_default: value.make_default, + } + } +} + +impl TryFrom + for crate::BlueprintActivationCommand +{ + type Error = TypeConversionError; + + #[inline] + fn try_from( + value: re_protos::log_msg::v0::BlueprintActivationCommand, + ) -> Result { + Ok(Self { + blueprint_id: value + .blueprint_id + .ok_or(missing_field!( + re_protos::log_msg::v0::BlueprintActivationCommand, + "blueprint_id", + ))? + .into(), + make_active: value.make_active, + make_default: value.make_default, + }) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn entity_path_conversion() { + let entity_path = crate::EntityPath::parse_strict("a/b/c").unwrap(); + let proto_entity_path: re_protos::common::v0::EntityPath = entity_path.clone().into(); + let entity_path2: crate::EntityPath = proto_entity_path.try_into().unwrap(); + assert_eq!(entity_path, entity_path2); + } + + #[test] + fn time_conversion() { + let time = crate::Time::from_ns_since_epoch(123456789); + let proto_time: re_protos::common::v0::Time = time.into(); + let time2: crate::Time = proto_time.into(); + assert_eq!(time, time2); + } + + #[test] + fn time_int_conversion() { + let time_int = crate::TimeInt::new_temporal(123456789); + let proto_time_int: re_protos::common::v0::TimeInt = time_int.into(); + let time_int2: crate::TimeInt = proto_time_int.into(); + assert_eq!(time_int, time_int2); + } + + #[test] + fn time_range_conversion() { + let time_range = crate::ResolvedTimeRange::new( + crate::TimeInt::new_temporal(123456789), + crate::TimeInt::new_temporal(987654321), + ); + let proto_time_range: re_protos::common::v0::TimeRange = time_range.into(); + let time_range2: crate::ResolvedTimeRange = proto_time_range.into(); + assert_eq!(time_range, time_range2); + } + + #[test] + fn index_range_conversion() { + let time_range = crate::ResolvedTimeRange::new( + crate::TimeInt::new_temporal(123456789), + crate::TimeInt::new_temporal(987654321), + ); + let proto_index_range: re_protos::common::v0::IndexRange = time_range.into(); + let time_range2: crate::ResolvedTimeRange = proto_index_range.try_into().unwrap(); + assert_eq!(time_range, time_range2); + } + + #[test] + fn index_column_selector_conversion() { + let timeline = crate::Timeline::new_temporal("log_time"); + let proto_index_column_selector: re_protos::common::v0::IndexColumnSelector = + re_protos::common::v0::IndexColumnSelector { + timeline: Some(timeline.into()), + }; + let timeline2: crate::Timeline = proto_index_column_selector.try_into().unwrap(); + assert_eq!(timeline, timeline2); + } + + #[test] + fn application_id_conversion() { + let application_id = crate::ApplicationId("test".to_owned()); + let proto_application_id: re_protos::common::v0::ApplicationId = + application_id.clone().into(); + let application_id2: crate::ApplicationId = proto_application_id.into(); + assert_eq!(application_id, application_id2); + } + + #[test] + fn store_kind_conversion() { + let store_kind = crate::StoreKind::Recording; + let proto_store_kind: re_protos::common::v0::StoreKind = store_kind.into(); + let store_kind2: crate::StoreKind = proto_store_kind.into(); + assert_eq!(store_kind, store_kind2); + } + + #[test] + fn store_id_conversion() { + let store_id = + crate::StoreId::from_string(crate::StoreKind::Recording, "test_recording".to_owned()); + let proto_store_id: re_protos::common::v0::StoreId = store_id.clone().into(); + let store_id2: crate::StoreId = proto_store_id.into(); + assert_eq!(store_id, store_id2); + } + + #[test] + fn recording_id_conversion() { + let store_id = + crate::StoreId::from_string(crate::StoreKind::Recording, "test_recording".to_owned()); + let proto_recording_id: re_protos::common::v0::RecordingId = store_id.clone().into(); + let store_id2: crate::StoreId = proto_recording_id.into(); + assert_eq!(store_id, store_id2); + } + + #[test] + fn store_source_conversion() { + let store_source = crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }); + let proto_store_source: re_protos::log_msg::v0::StoreSource = store_source.clone().into(); + let store_source2: crate::StoreSource = proto_store_source.try_into().unwrap(); + assert_eq!(store_source, store_source2); + } + + #[test] + fn file_source_conversion() { + let file_source = crate::FileSource::Uri; + let proto_file_source: re_protos::log_msg::v0::FileSource = file_source.clone().into(); + let file_source2: crate::FileSource = proto_file_source.try_into().unwrap(); + assert_eq!(file_source, file_source2); + } + + #[test] + fn store_info_conversion() { + let store_info = crate::StoreInfo { + application_id: crate::ApplicationId("test".to_owned()), + store_id: crate::StoreId::from_string( + crate::StoreKind::Recording, + "test_recording".to_owned(), + ), + cloned_from: None, + is_official_example: false, + started: crate::Time::now(), + store_source: crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }), + store_version: None, + }; + let proto_store_info: re_protos::log_msg::v0::StoreInfo = store_info.clone().into(); + let store_info2: crate::StoreInfo = proto_store_info.try_into().unwrap(); + assert_eq!(store_info, store_info2); + } + + #[test] + fn set_store_info_conversion() { + let set_store_info = crate::SetStoreInfo { + row_id: re_tuid::Tuid::new(), + info: crate::StoreInfo { + application_id: crate::ApplicationId("test".to_owned()), + store_id: crate::StoreId::from_string( + crate::StoreKind::Recording, + "test_recording".to_owned(), + ), + cloned_from: None, + is_official_example: false, + started: crate::Time::now(), + store_source: crate::StoreSource::PythonSdk(crate::PythonVersion { + major: 3, + minor: 8, + patch: 0, + suffix: "a".to_owned(), + }), + store_version: None, + }, + }; + let proto_set_store_info: re_protos::log_msg::v0::SetStoreInfo = + set_store_info.clone().into(); + let set_store_info2: crate::SetStoreInfo = proto_set_store_info.try_into().unwrap(); + assert_eq!(set_store_info, set_store_info2); + } + + #[test] + fn blueprint_activation_command_conversion() { + let blueprint_activation_command = crate::BlueprintActivationCommand { + blueprint_id: crate::StoreId::from_string( + crate::StoreKind::Blueprint, + "test".to_owned(), + ), + make_active: true, + make_default: false, + }; + let proto_blueprint_activation_command: re_protos::log_msg::v0::BlueprintActivationCommand = + blueprint_activation_command.clone().into(); + let blueprint_activation_command2: crate::BlueprintActivationCommand = + proto_blueprint_activation_command.try_into().unwrap(); + assert_eq!(blueprint_activation_command, blueprint_activation_command2); + } +} diff --git a/crates/store/re_protos/.gitattributes b/crates/store/re_protos/.gitattributes index be213900d102..7dd060db3f46 100644 --- a/crates/store/re_protos/.gitattributes +++ b/crates/store/re_protos/.gitattributes @@ -1 +1 @@ -src/v0/rerun.remote_store.v0.rs linguist-generated=true +src/v0/** linguist-generated=true diff --git a/crates/store/re_protos/proto/rerun/v0/log_msg.proto b/crates/store/re_protos/proto/rerun/v0/log_msg.proto new file mode 100644 index 000000000000..b35abec50a58 --- /dev/null +++ b/crates/store/re_protos/proto/rerun/v0/log_msg.proto @@ -0,0 +1,190 @@ +syntax = "proto3"; + +package rerun.log_msg.v0; + +import "rerun/v0/common.proto"; + +// Corresponds to `LogMsg::SetStoreInfo`. Used to identify a recording. +message SetStoreInfo { + // A time-based UID that is used to determine how a `StoreInfo` fits in the global ordering of events. + rerun.common.v0.Tuid row_id = 1; + + // The new store info. + StoreInfo info = 2; +} + +// The type of compression used on the payload. +enum Compression { + // No compression. + NONE = 0; + + // LZ4 block compression. + LZ4 = 1; +} + +// The encoding of the message payload. +enum Encoding { + // We don't know what encoding the payload is in. + UNKNOWN = 0; + + // The payload is encoded as Arrow-IPC. + ARROW_IPC = 1; +} + +// Corresponds to `LogMsg::ArrowMsg`. Used to transmit actual data. +message ArrowMsg { + // The ID of the store that this message is for. + rerun.common.v0.StoreId store_id = 1; + + // Compression algorithm used. + Compression compression = 2; + + int32 uncompressed_size = 3; + + // Encoding of the payload. + Encoding encoding = 4; + + // Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. + bytes payload = 1000; +} + +// Corresponds to `LogMsg::BlueprintActivationCommand`. +// +// Used for activating a blueprint once it has been fully transmitted, +// because showing a blueprint before it is fully transmitted can lead to +// a confusing user experience, or inconsistent results due to heuristics. +message BlueprintActivationCommand { + // The ID of the blueprint to activate. + rerun.common.v0.StoreId blueprint_id = 1; + + // Whether to make the blueprint active immediately. + bool make_active = 2; + + // Whether to make the blueprint the default. + bool make_default = 3; +} + +// Information about a recording or blueprint. +message StoreInfo { + // User-chosen name of the application doing the logging. + rerun.common.v0.ApplicationId application_id = 1; + + // Unique ID of the recording. + rerun.common.v0.StoreId store_id = 2; + + /// True if the recording is one of the official Rerun examples. + bool is_official_example = 3; + + // When the recording started. + rerun.common.v0.Time started = 4; + + // Where the recording came from. + StoreSource store_source = 5; + + // Version of the store crate. + StoreVersion store_version = 6; +} + +// The source of a recording or blueprint. +message StoreSource { + // Determines what is encoded in `extra`. + StoreSourceKind kind = 1; + + // Store source payload. See `StoreSourceKind` for what exactly is encoded here. + StoreSourceExtra extra = 2; +} + +// A newtype for `StoreSource` payload. +// +// This exists to that we can implement conversions on the newtype for convenience. +message StoreSourceExtra { + bytes payload = 1; +} + +// What kind of source a recording comes from. +enum StoreSourceKind { + // We don't know anything about the source of this recording. + // + // `extra` is unused. + UNKNOWN_KIND = 0; + + // The recording came from the C++ SDK. + // + // `extra` is unused. + C_SDK = 1; + + // The recording came from the Python SDK. + // + // `extra` is `PythonVersion`. + PYTHON_SDK = 2; + + // The recording came from the Rust SDK. + // + // `extra` is `CrateInfo`. + RUST_SDK = 3; + + // The recording came from a file. + // + // `extra` is `FileSource`. + FILE = 4; + + // The recording came from some action in the viewer. + // + // `extra` is unused. + VIEWER = 5; + + // The recording came from some other source. + // + // `extra` is a string. + OTHER = 6; +} + +// Version of the Python SDK that created the recording. +message PythonVersion { + int32 major = 1; + int32 minor = 2; + int32 patch = 3; + string suffix = 4; +} + +// Information about the Rust SDK that created the recording. +message CrateInfo { + // Version of the Rust compiler used to compile the SDK. + string rustc_version = 1; + + // Version of LLVM used by the Rust compiler. + string llvm_version = 2; +} + +// A recording which came from a file. +message FileSource { + FileSourceKind kind = 1; +} + +// Determines where the file came from. +enum FileSourceKind { + // We don't know where the file came from. + UNKNOWN_SOURCE = 0; + + // The file came from the command line. + CLI = 1; + + // The file was served over HTTP. + URI = 2; + + // The file was dragged into the viewer. + DRAG_AND_DROP = 3; + + // The file was opened using a file dialog. + FILE_DIALOG = 4; + + // The recording was produced using a data loader, such as when logging a mesh file. + SDK = 5; +} + +message StoreVersion { + // Crate version encoded using our custom scheme. + // + // See `CrateVersion` in `re_build_info`. + int32 crate_version_bits = 1; +} diff --git a/crates/store/re_protos/src/lib.rs b/crates/store/re_protos/src/lib.rs index 35da3c09c82a..545198db999b 100644 --- a/crates/store/re_protos/src/lib.rs +++ b/crates/store/re_protos/src/lib.rs @@ -24,6 +24,9 @@ mod v0 { #[path = "./rerun.common.v0.rs"] pub mod rerun_common_v0; + #[path = "./rerun.log_msg.v0.rs"] + pub mod rerun_log_msg_v0; + #[path = "./rerun.remote_store.v0.rs"] pub mod rerun_remote_store_v0; } @@ -34,6 +37,12 @@ pub mod common { } } +pub mod log_msg { + pub mod v0 { + pub use crate::v0::rerun_log_msg_v0::*; + } +} + /// Generated types for the remote store gRPC service API v0. pub mod remote_store { pub mod v0 { @@ -43,14 +52,16 @@ pub mod remote_store { #[derive(Debug, thiserror::Error)] pub enum TypeConversionError { - #[error("missing required field: {type_name}.{field_name}")] + #[error("missing required field: {package_name}.{type_name}.{field_name}")] MissingField { + package_name: &'static str, type_name: &'static str, field_name: &'static str, }, - #[error("invalid value for field {type_name}.{field_name}: {reason}")] + #[error("invalid value for field {package_name}.{type_name}.{field_name}: {reason}")] InvalidField { + package_name: &'static str, type_name: &'static str, field_name: &'static str, reason: String, @@ -67,10 +78,37 @@ pub enum TypeConversionError { } impl TypeConversionError { - pub fn missing_field(type_name: &'static str, field_name: &'static str) -> Self { + #[inline] + pub fn missing_field(field_name: &'static str) -> Self { Self::MissingField { - type_name, + package_name: T::PACKAGE, + type_name: T::NAME, field_name, } } + + #[allow(clippy::needless_pass_by_value)] // false-positive + #[inline] + pub fn invalid_field(field_name: &'static str, reason: &impl ToString) -> Self { + Self::InvalidField { + package_name: T::PACKAGE, + type_name: T::NAME, + field_name, + reason: reason.to_string(), + } + } +} + +#[macro_export] +macro_rules! missing_field { + ($type:ty, $field:expr $(,)?) => { + $crate::TypeConversionError::missing_field::<$type>($field) + }; +} + +#[macro_export] +macro_rules! invalid_field { + ($type:ty, $field:expr, $reason:expr $(,)?) => { + $crate::TypeConversionError::invalid_field::<$type>($field, &$reason) + }; } diff --git a/crates/store/re_protos/src/v0/rerun.common.v0.rs b/crates/store/re_protos/src/v0/rerun.common.v0.rs index d06d839e0c65..99bcbbd61774 100644 --- a/crates/store/re_protos/src/v0/rerun.common.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.common.v0.rs @@ -11,18 +11,48 @@ pub struct RerunChunk { #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for RerunChunk { + const NAME: &'static str = "RerunChunk"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.RerunChunk".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.RerunChunk".into() + } +} /// unique recording identifier. At this point in time it is the same id as the ChunkStore's StoreId #[derive(Clone, PartialEq, ::prost::Message)] pub struct RecordingId { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for RecordingId { + const NAME: &'static str = "RecordingId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.RecordingId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.RecordingId".into() + } +} /// A recording can have multiple timelines, each is identified by a name, for example `log_tick`, `log_time`, etc. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Timeline { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } +impl ::prost::Name for Timeline { + const NAME: &'static str = "Timeline"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Timeline".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Timeline".into() + } +} /// A time range between start and end time points. Each 64 bit number can represent different time point data /// depending on the timeline it is associated with. Time range is inclusive for both start and end time points. #[derive(Clone, Copy, PartialEq, ::prost::Message)] @@ -32,12 +62,32 @@ pub struct TimeRange { #[prost(int64, tag = "2")] pub end: i64, } +impl ::prost::Name for TimeRange { + const NAME: &'static str = "TimeRange"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeRange".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeRange".into() + } +} /// arrow IPC serialized schema #[derive(Clone, PartialEq, ::prost::Message)] pub struct Schema { #[prost(bytes = "vec", tag = "1")] pub arrow_schema: ::prost::alloc::vec::Vec, } +impl ::prost::Name for Schema { + const NAME: &'static str = "Schema"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Schema".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Schema".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct Query { /// The subset of the database that the query will run on: a set of EntityPath(s) and their @@ -100,11 +150,31 @@ pub struct Query { #[prost(enumeration = "SparseFillStrategy", tag = "11")] pub sparse_fill_strategy: i32, } +impl ::prost::Name for Query { + const NAME: &'static str = "Query"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Query".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Query".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnSelection { #[prost(message, repeated, tag = "1")] pub columns: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ColumnSelection { + const NAME: &'static str = "ColumnSelection"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ColumnSelection".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ColumnSelection".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnSelector { #[prost(oneof = "column_selector::SelectorType", tags = "2, 3")] @@ -120,40 +190,110 @@ pub mod column_selector { TimeColumn(super::TimeColumnSelector), } } +impl ::prost::Name for ColumnSelector { + const NAME: &'static str = "ColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ColumnSelector".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct IndexColumnSelector { /// TODO(zehiko) we need to add support for other types of index selectors #[prost(message, optional, tag = "1")] pub timeline: ::core::option::Option, } +impl ::prost::Name for IndexColumnSelector { + const NAME: &'static str = "IndexColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexColumnSelector".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct IndexRange { /// TODO(zehiko) support for other ranges for other index selectors #[prost(message, optional, tag = "1")] pub time_range: ::core::option::Option, } +impl ::prost::Name for IndexRange { + const NAME: &'static str = "IndexRange"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexRange".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexRange".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct IndexValues { /// TODO(zehiko) we need to add support for other types of index selectors #[prost(message, repeated, tag = "1")] pub time_points: ::prost::alloc::vec::Vec, } +impl ::prost::Name for IndexValues { + const NAME: &'static str = "IndexValues"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.IndexValues".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.IndexValues".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct SampledIndexValues { #[prost(message, repeated, tag = "1")] pub sample_points: ::prost::alloc::vec::Vec, } +impl ::prost::Name for SampledIndexValues { + const NAME: &'static str = "SampledIndexValues"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.SampledIndexValues".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.SampledIndexValues".into() + } +} /// A 64-bit number describing either nanoseconds, sequence numbers or fully static data. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct TimeInt { #[prost(int64, tag = "1")] pub time: i64, } +impl ::prost::Name for TimeInt { + const NAME: &'static str = "TimeInt"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeInt".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeInt".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ViewContents { #[prost(message, repeated, tag = "1")] pub contents: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ViewContents { + const NAME: &'static str = "ViewContents"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ViewContents".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ViewContents".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ViewContentsPart { #[prost(message, optional, tag = "1")] @@ -161,11 +301,31 @@ pub struct ViewContentsPart { #[prost(message, optional, tag = "2")] pub components: ::core::option::Option, } +impl ::prost::Name for ViewContentsPart { + const NAME: &'static str = "ViewContentsPart"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ViewContentsPart".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ViewContentsPart".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ComponentsSet { #[prost(message, repeated, tag = "1")] pub components: ::prost::alloc::vec::Vec, } +impl ::prost::Name for ComponentsSet { + const NAME: &'static str = "ComponentsSet"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ComponentsSet".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ComponentsSet".into() + } +} /// The unique identifier of an entity, e.g. `camera/3/points` /// See <> for more on entity paths. #[derive(Clone, PartialEq, ::prost::Message)] @@ -173,6 +333,16 @@ pub struct EntityPath { #[prost(string, tag = "1")] pub path: ::prost::alloc::string::String, } +impl ::prost::Name for EntityPath { + const NAME: &'static str = "EntityPath"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.EntityPath".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.EntityPath".into() + } +} /// Component describes semantic data that can be used by any number of rerun's archetypes. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Component { @@ -180,12 +350,32 @@ pub struct Component { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, } +impl ::prost::Name for Component { + const NAME: &'static str = "Component"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Component".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Component".into() + } +} /// Used to telect a time column. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TimeColumnSelector { #[prost(message, optional, tag = "1")] pub timeline: ::core::option::Option, } +impl ::prost::Name for TimeColumnSelector { + const NAME: &'static str = "TimeColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.TimeColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.TimeColumnSelector".into() + } +} /// Used to select a component based on its EntityPath and Component name. #[derive(Clone, PartialEq, ::prost::Message)] pub struct ComponentColumnSelector { @@ -194,11 +384,31 @@ pub struct ComponentColumnSelector { #[prost(message, optional, tag = "2")] pub component: ::core::option::Option, } +impl ::prost::Name for ComponentColumnSelector { + const NAME: &'static str = "ComponentColumnSelector"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ComponentColumnSelector".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ComponentColumnSelector".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ApplicationId { #[prost(string, tag = "1")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for ApplicationId { + const NAME: &'static str = "ApplicationId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.ApplicationId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.ApplicationId".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreId { #[prost(enumeration = "StoreKind", tag = "1")] @@ -206,12 +416,32 @@ pub struct StoreId { #[prost(string, tag = "2")] pub id: ::prost::alloc::string::String, } +impl ::prost::Name for StoreId { + const NAME: &'static str = "StoreId"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.StoreId".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.StoreId".into() + } +} /// A date-time represented as nanoseconds since unix epoch #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Time { #[prost(int64, tag = "1")] pub nanos_since_epoch: i64, } +impl ::prost::Name for Time { + const NAME: &'static str = "Time"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Time".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Time".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Tuid { /// Approximate nanoseconds since epoch. @@ -222,6 +452,16 @@ pub struct Tuid { #[prost(fixed64, tag = "2")] pub inc: u64, } +impl ::prost::Name for Tuid { + const NAME: &'static str = "Tuid"; + const PACKAGE: &'static str = "rerun.common.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.common.v0.Tuid".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.common.v0.Tuid".into() + } +} /// supported encoder versions for encoding data /// See `RerunData` and `RerunChunkData` for its usage #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] diff --git a/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs new file mode 100644 index 000000000000..3b415876fbc0 --- /dev/null +++ b/crates/store/re_protos/src/v0/rerun.log_msg.v0.rs @@ -0,0 +1,388 @@ +// This file is @generated by prost-build. +/// Corresponds to `LogMsg::SetStoreInfo`. Used to identify a recording. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetStoreInfo { + /// A time-based UID that is used to determine how a `StoreInfo` fits in the global ordering of events. + #[prost(message, optional, tag = "1")] + pub row_id: ::core::option::Option, + /// The new store info. + #[prost(message, optional, tag = "2")] + pub info: ::core::option::Option, +} +impl ::prost::Name for SetStoreInfo { + const NAME: &'static str = "SetStoreInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.SetStoreInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.SetStoreInfo".into() + } +} +/// Corresponds to `LogMsg::ArrowMsg`. Used to transmit actual data. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ArrowMsg { + /// The ID of the store that this message is for. + #[prost(message, optional, tag = "1")] + pub store_id: ::core::option::Option, + /// Compression algorithm used. + #[prost(enumeration = "Compression", tag = "2")] + pub compression: i32, + #[prost(int32, tag = "3")] + pub uncompressed_size: i32, + /// Encoding of the payload. + #[prost(enumeration = "Encoding", tag = "4")] + pub encoding: i32, + /// Arrow-IPC encoded schema and chunk, compressed according to the `compression` field. + #[prost(bytes = "vec", tag = "1000")] + pub payload: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for ArrowMsg { + const NAME: &'static str = "ArrowMsg"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.ArrowMsg".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.ArrowMsg".into() + } +} +/// Corresponds to `LogMsg::BlueprintActivationCommand`. +/// +/// Used for activating a blueprint once it has been fully transmitted, +/// because showing a blueprint before it is fully transmitted can lead to +/// a confusing user experience, or inconsistent results due to heuristics. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlueprintActivationCommand { + /// The ID of the blueprint to activate. + #[prost(message, optional, tag = "1")] + pub blueprint_id: ::core::option::Option, + /// Whether to make the blueprint active immediately. + #[prost(bool, tag = "2")] + pub make_active: bool, + /// Whether to make the blueprint the default. + #[prost(bool, tag = "3")] + pub make_default: bool, +} +impl ::prost::Name for BlueprintActivationCommand { + const NAME: &'static str = "BlueprintActivationCommand"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.BlueprintActivationCommand".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.BlueprintActivationCommand".into() + } +} +/// Information about a recording or blueprint. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreInfo { + /// User-chosen name of the application doing the logging. + #[prost(message, optional, tag = "1")] + pub application_id: ::core::option::Option, + /// Unique ID of the recording. + #[prost(message, optional, tag = "2")] + pub store_id: ::core::option::Option, + /// / True if the recording is one of the official Rerun examples. + #[prost(bool, tag = "3")] + pub is_official_example: bool, + /// When the recording started. + #[prost(message, optional, tag = "4")] + pub started: ::core::option::Option, + /// Where the recording came from. + #[prost(message, optional, tag = "5")] + pub store_source: ::core::option::Option, + /// Version of the store crate. + #[prost(message, optional, tag = "6")] + pub store_version: ::core::option::Option, +} +impl ::prost::Name for StoreInfo { + const NAME: &'static str = "StoreInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreInfo".into() + } +} +/// The source of a recording or blueprint. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreSource { + /// Determines what is encoded in `extra`. + #[prost(enumeration = "StoreSourceKind", tag = "1")] + pub kind: i32, + /// Store source payload. See `StoreSourceKind` for what exactly is encoded here. + #[prost(message, optional, tag = "2")] + pub extra: ::core::option::Option, +} +impl ::prost::Name for StoreSource { + const NAME: &'static str = "StoreSource"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreSource".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreSource".into() + } +} +/// A newtype for `StoreSource` payload. +/// +/// This exists to that we can implement conversions on the newtype for convenience. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreSourceExtra { + #[prost(bytes = "vec", tag = "1")] + pub payload: ::prost::alloc::vec::Vec, +} +impl ::prost::Name for StoreSourceExtra { + const NAME: &'static str = "StoreSourceExtra"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreSourceExtra".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreSourceExtra".into() + } +} +/// Version of the Python SDK that created the recording. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PythonVersion { + #[prost(int32, tag = "1")] + pub major: i32, + #[prost(int32, tag = "2")] + pub minor: i32, + #[prost(int32, tag = "3")] + pub patch: i32, + #[prost(string, tag = "4")] + pub suffix: ::prost::alloc::string::String, +} +impl ::prost::Name for PythonVersion { + const NAME: &'static str = "PythonVersion"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.PythonVersion".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.PythonVersion".into() + } +} +/// Information about the Rust SDK that created the recording. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CrateInfo { + /// Version of the Rust compiler used to compile the SDK. + #[prost(string, tag = "1")] + pub rustc_version: ::prost::alloc::string::String, + /// Version of LLVM used by the Rust compiler. + #[prost(string, tag = "2")] + pub llvm_version: ::prost::alloc::string::String, +} +impl ::prost::Name for CrateInfo { + const NAME: &'static str = "CrateInfo"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.CrateInfo".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.CrateInfo".into() + } +} +/// A recording which came from a file. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct FileSource { + #[prost(enumeration = "FileSourceKind", tag = "1")] + pub kind: i32, +} +impl ::prost::Name for FileSource { + const NAME: &'static str = "FileSource"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.FileSource".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.FileSource".into() + } +} +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct StoreVersion { + /// Crate version encoded using our custom scheme. + /// + /// See `CrateVersion` in `re_build_info`. + #[prost(int32, tag = "1")] + pub crate_version_bits: i32, +} +impl ::prost::Name for StoreVersion { + const NAME: &'static str = "StoreVersion"; + const PACKAGE: &'static str = "rerun.log_msg.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.log_msg.v0.StoreVersion".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.log_msg.v0.StoreVersion".into() + } +} +/// The type of compression used on the payload. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Compression { + /// No compression. + None = 0, + /// LZ4 block compression. + Lz4 = 1, +} +impl Compression { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::None => "NONE", + Self::Lz4 => "LZ4", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONE" => Some(Self::None), + "LZ4" => Some(Self::Lz4), + _ => None, + } + } +} +/// The encoding of the message payload. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum Encoding { + /// We don't know what encoding the payload is in. + Unknown = 0, + /// The payload is encoded as Arrow-IPC. + ArrowIpc = 1, +} +impl Encoding { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unknown => "UNKNOWN", + Self::ArrowIpc => "ARROW_IPC", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN" => Some(Self::Unknown), + "ARROW_IPC" => Some(Self::ArrowIpc), + _ => None, + } + } +} +/// What kind of source a recording comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum StoreSourceKind { + /// We don't know anything about the source of this recording. + /// + /// `extra` is unused. + UnknownKind = 0, + /// The recording came from the C++ SDK. + /// + /// `extra` is unused. + CSdk = 1, + /// The recording came from the Python SDK. + /// + /// `extra` is `PythonVersion`. + PythonSdk = 2, + /// The recording came from the Rust SDK. + /// + /// `extra` is `CrateInfo`. + RustSdk = 3, + /// The recording came from a file. + /// + /// `extra` is `FileSource`. + File = 4, + /// The recording came from some action in the viewer. + /// + /// `extra` is unused. + Viewer = 5, + /// The recording came from some other source. + /// + /// `extra` is a string. + Other = 6, +} +impl StoreSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::UnknownKind => "UNKNOWN_KIND", + Self::CSdk => "C_SDK", + Self::PythonSdk => "PYTHON_SDK", + Self::RustSdk => "RUST_SDK", + Self::File => "FILE", + Self::Viewer => "VIEWER", + Self::Other => "OTHER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN_KIND" => Some(Self::UnknownKind), + "C_SDK" => Some(Self::CSdk), + "PYTHON_SDK" => Some(Self::PythonSdk), + "RUST_SDK" => Some(Self::RustSdk), + "FILE" => Some(Self::File), + "VIEWER" => Some(Self::Viewer), + "OTHER" => Some(Self::Other), + _ => None, + } + } +} +/// Determines where the file came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FileSourceKind { + /// We don't know where the file came from. + UnknownSource = 0, + /// The file came from the command line. + Cli = 1, + /// The file was served over HTTP. + Uri = 2, + /// The file was dragged into the viewer. + DragAndDrop = 3, + /// The file was opened using a file dialog. + FileDialog = 4, + /// The recording was produced using a data loader, such as when logging a mesh file. + Sdk = 5, +} +impl FileSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::UnknownSource => "UNKNOWN_SOURCE", + Self::Cli => "CLI", + Self::Uri => "URI", + Self::DragAndDrop => "DRAG_AND_DROP", + Self::FileDialog => "FILE_DIALOG", + Self::Sdk => "SDK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "UNKNOWN_SOURCE" => Some(Self::UnknownSource), + "CLI" => Some(Self::Cli), + "URI" => Some(Self::Uri), + "DRAG_AND_DROP" => Some(Self::DragAndDrop), + "FILE_DIALOG" => Some(Self::FileDialog), + "SDK" => Some(Self::Sdk), + _ => None, + } + } +} diff --git a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs index 144ddc584ef7..1429146e776d 100644 --- a/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs +++ b/crates/store/re_protos/src/v0/rerun.remote_store.v0.rs @@ -9,6 +9,16 @@ pub struct DataframePart { #[prost(bytes = "vec", tag = "1000")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for DataframePart { + const NAME: &'static str = "DataframePart"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.DataframePart".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.DataframePart".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct RegisterRecordingRequest { /// human readable description of the recording @@ -25,6 +35,16 @@ pub struct RegisterRecordingRequest { #[prost(message, optional, tag = "4")] pub metadata: ::core::option::Option, } +impl ::prost::Name for RegisterRecordingRequest { + const NAME: &'static str = "RegisterRecordingRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.RegisterRecordingRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.RegisterRecordingRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateCatalogRequest { #[prost(message, optional, tag = "1")] @@ -32,8 +52,28 @@ pub struct UpdateCatalogRequest { #[prost(message, optional, tag = "2")] pub metadata: ::core::option::Option, } +impl ::prost::Name for UpdateCatalogRequest { + const NAME: &'static str = "UpdateCatalogRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.UpdateCatalogRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.UpdateCatalogRequest".into() + } +} #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct UpdateCatalogResponse {} +impl ::prost::Name for UpdateCatalogResponse { + const NAME: &'static str = "UpdateCatalogResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.UpdateCatalogResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.UpdateCatalogResponse".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryRequest { /// unique identifier of the recording @@ -43,6 +83,16 @@ pub struct QueryRequest { #[prost(message, optional, tag = "3")] pub query: ::core::option::Option, } +impl ::prost::Name for QueryRequest { + const NAME: &'static str = "QueryRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryCatalogRequest { /// Column projection - define which columns should be returned. @@ -53,11 +103,31 @@ pub struct QueryCatalogRequest { #[prost(message, optional, tag = "2")] pub filter: ::core::option::Option, } +impl ::prost::Name for QueryCatalogRequest { + const NAME: &'static str = "QueryCatalogRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryCatalogRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryCatalogRequest".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct ColumnProjection { #[prost(string, repeated, tag = "1")] pub columns: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, } +impl ::prost::Name for ColumnProjection { + const NAME: &'static str = "ColumnProjection"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.ColumnProjection".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.ColumnProjection".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct CatalogFilter { /// Filtering is very simple right now, we can only select @@ -65,6 +135,16 @@ pub struct CatalogFilter { #[prost(message, repeated, tag = "1")] pub recording_ids: ::prost::alloc::vec::Vec, } +impl ::prost::Name for CatalogFilter { + const NAME: &'static str = "CatalogFilter"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.CatalogFilter".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.CatalogFilter".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct QueryCatalogResponse { #[prost(enumeration = "super::super::common::v0::EncoderVersion", tag = "1")] @@ -73,11 +153,31 @@ pub struct QueryCatalogResponse { #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for QueryCatalogResponse { + const NAME: &'static str = "QueryCatalogResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.QueryCatalogResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.QueryCatalogResponse".into() + } +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct FetchRecordingRequest { #[prost(message, optional, tag = "1")] pub recording_id: ::core::option::Option, } +impl ::prost::Name for FetchRecordingRequest { + const NAME: &'static str = "FetchRecordingRequest"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.FetchRecordingRequest".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.FetchRecordingRequest".into() + } +} /// TODO(jleibs): Eventually this becomes either query-mediated in some way, but for now /// it's useful to be able to just get back the whole RRD somehow. #[derive(Clone, PartialEq, ::prost::Message)] @@ -91,6 +191,16 @@ pub struct FetchRecordingResponse { #[prost(bytes = "vec", tag = "2")] pub payload: ::prost::alloc::vec::Vec, } +impl ::prost::Name for FetchRecordingResponse { + const NAME: &'static str = "FetchRecordingResponse"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.FetchRecordingResponse".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.FetchRecordingResponse".into() + } +} /// Application level error - used as `details` in the `google.rpc.Status` message #[derive(Clone, PartialEq, ::prost::Message)] pub struct RemoteStoreError { @@ -104,6 +214,16 @@ pub struct RemoteStoreError { #[prost(string, tag = "3")] pub message: ::prost::alloc::string::String, } +impl ::prost::Name for RemoteStoreError { + const NAME: &'static str = "RemoteStoreError"; + const PACKAGE: &'static str = "rerun.remote_store.v0"; + fn full_name() -> ::prost::alloc::string::String { + "rerun.remote_store.v0.RemoteStoreError".into() + } + fn type_url() -> ::prost::alloc::string::String { + "/rerun.remote_store.v0.RemoteStoreError".into() + } +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum RecordingType { diff --git a/crates/store/re_sdk_comms/src/buffered_client.rs b/crates/store/re_sdk_comms/src/buffered_client.rs index 6a48e553bd4b..4616bcce5f7f 100644 --- a/crates/store/re_sdk_comms/src/buffered_client.rs +++ b/crates/store/re_sdk_comms/src/buffered_client.rs @@ -71,7 +71,7 @@ impl Client { // We don't compress the stream because we assume the SDK // and server are on the same machine and compression // can be expensive, see https://github.com/rerun-io/rerun/issues/2216 - let encoding_options = re_log_encoding::EncodingOptions::UNCOMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_UNCOMPRESSED; let encode_join = std::thread::Builder::new() .name("msg_encoder".into()) diff --git a/crates/store/re_sdk_comms/src/server.rs b/crates/store/re_sdk_comms/src/server.rs index fc7ceea9037c..48a1c381c700 100644 --- a/crates/store/re_sdk_comms/src/server.rs +++ b/crates/store/re_sdk_comms/src/server.rs @@ -238,7 +238,7 @@ fn run_client( congestion_manager.register_latency(tx.latency_sec()); - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; for msg in re_log_encoding::decoder::decode_bytes(version_policy, &packet)? { if congestion_manager.should_send(&msg) { tx.send(msg)?; diff --git a/crates/top/re_sdk/src/binary_stream_sink.rs b/crates/top/re_sdk/src/binary_stream_sink.rs index a673ea24992a..102f453271aa 100644 --- a/crates/top/re_sdk/src/binary_stream_sink.rs +++ b/crates/top/re_sdk/src/binary_stream_sink.rs @@ -129,7 +129,7 @@ impl BinaryStreamSink { // We always compress when writing to a stream // TODO(jleibs): Make this configurable - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let (tx, rx) = std::sync::mpsc::channel(); diff --git a/crates/top/rerun/Cargo.toml b/crates/top/rerun/Cargo.toml index 068c98028fdd..d9739ec159f8 100644 --- a/crates/top/rerun/Cargo.toml +++ b/crates/top/rerun/Cargo.toml @@ -99,7 +99,8 @@ run = [ "unindent", "dep:re_chunk_store", "dep:re_data_source", - "dep:re_log_encoding", + "re_log_encoding/encoder", + "re_log_encoding/decoder", "dep:re_sdk_comms", "dep:re_ws_comms", ] @@ -128,6 +129,7 @@ re_crash_handler.workspace = true re_entity_db.workspace = true re_error.workspace = true re_format.workspace = true +re_log_encoding.workspace = true re_log_types.workspace = true re_video.workspace = true re_log.workspace = true @@ -145,10 +147,6 @@ re_analytics = { workspace = true, optional = true } re_chunk_store = { workspace = true, optional = true } re_data_source = { workspace = true, optional = true } re_dataframe = { workspace = true, optional = true } -re_log_encoding = { workspace = true, optional = true, features = [ - "decoder", - "encoder", -] } re_sdk = { workspace = true, optional = true } re_sdk_comms = { workspace = true, optional = true } re_types = { workspace = true, optional = true } diff --git a/crates/top/rerun/src/commands/entrypoint.rs b/crates/top/rerun/src/commands/entrypoint.rs index b34289ac1d09..bc437f46ce35 100644 --- a/crates/top/rerun/src/commands/entrypoint.rs +++ b/crates/top/rerun/src/commands/entrypoint.rs @@ -1007,7 +1007,7 @@ fn stream_to_rrd_on_disk( re_log::info!("Saving incoming log stream to {path:?}. Abort with Ctrl-C."); - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let file = std::fs::File::create(path).map_err(|err| FileSinkError::CreateFile(path.clone(), err))?; let mut encoder = re_log_encoding::encoder::DroppableEncoder::new( diff --git a/crates/top/rerun/src/commands/rrd/compare.rs b/crates/top/rerun/src/commands/rrd/compare.rs index cd2c87b5aa09..3472eacd8f90 100644 --- a/crates/top/rerun/src/commands/rrd/compare.rs +++ b/crates/top/rerun/src/commands/rrd/compare.rs @@ -93,7 +93,7 @@ fn compute_uber_table( let rrd_file = std::io::BufReader::new(rrd_file); let mut stores: std::collections::HashMap = Default::default(); - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; let decoder = re_log_encoding::decoder::Decoder::new(version_policy, rrd_file)?; for msg in decoder { let msg = msg.context("decode rrd message")?; diff --git a/crates/top/rerun/src/commands/rrd/filter.rs b/crates/top/rerun/src/commands/rrd/filter.rs index 53efe7cf9d4e..b1e9ca2c0e5e 100644 --- a/crates/top/rerun/src/commands/rrd/filter.rs +++ b/crates/top/rerun/src/commands/rrd/filter.rs @@ -60,7 +60,7 @@ impl FilterCommand { .collect(); // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx_decoder, rx_size_bytes) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); @@ -83,7 +83,7 @@ impl FilterCommand { let mut encoder = { // TODO(cmc): encoding options & version should match the original. let version = CrateVersion::LOCAL; - let options = re_log_encoding::EncodingOptions::COMPRESSED; + let options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::DroppableEncoder::new(version, options, &mut rrd_out) .context("couldn't init encoder")? }; diff --git a/crates/top/rerun/src/commands/rrd/merge_compact.rs b/crates/top/rerun/src/commands/rrd/merge_compact.rs index 9aa434f84b1a..82a2ffb3cd69 100644 --- a/crates/top/rerun/src/commands/rrd/merge_compact.rs +++ b/crates/top/rerun/src/commands/rrd/merge_compact.rs @@ -157,7 +157,7 @@ fn merge_and_compact( ); // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx, rx_size_bytes) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); @@ -215,7 +215,7 @@ fn merge_and_compact( .flat_map(|entity_db| entity_db.to_messages(None /* time selection */)); // TODO(cmc): encoding options should match the original. - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; let version = entity_dbs .values() .next() diff --git a/crates/top/rerun/src/commands/rrd/print.rs b/crates/top/rerun/src/commands/rrd/print.rs index 2d513c8fc091..f5145f62fb20 100644 --- a/crates/top/rerun/src/commands/rrd/print.rs +++ b/crates/top/rerun/src/commands/rrd/print.rs @@ -42,7 +42,7 @@ impl PrintCommand { } = self; // TODO(cmc): might want to make this configurable at some point. - let version_policy = re_log_encoding::decoder::VersionPolicy::Warn; + let version_policy = re_log_encoding::VersionPolicy::Warn; let (rx, _) = read_rrd_streams_from_file_or_stdin(version_policy, path_to_input_rrds); for res in rx { diff --git a/crates/top/rerun/src/commands/stdio.rs b/crates/top/rerun/src/commands/stdio.rs index 98599dd7b5eb..e34a67f155c6 100644 --- a/crates/top/rerun/src/commands/stdio.rs +++ b/crates/top/rerun/src/commands/stdio.rs @@ -23,7 +23,7 @@ use re_log_types::LogMsg; /// /// This function is capable of decoding multiple independent recordings from a single stream. pub fn read_rrd_streams_from_file_or_stdin( - version_policy: re_log_encoding::decoder::VersionPolicy, + version_policy: re_log_encoding::VersionPolicy, paths: &[String], ) -> ( channel::Receiver>, diff --git a/crates/top/rerun/src/lib.rs b/crates/top/rerun/src/lib.rs index fb1b3953db98..2f6dbf8f1c8b 100644 --- a/crates/top/rerun/src/lib.rs +++ b/crates/top/rerun/src/lib.rs @@ -133,6 +133,8 @@ pub use log_integration::Logger; #[cfg(feature = "run")] pub use commands::{run, CallSource}; +pub use re_log_encoding::VersionPolicy; + #[cfg(feature = "sdk")] pub use sdk::*; @@ -145,7 +147,7 @@ pub mod dataframe { /// Everything needed to build custom `ChunkStoreSubscriber`s. pub use re_entity_db::external::re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreDiff, ChunkStoreDiffKind, ChunkStoreEvent, - ChunkStoreGeneration, ChunkStoreHandle, ChunkStoreSubscriber, VersionPolicy, + ChunkStoreGeneration, ChunkStoreHandle, ChunkStoreSubscriber, }; pub use re_log_types::StoreKind; diff --git a/crates/utils/re_tuid/src/protobuf_conversions.rs b/crates/utils/re_tuid/src/protobuf_conversions.rs index 2dfd04627d4b..a04dba09ddbf 100644 --- a/crates/utils/re_tuid/src/protobuf_conversions.rs +++ b/crates/utils/re_tuid/src/protobuf_conversions.rs @@ -15,3 +15,14 @@ impl From for re_protos::common::v0::Tuid { } } } + +#[cfg(test)] +mod tests { + #[test] + fn test_tuid_conversion() { + let tuid = crate::Tuid::new(); + let proto_tuid: re_protos::common::v0::Tuid = tuid.into(); + let tuid2: crate::Tuid = proto_tuid.into(); + assert_eq!(tuid, tuid2); + } +} diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index 870e52527f4c..6de723f67dca 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -2283,7 +2283,7 @@ async fn async_save_dialog( let bytes = re_log_encoding::encoder::encode_as_bytes( rrd_version, - re_log_encoding::EncodingOptions::COMPRESSED, + re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED, messages, )?; file_handle.write(&bytes).await.context("Failed to save") diff --git a/crates/viewer/re_viewer/src/loading.rs b/crates/viewer/re_viewer/src/loading.rs index a6e2c5317718..476e3d969afb 100644 --- a/crates/viewer/re_viewer/src/loading.rs +++ b/crates/viewer/re_viewer/src/loading.rs @@ -25,7 +25,7 @@ pub fn load_blueprint_file( // Blueprint files change often. Be strict about the version, and then ignore any errors. // See https://github.com/rerun-io/rerun/issues/2830 - let version_policy = re_log_encoding::decoder::VersionPolicy::Error; + let version_policy = re_log_encoding::VersionPolicy::Error; Ok(StoreBundle::from_rrd(version_policy, file)?) } diff --git a/crates/viewer/re_viewer/src/saving.rs b/crates/viewer/re_viewer/src/saving.rs index 142fda1997df..001912449730 100644 --- a/crates/viewer/re_viewer/src/saving.rs +++ b/crates/viewer/re_viewer/src/saving.rs @@ -67,7 +67,7 @@ pub fn encode_to_file( let mut file = std::fs::File::create(path) .with_context(|| format!("Failed to create file at {path:?}"))?; - let encoding_options = re_log_encoding::EncodingOptions::COMPRESSED; + let encoding_options = re_log_encoding::EncodingOptions::MSGPACK_COMPRESSED; re_log_encoding::encoder::encode(version, encoding_options, messages, &mut file) .map(|_| ()) .context("Message encode") diff --git a/rerun_py/src/dataframe.rs b/rerun_py/src/dataframe.rs index 3a66680f2114..3f75a459039e 100644 --- a/rerun_py/src/dataframe.rs +++ b/rerun_py/src/dataframe.rs @@ -21,9 +21,10 @@ use pyo3::{ use re_chunk_store::{ ChunkStore, ChunkStoreConfig, ChunkStoreHandle, ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor, ComponentColumnSelector, QueryExpression, SparseFillStrategy, - TimeColumnDescriptor, TimeColumnSelector, VersionPolicy, ViewContentsSelector, + TimeColumnDescriptor, TimeColumnSelector, ViewContentsSelector, }; use re_dataframe::{QueryEngine, StorageEngine}; +use re_log_encoding::VersionPolicy; use re_log_types::{EntityPathFilter, ResolvedTimeRange, TimeType}; use re_sdk::{ComponentName, EntityPath, StoreId, StoreKind};