diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 31fb2a572c63..eef004da63cb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -37,6 +37,24 @@ The web viewer can load `.rrd` files (just drag-drop them into the browser), or NOTE: `.rrd` files do not yet guarantee any backwards or forwards compatibility. One version of Rerun will likely not be able to open an `.rrd` file generated by another Rerun version. +## Crates +In order to get an overview of our in-house crates and how they depend on each other, we recommend you run: + +``` +cargo install cargo-depgraph +cargo depgraph --all-deps --workspace-only --all-features --dedup-transitive-deps | dot -Tpng > deps.png +open deps.png +``` + +and: + +``` +cargo doc --no-deps --open +``` + +and then browse through the `re_*` crates. + + ## Technologies we use ### Apache Arrow [Apache Arrow](https://arrow.apache.org/) is a language-independent columnar memory format for arbitrary data. We use it to encode the log data when transmitting it over the network or storing it in an `.rrd` file. We also use it in our in-RAM data store, [`re_arrow_store`](crates/re_arrow_store/README.md). diff --git a/Cargo.lock b/Cargo.lock index e9a1d44fe3ef..d4ef75b75afd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3865,6 +3865,7 @@ dependencies = [ "polars-ops", "puffin", "rand", + "re_components", "re_format", "re_log", "re_log_types", @@ -3896,6 +3897,36 @@ dependencies = [ "wasm-bindgen-cli-support", ] +[[package]] +name = "re_components" +version = "0.7.0-alpha.0" +dependencies = [ + "ahash 0.8.3", + "array-init", + "arrow2", + "arrow2_convert", + "bytemuck", + "crossbeam", + "document-features", + "ecolor", + "glam", + "half 2.2.1", + "image", + "itertools", + "lazy_static", + "ndarray", + "nohash-hasher", + "puffin", + "rand", + "re_log", + "re_log_types", + "rmp-serde", + "serde", + "similar-asserts", + "thiserror", + "uuid", +] + [[package]] name = "re_crash_handler" version = "0.7.0-alpha.0" @@ -3921,6 +3952,7 @@ dependencies = [ "puffin", "rand", "re_arrow_store", + "re_components", "re_format", "re_int_histogram", "re_log", @@ -3944,6 +3976,7 @@ dependencies = [ "nohash-hasher", "puffin", "re_arrow_store", + "re_components", "re_data_store", "re_error", "re_format", @@ -4010,6 +4043,7 @@ dependencies = [ "parking_lot 0.12.1", "puffin", "re_build_info", + "re_components", "re_error", "re_log", "re_log_types", @@ -4028,25 +4062,16 @@ name = "re_log_types" version = "0.7.0-alpha.0" dependencies = [ "ahash 0.8.3", - "array-init", "arrow2", "arrow2_convert", - "bytemuck", "crossbeam", "document-features", - "ecolor", "fixed", - "glam", - "half 2.2.1", - "image", "itertools", - "lazy_static", - "ndarray", "nohash-hasher", "num-derive", "num-traits", "puffin", - "rand", "re_format", "re_log", "re_string_interner", @@ -4095,6 +4120,7 @@ dependencies = [ "polars-core", "puffin", "re_arrow_store", + "re_components", "re_data_store", "re_format", "re_log", @@ -4168,6 +4194,7 @@ dependencies = [ "parking_lot 0.12.1", "rand", "re_build_build_info", + "re_components", "re_log", "re_log_encoding", "re_log_types", @@ -4218,7 +4245,7 @@ version = "0.7.0-alpha.0" dependencies = [ "ahash 0.8.3", "ndarray", - "re_log_types", + "re_components", "serde", ] @@ -4295,6 +4322,7 @@ dependencies = [ "re_arrow_store", "re_build_build_info", "re_build_info", + "re_components", "re_data_store", "re_data_ui", "re_error", @@ -4339,6 +4367,7 @@ dependencies = [ "nohash-hasher", "puffin", "re_arrow_store", + "re_components", "re_data_store", "re_log", "re_log_types", @@ -4376,6 +4405,7 @@ dependencies = [ "nohash-hasher", "puffin", "re_arrow_store", + "re_components", "re_data_store", "re_data_ui", "re_error", @@ -4551,6 +4581,7 @@ dependencies = [ "rand", "re_build_build_info", "re_build_info", + "re_components", "re_error", "re_log", "re_log_types", diff --git a/Cargo.toml b/Cargo.toml index 7b04a9bcb33c..2074f50993c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ re_arrow_store = { path = "crates/re_arrow_store", version = "0.7.0-alpha.0", de re_build_build_info = { path = "crates/re_build_build_info", version = "0.7.0-alpha.0", default-features = false } re_build_info = { path = "crates/re_build_info", version = "0.7.0-alpha.0", default-features = false } re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "0.7.0-alpha.0", default-features = false } +re_components = { path = "crates/re_components", version = "0.7.0-alpha.0", default-features = false } re_crash_handler = { path = "crates/re_crash_handler", version = "0.7.0-alpha.0", default-features = false } re_data_store = { path = "crates/re_data_store", version = "0.7.0-alpha.0", default-features = false } re_data_ui = { path = "crates/re_data_ui", version = "0.7.0-alpha.0", default-features = false } diff --git a/crates/re_arrow_store/Cargo.toml b/crates/re_arrow_store/Cargo.toml index 2993e7f71e80..5ee21ebf159f 100644 --- a/crates/re_arrow_store/Cargo.toml +++ b/crates/re_arrow_store/Cargo.toml @@ -70,6 +70,8 @@ polars-ops = { workspace = true, optional = true, features = [ [dev-dependencies] +re_components = { workspace = true, features = ["arrow_datagen"] } + anyhow.workspace = true criterion = "0.5" mimalloc.workspace = true diff --git a/crates/re_arrow_store/benches/arrow2.rs b/crates/re_arrow_store/benches/arrow2.rs index 51171661f06e..a1f46e11531c 100644 --- a/crates/re_arrow_store/benches/arrow2.rs +++ b/crates/re_arrow_store/benches/arrow2.rs @@ -8,11 +8,13 @@ use std::sync::Arc; use arrow2::array::{Array, PrimitiveArray, StructArray, UnionArray}; use criterion::{criterion_group, Criterion}; use itertools::Itertools; -use re_log_types::{ - component_types::{InstanceKey, Point2D, Rect2D}, +use re_components::{ datagen::{build_some_instances, build_some_point2d, build_some_rects}, - external::arrow2_convert::serialize::TryIntoArrow, - DataCell, SerializableComponent, SizeBytes as _, + Point2D, Rect2D, +}; +use re_log_types::{ + external::arrow2_convert::serialize::TryIntoArrow, DataCell, InstanceKey, + SerializableComponent, SizeBytes as _, }; // --- diff --git a/crates/re_arrow_store/benches/arrow2_convert.rs b/crates/re_arrow_store/benches/arrow2_convert.rs index e0d4ce95abdc..1afe46bcf266 100644 --- a/crates/re_arrow_store/benches/arrow2_convert.rs +++ b/crates/re_arrow_store/benches/arrow2_convert.rs @@ -6,8 +6,7 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use arrow2::{array::PrimitiveArray, datatypes::PhysicalType, types::PrimitiveType}; use criterion::{criterion_group, Criterion}; use re_log_types::{ - component_types::InstanceKey, external::arrow2_convert::deserialize::TryIntoCollection, - Component as _, DataCell, + external::arrow2_convert::deserialize::TryIntoCollection, Component as _, DataCell, InstanceKey, }; // --- diff --git a/crates/re_arrow_store/benches/data_store.rs b/crates/re_arrow_store/benches/data_store.rs index 0e254819f355..82855a598ed9 100644 --- a/crates/re_arrow_store/benches/data_store.rs +++ b/crates/re_arrow_store/benches/data_store.rs @@ -8,11 +8,13 @@ use re_arrow_store::{ DataStore, DataStoreConfig, GarbageCollectionTarget, LatestAtQuery, RangeQuery, TimeInt, TimeRange, }; -use re_log_types::{ - component_types::{InstanceKey, Rect2D}, +use re_components::{ datagen::{build_frame_nr, build_some_instances, build_some_rects}, - Component as _, ComponentName, DataCell, DataRow, DataTable, EntityPath, RowId, TableId, - TimeType, Timeline, + Rect2D, +}; +use re_log_types::{ + Component as _, ComponentName, DataCell, DataRow, DataTable, EntityPath, InstanceKey, RowId, + TableId, TimeType, Timeline, }; criterion_group!(benches, insert, latest_at, latest_at_missing, range, gc); diff --git a/crates/re_arrow_store/examples/dump_dataframe.rs b/crates/re_arrow_store/examples/dump_dataframe.rs index 74b5e111ac69..cb79aa96f07e 100644 --- a/crates/re_arrow_store/examples/dump_dataframe.rs +++ b/crates/re_arrow_store/examples/dump_dataframe.rs @@ -5,14 +5,11 @@ //! ``` use re_arrow_store::{test_row, DataStore}; -use re_log_types::{ - component_types::InstanceKey, - datagen::{ - build_frame_nr, build_log_time, build_some_instances, build_some_instances_from, - build_some_point2d, build_some_rects, - }, - Component as _, EntityPath, Time, +use re_components::datagen::{ + build_frame_nr, build_log_time, build_some_instances, build_some_instances_from, + build_some_point2d, build_some_rects, }; +use re_log_types::{Component as _, EntityPath, InstanceKey, Time}; // --- diff --git a/crates/re_arrow_store/examples/latest_component.rs b/crates/re_arrow_store/examples/latest_component.rs index ccfc7f5c8685..b32ca0e02350 100644 --- a/crates/re_arrow_store/examples/latest_component.rs +++ b/crates/re_arrow_store/examples/latest_component.rs @@ -6,13 +6,11 @@ use re_arrow_store::polars_util::latest_component; use re_arrow_store::{test_row, DataStore, LatestAtQuery, TimeType, Timeline}; -use re_log_types::component_types::Rect2D; -use re_log_types::datagen::build_some_rects; -use re_log_types::{ - component_types::{InstanceKey, Point2D}, - datagen::{build_frame_nr, build_some_point2d}, - Component, EntityPath, +use re_components::{ + datagen::{build_frame_nr, build_some_point2d, build_some_rects}, + Point2D, Rect2D, }; +use re_log_types::{Component, EntityPath, InstanceKey}; fn main() { let mut store = DataStore::new(InstanceKey::name(), Default::default()); diff --git a/crates/re_arrow_store/examples/latest_components.rs b/crates/re_arrow_store/examples/latest_components.rs index 89abf7771958..d623492fdf7d 100644 --- a/crates/re_arrow_store/examples/latest_components.rs +++ b/crates/re_arrow_store/examples/latest_components.rs @@ -5,13 +5,14 @@ //! ``` use polars_core::prelude::*; + use re_arrow_store::polars_util::latest_components; use re_arrow_store::{test_row, DataStore, LatestAtQuery, TimeType, Timeline}; -use re_log_types::{ - component_types::{InstanceKey, Point2D, Rect2D}, +use re_components::{ datagen::{build_frame_nr, build_some_point2d, build_some_rects}, - Component, EntityPath, + Point2D, Rect2D, }; +use re_log_types::{Component, EntityPath, InstanceKey}; fn main() { let mut store = DataStore::new(InstanceKey::name(), Default::default()); diff --git a/crates/re_arrow_store/examples/range_components.rs b/crates/re_arrow_store/examples/range_components.rs index 9d4130a1a534..f9bd8e69c25d 100644 --- a/crates/re_arrow_store/examples/range_components.rs +++ b/crates/re_arrow_store/examples/range_components.rs @@ -6,11 +6,11 @@ use polars_core::prelude::JoinType; use re_arrow_store::{polars_util, test_row, DataStore, RangeQuery, TimeRange}; -use re_log_types::{ - component_types::{InstanceKey, Point2D, Rect2D}, +use re_components::{ datagen::{build_frame_nr, build_some_point2d, build_some_rects}, - Component as _, EntityPath, TimeType, Timeline, + Point2D, Rect2D, }; +use re_log_types::{Component as _, EntityPath, InstanceKey, TimeType, Timeline}; fn main() { let mut store = DataStore::new(InstanceKey::name(), Default::default()); diff --git a/crates/re_arrow_store/src/arrow_util.rs b/crates/re_arrow_store/src/arrow_util.rs index 44bf32b5bbd8..ea7eb51f2f84 100644 --- a/crates/re_arrow_store/src/arrow_util.rs +++ b/crates/re_arrow_store/src/arrow_util.rs @@ -224,7 +224,7 @@ impl ArrayExt for dyn Array { #[test] fn test_clean_for_polars_nomodify() { - use re_log_types::datagen::build_some_colors; + use re_components::datagen::build_some_colors; use re_log_types::DataCell; // Colors don't need polars cleaning @@ -237,7 +237,8 @@ fn test_clean_for_polars_nomodify() { mod tests { use arrow2::datatypes::{DataType, Field, UnionMode}; use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; - use re_log_types::{component_types::Vec3D, Component, DataCell}; + use re_components::Vec3D; + use re_log_types::DataCell; use crate::ArrayExt; @@ -248,7 +249,7 @@ mod tests { Vec3D(Vec3D), } - impl Component for TestComponentWithUnionAndFixedSizeList { + impl re_log_types::Component for TestComponentWithUnionAndFixedSizeList { fn name() -> re_log_types::ComponentName { "test_component_with_union_and_fixed_size_list".into() } diff --git a/crates/re_arrow_store/src/store.rs b/crates/re_arrow_store/src/store.rs index 1edffd96b7c2..81a715a20d83 100644 --- a/crates/re_arrow_store/src/store.rs +++ b/crates/re_arrow_store/src/store.rs @@ -3,10 +3,10 @@ use std::sync::atomic::AtomicU64; use ahash::HashMap; use arrow2::datatypes::DataType; -use smallvec::SmallVec; - use nohash_hasher::{IntMap, IntSet}; use parking_lot::RwLock; +use smallvec::SmallVec; + use re_log_types::{ ComponentName, DataCell, DataCellColumn, EntityPath, EntityPathHash, ErasedTimeVec, NumInstancesVec, RowId, RowIdVec, SizeBytes, TimeInt, TimePoint, TimeRange, Timeline, @@ -324,7 +324,8 @@ impl DataStore { /// ``` #[test] fn datastore_internal_repr() { - use re_log_types::{component_types::InstanceKey, Component as _, DataTable}; + use re_components::datagen::data_table_example; + use re_log_types::{Component as _, InstanceKey}; let mut store = DataStore::new( InstanceKey::name(), @@ -335,11 +336,11 @@ fn datastore_internal_repr() { }, ); - let timeless = DataTable::example(true); + let timeless = data_table_example(true); eprintln!("{timeless}"); store.insert_table(&timeless).unwrap(); - let temporal = DataTable::example(false); + let temporal = data_table_example(false); eprintln!("{temporal}"); store.insert_table(&temporal).unwrap(); diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs index bc2413678027..ad61c9a0a5f0 100644 --- a/crates/re_arrow_store/src/store_write.rs +++ b/crates/re_arrow_store/src/store_write.rs @@ -6,8 +6,8 @@ use smallvec::SmallVec; use re_log::{debug, trace}; use re_log_types::{ - component_types::InstanceKey, ComponentName, DataCell, DataCellColumn, DataCellError, DataRow, - DataTable, RowId, SizeBytes as _, TimeInt, TimePoint, TimeRange, + ComponentName, DataCell, DataCellColumn, DataCellError, DataRow, DataTable, InstanceKey, RowId, + SizeBytes as _, TimeInt, TimePoint, TimeRange, }; use crate::{ diff --git a/crates/re_arrow_store/tests/correctness.rs b/crates/re_arrow_store/tests/correctness.rs index fba86298332e..01839d5ba34f 100644 --- a/crates/re_arrow_store/tests/correctness.rs +++ b/crates/re_arrow_store/tests/correctness.rs @@ -10,12 +10,11 @@ use re_arrow_store::{ test_row, test_util::sanity_unwrap, DataStore, DataStoreConfig, DataStoreStats, GarbageCollectionTarget, LatestAtQuery, WriteError, }; +use re_components::datagen::{ + build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_point2d, +}; use re_log_types::{ - component_types::InstanceKey, - datagen::{ - build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_point2d, - }, - Component as _, DataCell, Duration, EntityPath, Time, TimeType, Timeline, + Component as _, DataCell, Duration, EntityPath, InstanceKey, Time, TimeType, Timeline, }; // --- @@ -220,7 +219,7 @@ fn range_join_across_single_row_impl(store: &mut DataStore) { prelude::{DataFrame, JoinType}, series::Series, }; - use re_log_types::component_types::{ColorRGBA, Point2D}; + use re_components::{ColorRGBA, Point2D}; use re_log_types::external::arrow2_convert::serialize::TryIntoArrow as _; let ent_path = EntityPath::from("this/that"); diff --git a/crates/re_arrow_store/tests/data_store.rs b/crates/re_arrow_store/tests/data_store.rs index b7d62a44d425..b17aaa72da84 100644 --- a/crates/re_arrow_store/tests/data_store.rs +++ b/crates/re_arrow_store/tests/data_store.rs @@ -14,12 +14,14 @@ use re_arrow_store::{ polars_util, test_row, test_util::sanity_unwrap, DataStore, DataStoreConfig, DataStoreStats, GarbageCollectionTarget, LatestAtQuery, RangeQuery, TimeInt, TimeRange, }; -use re_log_types::{ - component_types::{ColorRGBA, InstanceKey, Point2D, Rect2D}, +use re_components::{ datagen::{ build_frame_nr, build_some_colors, build_some_instances, build_some_instances_from, build_some_point2d, build_some_rects, }, + ColorRGBA, InstanceKey, Point2D, Rect2D, +}; +use re_log_types::{ Component as _, ComponentName, DataCell, DataRow, DataTable, EntityPath, TableId, TimeType, Timeline, }; diff --git a/crates/re_arrow_store/tests/dump.rs b/crates/re_arrow_store/tests/dump.rs index ed5ed0ef1ca1..336e1042f073 100644 --- a/crates/re_arrow_store/tests/dump.rs +++ b/crates/re_arrow_store/tests/dump.rs @@ -7,13 +7,10 @@ use re_arrow_store::{ test_row, test_util::sanity_unwrap, DataStore, DataStoreStats, GarbageCollectionTarget, TimeInt, TimeRange, Timeline, }; -use re_log_types::{ - component_types::InstanceKey, - datagen::{ - build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_point2d, - }, - Component as _, DataTable, EntityPath, TableId, +use re_components::datagen::{ + build_frame_nr, build_log_time, build_some_colors, build_some_instances, build_some_point2d, }; +use re_log_types::{Component as _, DataTable, EntityPath, InstanceKey, TableId}; // --- Dump --- diff --git a/crates/re_arrow_store/tests/internals.rs b/crates/re_arrow_store/tests/internals.rs index a82541ebbf28..10c9cc3ffffc 100644 --- a/crates/re_arrow_store/tests/internals.rs +++ b/crates/re_arrow_store/tests/internals.rs @@ -5,11 +5,8 @@ use std::sync::atomic::{AtomicBool, Ordering::SeqCst}; use re_arrow_store::{DataStore, DataStoreConfig}; -use re_log_types::{ - component_types::InstanceKey, - datagen::{build_frame_nr, build_some_instances}, - Component as _, DataRow, EntityPath, RowId, TimePoint, -}; +use re_components::datagen::{build_frame_nr, build_some_instances}; +use re_log_types::{Component as _, DataRow, EntityPath, InstanceKey, RowId, TimePoint}; // --- Internals --- diff --git a/crates/re_components/Cargo.toml b/crates/re_components/Cargo.toml new file mode 100644 index 000000000000..f78251063034 --- /dev/null +++ b/crates/re_components/Cargo.toml @@ -0,0 +1,78 @@ +[package] +name = "re_components" +authors.workspace = true +description = "The standard rerun data types, component types, and archetypes" +edition.workspace = true +homepage.workspace = true +include.workspace = true +license.workspace = true +publish = true +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[package.metadata.docs.rs] +all-features = true + + +[features] +default = [] + +## Enables the `datagen` module, which exposes a number of tools for generating random data for +## tests and benchmarks. +arrow_datagen = ["dep:rand"] + +## Enable conversions +ecolor = ["dep:ecolor"] + +## Add support for some math operations using [`glam`](https://crates.io/crates/glam/). +glam = ["dep:glam"] + +## Integration with the [`image`](https://crates.io/crates/image/) crate. +image = ["dep:ecolor", "dep:image"] + +## Enable (de)serialization using serde. +serde = ["dep:serde", "half/serde", "re_log_types/serde"] + + +[dependencies] + +# Rerun +re_log_types.workspace = true +re_log.workspace = true + +# External +ahash.workspace = true +array-init = "2.1.0" +arrow2.workspace = true +arrow2_convert.workspace = true +bytemuck = "1.11" +document-features = "0.2" +half = { workspace = true, features = ["bytemuck"] } +itertools = { workspace = true } +lazy_static.workspace = true +ndarray.workspace = true +nohash-hasher = "0.2" +thiserror.workspace = true +uuid = { version = "1.1", features = ["serde", "v4", "js"] } + + +# Optional dependencies: +ecolor = { workspace = true, optional = true } +glam = { workspace = true, optional = true } +image = { workspace = true, optional = true, default-features = false, features = [ + "jpeg", +] } +rand = { version = "0.8", optional = true } +serde = { version = "1", optional = true, features = ["derive", "rc"] } + +# Native dependencies: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +crossbeam.workspace = true +puffin.workspace = true + + +[dev-dependencies] +rmp-serde = "1.1" +similar-asserts = "1.4.2" diff --git a/crates/re_components/README.md b/crates/re_components/README.md new file mode 100644 index 000000000000..b08b34f7056d --- /dev/null +++ b/crates/re_components/README.md @@ -0,0 +1,10 @@ +# re_log_types + +Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates. + +[![Latest version](https://img.shields.io/crates/v/re_log_types.svg)](https://crates.io/crates/re_log_types) +[![Documentation](https://docs.rs/re_log_types/badge.svg)](https://docs.rs/re_log_types) +![MIT](https://img.shields.io/badge/license-MIT-blue.svg) +![Apache](https://img.shields.io/badge/license-Apache-blue.svg) + +The standard Rerun data types, component types, and archetypes. diff --git a/crates/re_log_types/src/component_types/arrow.rs b/crates/re_components/src/arrow.rs similarity index 92% rename from crates/re_log_types/src/component_types/arrow.rs rename to crates/re_components/src/arrow.rs index 05ef69ee1910..d1b42c54f170 100644 --- a/crates/re_log_types/src/component_types/arrow.rs +++ b/crates/re_components/src/arrow.rs @@ -1,14 +1,12 @@ use super::Vec3D; use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// A 3D Arrow /// /// ## Examples /// /// ``` -/// use re_log_types::component_types::Arrow3D; +/// use re_components::Arrow3D; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -37,9 +35,9 @@ pub struct Arrow3D { pub vector: Vec3D, } -impl Component for Arrow3D { +impl re_log_types::Component for Arrow3D { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.arrow3d".into() } } diff --git a/crates/re_log_types/src/component_types/bbox.rs b/crates/re_components/src/bbox.rs similarity index 94% rename from crates/re_log_types/src/component_types/bbox.rs rename to crates/re_components/src/bbox.rs index d1cdb099aaa2..91d68abe77a8 100644 --- a/crates/re_log_types/src/component_types/bbox.rs +++ b/crates/re_components/src/bbox.rs @@ -6,12 +6,10 @@ use arrow2_convert::{ serialize::ArrowSerialize, }; -use crate::Component; - /// A 3D bounding box represented by it's half-lengths /// /// ``` -/// use re_log_types::component_types::Box3D; +/// use re_components::Box3D; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -39,9 +37,9 @@ impl Box3D { } } -impl Component for Box3D { +impl re_log_types::Component for Box3D { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.box3d".into() } } diff --git a/crates/re_log_types/src/component_types/class_id.rs b/crates/re_components/src/class_id.rs similarity index 86% rename from crates/re_log_types/src/component_types/class_id.rs rename to crates/re_components/src/class_id.rs index e12f9887e2c3..af6f6d98ee96 100644 --- a/crates/re_log_types/src/component_types/class_id.rs +++ b/crates/re_components/src/class_id.rs @@ -1,7 +1,5 @@ use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// A 16-bit ID representing a type of semantic class. /// /// Used to look up a [`crate::context::ClassDescription`] within the [`crate::context::AnnotationContext`]. @@ -23,9 +21,9 @@ use crate::Component; #[arrow_field(transparent)] pub struct ClassId(pub u16); -impl Component for ClassId { +impl re_log_types::Component for ClassId { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.class_id".into() } } diff --git a/crates/re_log_types/src/component_types/color.rs b/crates/re_components/src/color.rs similarity index 93% rename from crates/re_log_types/src/component_types/color.rs rename to crates/re_components/src/color.rs index 6a0a1107140d..ceb4371a2367 100644 --- a/crates/re_log_types/src/component_types/color.rs +++ b/crates/re_components/src/color.rs @@ -1,10 +1,8 @@ -use crate::Component; - /// An RGBA color tuple with unmultiplied/separate alpha, /// in sRGB gamma space with linear alpha. /// /// ``` -/// use re_log_types::component_types::ColorRGBA; +/// use re_components::ColorRGBA; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -61,9 +59,9 @@ impl From<[u8; 4]> for ColorRGBA { } } -impl Component for ColorRGBA { +impl re_log_types::Component for ColorRGBA { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.colorrgba".into() } } diff --git a/crates/re_log_types/src/component_types/context.rs b/crates/re_components/src/context.rs similarity index 97% rename from crates/re_log_types/src/component_types/context.rs rename to crates/re_components/src/context.rs index da802530ef2a..a93ac4e47b92 100644 --- a/crates/re_log_types/src/component_types/context.rs +++ b/crates/re_components/src/context.rs @@ -5,12 +5,7 @@ use arrow2_convert::{ ArrowField, ArrowSerialize, }; -use crate::{ - component_types::{ClassId, KeypointId}, - Component, -}; - -use super::{ColorRGBA, Label}; +use crate::{ClassId, ColorRGBA, KeypointId, Label}; /// Information about an Annotation. /// @@ -111,7 +106,7 @@ impl From for ClassDescription { /// path. /// /// ``` -/// use re_log_types::component_types::AnnotationContext; +/// use re_components::AnnotationContext; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -172,9 +167,9 @@ pub struct AnnotationContext { pub class_map: HashMap, } -impl Component for AnnotationContext { +impl re_log_types::Component for AnnotationContext { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.annotation_context".into() } } diff --git a/crates/re_log_types/src/component_types/coordinates.rs b/crates/re_components/src/coordinates.rs similarity index 98% rename from crates/re_log_types/src/component_types/coordinates.rs rename to crates/re_components/src/coordinates.rs index 9e11df9e5fa7..d7e57901d254 100644 --- a/crates/re_log_types/src/component_types/coordinates.rs +++ b/crates/re_components/src/coordinates.rs @@ -5,8 +5,6 @@ use arrow2_convert::{ serialize::ArrowSerialize, }; -use crate::Component; - /// The six cardinal directions for 3D view-space and image-space. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] @@ -84,7 +82,7 @@ impl TryFrom for ViewDir { /// For 3D view-space and image-space. /// /// ``` -/// use re_log_types::component_types::ViewCoordinates; +/// use re_components::ViewCoordinates; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -97,9 +95,9 @@ impl TryFrom for ViewDir { #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct ViewCoordinates(pub [ViewDir; 3]); -impl Component for ViewCoordinates { +impl re_log_types::Component for ViewCoordinates { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.view_coordinates".into() } } diff --git a/crates/re_components/src/datagen.rs b/crates/re_components/src/datagen.rs new file mode 100644 index 000000000000..7d819f423411 --- /dev/null +++ b/crates/re_components/src/datagen.rs @@ -0,0 +1,149 @@ +//! Generate random data for tests and benchmarks. + +// TODO(#1810): It really is time for whole module to disappear. + +use re_log_types::{InstanceKey, Time, TimeInt, TimeType, Timeline}; + +/// Create `len` dummy rectangles +pub fn build_some_rects(len: usize) -> Vec { + (0..len) + .map(|i| crate::Rect2D::from_xywh(i as f32, i as f32, (i / 2) as f32, (i / 2) as f32)) + .collect() +} + +/// Create `len` dummy colors +pub fn build_some_colors(len: usize) -> Vec { + (0..len).map(|i| crate::ColorRGBA(i as u32)).collect() +} + +/// Create `len` dummy labels +pub fn build_some_labels(len: usize) -> Vec { + (0..len).map(|i| format!("label{i}")).collect() +} + +/// Create `len` dummy `Point2D` +pub fn build_some_point2d(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| crate::Point2D { + x: rng.gen_range(0.0..10.0), + y: rng.gen_range(0.0..10.0), + }) + .collect() +} + +/// Create `len` dummy `Vec3D` +pub fn build_some_vec3d(len: usize) -> Vec { + use rand::Rng as _; + let mut rng = rand::thread_rng(); + + (0..len) + .map(|_| { + crate::Vec3D::new( + rng.gen_range(0.0..10.0), + rng.gen_range(0.0..10.0), + rng.gen_range(0.0..10.0), + ) + }) + .collect() +} + +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `log_time` suitable for inserting in a [`re_log_types::TimePoint`]. +pub fn build_log_time(log_time: Time) -> (Timeline, TimeInt) { + (Timeline::log_time(), log_time.into()) +} + +/// Build a ([`Timeline`], [`TimeInt`]) tuple from `frame_nr` suitable for inserting in a [`re_log_types::TimePoint`]. +pub fn build_frame_nr(frame_nr: TimeInt) -> (Timeline, TimeInt) { + (Timeline::new("frame_nr", TimeType::Sequence), frame_nr) +} + +/// Create `len` dummy `InstanceKey` keys. These keys will be sorted. +pub fn build_some_instances(num_instances: usize) -> Vec { + use rand::seq::SliceRandom; + let mut rng = rand::thread_rng(); + + // Allocate pool of 10x the potential instance keys, draw a random sampling, and then sort it + let mut instance_pool = (0..(num_instances * 10)).collect::>(); + let (rand_instances, _) = instance_pool.partial_shuffle(&mut rng, num_instances); + let mut sorted_instances = rand_instances.to_vec(); + sorted_instances.sort(); + + sorted_instances + .into_iter() + .map(|id| InstanceKey(id as u64)) + .collect() +} + +pub fn build_some_instances_from(instances: impl IntoIterator) -> Vec { + let mut instances = instances.into_iter().map(InstanceKey).collect::>(); + instances.sort(); + instances +} + +/// Crafts a simple but interesting [`re_log_types::DataTable`]. +#[cfg(not(target_arch = "wasm32"))] +pub fn data_table_example(timeless: bool) -> re_log_types::DataTable { + use crate::{ColorRGBA, Label, Point2D}; + use re_log_types::{DataRow, DataTable, RowId, TableId, TimePoint}; + + let table_id = TableId::random(); + + let mut tick = 0i64; + let mut timepoint = |frame_nr: i64| { + let tp = if timeless { + TimePoint::timeless() + } else { + TimePoint::from([ + (Timeline::log_time(), Time::now().into()), + (Timeline::log_tick(), tick.into()), + (Timeline::new_sequence("frame_nr"), frame_nr.into()), + ]) + }; + tick += 1; + tp + }; + + let row0 = { + let num_instances = 2; + let points: &[Point2D] = &[[10.0, 10.0].into(), [20.0, 20.0].into()]; + let colors: &[_] = &[ColorRGBA::from_rgb(128, 128, 128)]; + let labels: &[Label] = &[]; + + DataRow::from_cells3( + RowId::random(), + "a", + timepoint(1), + num_instances, + (points, colors, labels), + ) + }; + + let row1 = { + let num_instances = 0; + let colors: &[ColorRGBA] = &[]; + + DataRow::from_cells1(RowId::random(), "b", timepoint(1), num_instances, colors) + }; + + let row2 = { + let num_instances = 1; + let colors: &[_] = &[ColorRGBA::from_rgb(255, 255, 255)]; + let labels: &[_] = &[Label("hey".into())]; + + DataRow::from_cells2( + RowId::random(), + "c", + timepoint(2), + num_instances, + (colors, labels), + ) + }; + + let mut table = DataTable::from_rows(table_id, [row0, row1, row2]); + table.compute_all_size_bytes(); + + table +} diff --git a/crates/re_log_types/src/component_types/disconnected_space.rs b/crates/re_components/src/disconnected_space.rs similarity index 90% rename from crates/re_log_types/src/component_types/disconnected_space.rs rename to crates/re_components/src/disconnected_space.rs index 07102ec26d57..689d96765699 100644 --- a/crates/re_log_types/src/component_types/disconnected_space.rs +++ b/crates/re_components/src/disconnected_space.rs @@ -1,7 +1,5 @@ use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// Specifies that the entity path at which this is logged is disconnected from its parent. /// /// If a transform or pinhole is logged on the same path, this component will be ignored. @@ -28,9 +26,9 @@ impl Default for DisconnectedSpace { } } -impl Component for DisconnectedSpace { +impl re_log_types::Component for DisconnectedSpace { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.disconnected_space".into() } } diff --git a/crates/re_log_types/src/component_types/draw_order.rs b/crates/re_components/src/draw_order.rs similarity index 94% rename from crates/re_log_types/src/component_types/draw_order.rs rename to crates/re_components/src/draw_order.rs index 9741d9413dab..72881cf40e38 100644 --- a/crates/re_log_types/src/component_types/draw_order.rs +++ b/crates/re_components/src/draw_order.rs @@ -1,7 +1,5 @@ use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// Draw order used for the display order of 2D elements. /// /// Higher values are drawn on top of lower values. @@ -13,7 +11,7 @@ use crate::Component; /// This component is a "mono-component". See [the crate level docs](crate) for details. /// /// ``` -/// use re_log_types::component_types::DrawOrder; +/// use re_components::DrawOrder; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -38,9 +36,9 @@ impl DrawOrder { pub const DEFAULT_POINTS2D: DrawOrder = DrawOrder(30.0); } -impl Component for DrawOrder { +impl re_log_types::Component for DrawOrder { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.draw_order".into() } } diff --git a/crates/re_log_types/src/component_types/keypoint_id.rs b/crates/re_components/src/keypoint_id.rs similarity index 88% rename from crates/re_log_types/src/component_types/keypoint_id.rs rename to crates/re_components/src/keypoint_id.rs index 0381b81f44d4..edc2133ffed9 100644 --- a/crates/re_log_types/src/component_types/keypoint_id.rs +++ b/crates/re_components/src/keypoint_id.rs @@ -1,7 +1,5 @@ use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// A 16-bit ID representing a type of semantic keypoint within a class. /// /// `KeypointId`s are only meaningful within the context of a [`crate::context::ClassDescription`]. @@ -25,9 +23,9 @@ use crate::Component; #[arrow_field(transparent)] pub struct KeypointId(pub u16); -impl Component for KeypointId { +impl re_log_types::Component for KeypointId { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.keypoint_id".into() } } diff --git a/crates/re_log_types/src/component_types/label.rs b/crates/re_components/src/label.rs similarity index 89% rename from crates/re_log_types/src/component_types/label.rs rename to crates/re_components/src/label.rs index af29c6609cee..fa5dde5458d2 100644 --- a/crates/re_log_types/src/component_types/label.rs +++ b/crates/re_components/src/label.rs @@ -1,11 +1,9 @@ use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; -use crate::Component; - /// A String label component /// /// ``` -/// use re_log_types::component_types::Label; +/// use re_components::Label; /// use arrow2_convert::field::ArrowField; /// use arrow2::datatypes::{DataType, Field}; /// @@ -23,9 +21,9 @@ impl Label { } } -impl Component for Label { +impl re_log_types::Component for Label { #[inline] - fn name() -> crate::ComponentName { + fn name() -> re_log_types::ComponentName { "rerun.label".into() } } diff --git a/crates/re_log_types/src/component_types/mod.rs b/crates/re_components/src/lib.rs similarity index 74% rename from crates/re_log_types/src/component_types/mod.rs rename to crates/re_components/src/lib.rs index bd681e05ee6b..01133aab1144 100644 --- a/crates/re_log_types/src/component_types/mod.rs +++ b/crates/re_components/src/lib.rs @@ -1,7 +1,11 @@ -//! Potentially user-facing component types. +//! User-facing data types, component types, and archetypes. //! //! The SDK is responsible for submitting component columns that conforms to these schemas. The //! schemas are additionally documented in doctests. +//! +//! ## Feature flags +#![doc = document_features::document_features!()] +//! use arrow2::{ array::{FixedSizeListArray, MutableFixedSizeListArray, PrimitiveArray}, @@ -14,8 +18,6 @@ use arrow2_convert::{ }; use lazy_static::lazy_static; -use crate::Component; - mod arrow; mod bbox; mod class_id; @@ -24,7 +26,6 @@ pub mod context; pub mod coordinates; mod disconnected_space; mod draw_order; -mod instance_key; mod keypoint_id; mod label; mod linestrip; @@ -38,45 +39,77 @@ mod rect; mod scalar; mod size; mod tensor; +mod tensor_data; mod text_box; mod text_entry; mod transform3d; mod vec; -pub use arrow::Arrow3D; -pub use bbox::Box3D; -pub use class_id::ClassId; -pub use color::ColorRGBA; -pub use context::{AnnotationContext, AnnotationInfo, ClassDescription}; -pub use coordinates::ViewCoordinates; -pub use disconnected_space::DisconnectedSpace; -pub use draw_order::DrawOrder; -pub use instance_key::InstanceKey; -pub use keypoint_id::KeypointId; -pub use label::Label; -pub use linestrip::{LineStrip2D, LineStrip3D}; -pub use mat::Mat3x3; -pub use mesh3d::{EncodedMesh3D, Mesh3D, MeshFormat, MeshId, RawMesh3D}; -pub use pinhole::Pinhole; -pub use point::{Point2D, Point3D}; -pub use quaternion::Quaternion; -pub use radius::Radius; -pub use rect::Rect2D; -pub use scalar::{Scalar, ScalarPlotProps}; -pub use size::Size3D; -pub use tensor::{ - DecodedTensor, Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension, - TensorId, +#[cfg(not(target_arch = "wasm32"))] +mod load_file; + +#[cfg(feature = "arrow_datagen")] +pub mod datagen; + +// ---------------------------------------------------------------------------- +// TODO(emilk): split into modules, like we do in re_sdk/src/lib.rs + +pub use self::{ + arrow::Arrow3D, + bbox::Box3D, + class_id::ClassId, + color::ColorRGBA, + context::{AnnotationContext, AnnotationInfo, ClassDescription}, + coordinates::ViewCoordinates, + disconnected_space::DisconnectedSpace, + draw_order::DrawOrder, + keypoint_id::KeypointId, + label::Label, + linestrip::{LineStrip2D, LineStrip3D}, + mat::Mat3x3, + mesh3d::{EncodedMesh3D, Mesh3D, MeshFormat, MeshId, RawMesh3D}, + pinhole::Pinhole, + point::{Point2D, Point3D}, + quaternion::Quaternion, + radius::Radius, + rect::Rect2D, + scalar::{Scalar, ScalarPlotProps}, + size::Size3D, + tensor::{ + DecodedTensor, Tensor, TensorCastError, TensorData, TensorDataMeaning, TensorDimension, + TensorId, + }, + tensor_data::{TensorDataType, TensorDataTypeTrait, TensorElement}, + text_box::TextBox, + text_entry::TextEntry, + transform3d::{ + Angle, Rotation3D, RotationAxisAngle, Scale3D, Transform3D, Transform3DRepr, + TranslationAndMat3, TranslationRotationScale3D, + }, + vec::{Vec2D, Vec3D, Vec4D}, }; + #[cfg(feature = "image")] -pub use tensor::{TensorImageLoadError, TensorImageSaveError}; -pub use text_box::TextBox; -pub use text_entry::TextEntry; -pub use transform3d::{ - Angle, Rotation3D, RotationAxisAngle, Scale3D, Transform3D, Transform3DRepr, - TranslationAndMat3, TranslationRotationScale3D, -}; -pub use vec::{Vec2D, Vec3D, Vec4D}; +pub use self::tensor::{TensorImageLoadError, TensorImageSaveError}; + +#[cfg(not(target_arch = "wasm32"))] +pub use self::load_file::{data_cell_from_file_path, data_cell_from_mesh_file_path, FromFileError}; + +// This is a component +pub use re_log_types::InstanceKey; + +// This is very convenient to re-export +pub use re_log_types::Component; + +pub mod external { + #[cfg(feature = "glam")] + pub use glam; + + #[cfg(feature = "image")] + pub use image; +} + +// ---------------------------------------------------------------------------- lazy_static! { //TODO(john): use a run-time type registry @@ -88,7 +121,7 @@ lazy_static! { ::field(), ::field(), ::field(), - ::field(), + ::field(), ::field(),