Implement ChunkStore and integrate it everywhere (#6570)

Implements the new chunk store and integrates it all over the place.

The highest-level query APIs and caches are unchanged -- for now.

- Fixes #6066
- Fixes #4415
- Fixes #442
- Fixes #1894
 
_Requires 0.17 to be branched off of main before merging._

ARCHITECTURE.md

@@ -39,7 +39,7 @@ NOTE: `.rrd` files do not yet guarantee any backwards or forwards compatibility.
 
 ## Technologies we use
 ### Apache Arrow
-[Apache Arrow](https://arrow.apache.org/) is a language-independent columnar memory format for arbitrary data. We use it to encode the log data when transmitting it over the network or storing it in an `.rrd` file. We also use it in our in-RAM data store, [`re_data_store`](crates/re_data_store/README.md).
+[Apache Arrow](https://arrow.apache.org/) is a language-independent columnar memory format for arbitrary data. We use it to encode the log data when transmitting it over the network or storing it in an `.rrd` file. We also use it in our in-RAM data store, [`re_chunk_store`](crates/re_chunk_store/README.md).
 
 In Rust, we use the [`arrow2` crate](https://crates.io/crates/arrow2).
 
@@ -88,11 +88,11 @@ Of course, this will only take us so far. In the future we plan on caching queri
 Here is an overview of the crates included in the project:
 
 <picture>
-  <img src="https://static.rerun.io/crates/710562417756943b47657d3260622b5effe392f7/full.png" alt="">
-  <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/710562417756943b47657d3260622b5effe392f7/480w.png">
-  <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/710562417756943b47657d3260622b5effe392f7/768w.png">
-  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/710562417756943b47657d3260622b5effe392f7/1024w.png">
-  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/710562417756943b47657d3260622b5effe392f7/1200w.png">
+  <img src="https://static.rerun.io/crates/ea88def8bf549cd0d3abbe85ae787cf1a4db03bc/full.png" alt="">
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/ea88def8bf549cd0d3abbe85ae787cf1a4db03bc/480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/ea88def8bf549cd0d3abbe85ae787cf1a4db03bc/768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/ea88def8bf549cd0d3abbe85ae787cf1a4db03bc/1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/ea88def8bf549cd0d3abbe85ae787cf1a4db03bc/1200w.png">
 </picture>
 
 
@@ -160,7 +160,7 @@ Update instructions:
 | Crate                | Description                                                              |
 |----------------------|--------------------------------------------------------------------------|
 | re_entity_db         | In-memory storage of Rerun entities                                      |
-| re_query             | Querying data in the re_data_store                                       |
+| re_query             | Querying data in the re_chunk_store                                      |
 | re_types             | The built-in Rerun data types, component types, and archetypes.          |
 | re_types_blueprint   | The core traits and types that power Rerun's Blueprint sub-system.       |
 | re_log_encoding      | Helpers for encoding and transporting Rerun log messages                 |
@@ -171,7 +171,7 @@ Update instructions:
 | Crate           | Description                                                                                   |
 |-----------------|-----------------------------------------------------------------------------------------------|
 | re_chunk        | A chunk of Rerun data, encoded using Arrow. Used for logging, transport, storage and compute. |
-| re_data_store   | An in-memory time series database for Rerun log data, based on Apache Arrow.                  |
+| re_chunk_store  | An in-memory time series database for Rerun log data, based on Apache Arrow.                  |
 | re_log_types    | The basic building blocks of the Rerun data types and tables.                                 |
 | re_types_core   | The core traits and types that power Rerun's data model.                                      |
 | re_format_arrow | Formatting of Apache Arrow tables.                                                            |

Cargo.lock

@@ -4305,12 +4305,46 @@ dependencies = [
  "re_tracing",
  "re_tuid",
  "re_types_core",
+ "serde",
+ "serde_bytes",
  "similar-asserts",
  "smallvec",
  "static_assertions",
  "thiserror",
 ]
 
+[[package]]
+name = "re_chunk_store"
+version = "0.17.0-alpha.9"
+dependencies = [
+ "ahash",
+ "anyhow",
+ "criterion",
+ "document-features",
+ "indent",
+ "insta",
+ "itertools 0.13.0",
+ "mimalloc",
+ "nohash-hasher",
+ "once_cell",
+ "parking_lot",
+ "rand",
+ "re_arrow2",
+ "re_chunk",
+ "re_format",
+ "re_format_arrow",
+ "re_log",
+ "re_log_types",
+ "re_tracing",
+ "re_types",
+ "re_types_core",
+ "similar-asserts",
+ "smallvec",
+ "thiserror",
+ "tinyvec",
+ "web-time",
+]
+
 [[package]]
 name = "re_context_menu"
 version = "0.17.0-alpha.9"
@@ -4356,6 +4390,7 @@ dependencies = [
  "rayon",
  "re_build_info",
  "re_build_tools",
+ "re_chunk",
  "re_log",
  "re_log_encoding",
  "re_log_types",
@@ -4383,37 +4418,6 @@ dependencies = [
  "re_ws_comms",
 ]
 
-[[package]]
-name = "re_data_store"
-version = "0.17.0-alpha.9"
-dependencies = [
- "ahash",
- "anyhow",
- "criterion",
- "document-features",
- "indent",
- "insta",
- "itertools 0.13.0",
- "mimalloc",
- "nohash-hasher",
- "once_cell",
- "parking_lot",
- "rand",
- "re_arrow2",
- "re_format",
- "re_format_arrow",
- "re_log",
- "re_log_types",
- "re_tracing",
- "re_types",
- "re_types_core",
- "similar-asserts",
- "smallvec",
- "thiserror",
- "tinyvec",
- "web-time",
-]
-
 [[package]]
 name = "re_data_ui"
 version = "0.17.0-alpha.9"
@@ -4426,7 +4430,7 @@ dependencies = [
  "egui_plot",
  "image",
  "itertools 0.13.0",
- "re_data_store",
+ "re_chunk_store",
  "re_entity_db",
  "re_error",
  "re_format",
@@ -4498,7 +4502,8 @@ dependencies = [
  "parking_lot",
  "rand",
  "re_build_info",
- "re_data_store",
+ "re_chunk",
+ "re_chunk_store",
  "re_format",
  "re_int_histogram",
  "re_log",
@@ -4574,6 +4579,7 @@ dependencies = [
  "mimalloc",
  "parking_lot",
  "re_build_info",
+ "re_chunk",
  "re_log",
  "re_log_types",
  "re_smart_channel",
@@ -4666,7 +4672,8 @@ dependencies = [
  "paste",
  "rand",
  "re_arrow2",
- "re_data_store",
+ "re_chunk",
+ "re_chunk_store",
  "re_error",
  "re_format",
  "re_log",
@@ -4770,8 +4777,8 @@ dependencies = [
  "re_build_info",
  "re_build_tools",
  "re_chunk",
+ "re_chunk_store",
  "re_data_loader",
- "re_data_store",
  "re_log",
  "re_log_encoding",
  "re_log_types",
@@ -4811,8 +4818,9 @@ dependencies = [
  "itertools 0.13.0",
  "nohash-hasher",
  "once_cell",
+ "re_chunk",
+ "re_chunk_store",
  "re_context_menu",
- "re_data_store",
  "re_data_ui",
  "re_entity_db",
  "re_log",
@@ -4850,7 +4858,7 @@ dependencies = [
  "ahash",
  "egui",
  "nohash-hasher",
- "re_data_store",
+ "re_chunk_store",
  "re_entity_db",
  "re_log",
  "re_log_types",
@@ -4868,7 +4876,7 @@ version = "0.17.0-alpha.9"
 dependencies = [
  "egui",
  "egui_plot",
- "re_data_store",
+ "re_chunk_store",
  "re_entity_db",
  "re_log",
  "re_log_types",
@@ -4887,7 +4895,7 @@ version = "0.17.0-alpha.9"
 dependencies = [
  "egui",
  "egui_extras",
- "re_data_store",
+ "re_chunk_store",
  "re_data_ui",
  "re_entity_db",
  "re_log_types",
@@ -4914,7 +4922,7 @@ dependencies = [
  "mimalloc",
  "nohash-hasher",
  "once_cell",
- "re_data_store",
+ "re_chunk_store",
  "re_data_ui",
  "re_entity_db",
  "re_error",
@@ -4944,7 +4952,7 @@ dependencies = [
  "egui",
  "half 2.3.1",
  "ndarray",
- "re_data_store",
+ "re_chunk_store",
  "re_data_ui",
  "re_entity_db",
  "re_log",
@@ -4967,7 +4975,7 @@ version = "0.17.0-alpha.9"
 dependencies = [
  "egui",
  "egui_commonmark",
- "re_data_store",
+ "re_chunk_store",
  "re_renderer",
  "re_space_view",
  "re_tracing",
@@ -4982,7 +4990,7 @@ version = "0.17.0-alpha.9"
 dependencies = [
  "egui",
  "egui_extras",
- "re_data_store",
+ "re_chunk_store",
  "re_data_ui",
  "re_entity_db",
  "re_log",
@@ -5004,7 +5012,7 @@ dependencies = [
  "egui_plot",
  "itertools 0.13.0",
  "rayon",
- "re_data_store",
+ "re_chunk_store",
  "re_format",
  "re_log",
  "re_log_types",
@@ -5036,8 +5044,8 @@ version = "0.17.0-alpha.9"
 dependencies = [
  "egui",
  "itertools 0.13.0",
+ "re_chunk_store",
  "re_context_menu",
- "re_data_store",
  "re_data_ui",
  "re_entity_db",
  "re_format",
@@ -5220,9 +5228,10 @@ dependencies = [
  "re_blueprint_tree",
  "re_build_info",
  "re_build_tools",
+ "re_chunk",
+ "re_chunk_store",
  "re_data_loader",
  "re_data_source",
- "re_data_store",
  "re_data_ui",
  "re_edit_ui",
  "re_entity_db",
@@ -5294,8 +5303,9 @@ dependencies = [
  "nohash-hasher",
  "once_cell",
  "parking_lot",
+ "re_chunk",
+ "re_chunk_store",
  "re_data_source",
- "re_data_store",
  "re_entity_db",
  "re_error",
  "re_format",
@@ -5354,7 +5364,8 @@ dependencies = [
  "nohash-hasher",
  "once_cell",
  "parking_lot",
- "re_data_store",
+ "re_chunk",
+ "re_chunk_store",
  "re_entity_db",
  "re_log",
  "re_log_types",
@@ -5482,6 +5493,7 @@ dependencies = [
  "re_analytics",
  "re_build_info",
  "re_build_tools",
+ "re_chunk",
  "re_crash_handler",
  "re_data_source",
  "re_entity_db",
@@ -5498,6 +5510,7 @@ dependencies = [
  "re_viewer",
  "re_web_viewer_server",
  "re_ws_comms",
+ "similar-asserts",
 ]
 
 [[package]]

Cargo.toml

@@ -34,6 +34,7 @@ re_build_info = { path = "crates/re_build_info", version = "=0.17.0-alpha.9", de
 re_build_tools = { path = "crates/re_build_tools", version = "=0.17.0-alpha.9", default-features = false }
 re_case = { path = "crates/re_case", version = "=0.17.0-alpha.9", default-features = false }
 re_chunk = { path = "crates/re_chunk", version = "=0.17.0-alpha.9", default-features = false }
+re_chunk_store = { path = "crates/re_chunk_store", version = "=0.17.0-alpha.9", default-features = false }
 re_context_menu = { path = "crates/re_context_menu", version = "=0.17.0-alpha.9", default-features = false }
 re_crash_handler = { path = "crates/re_crash_handler", version = "=0.17.0-alpha.9", default-features = false }
 re_data_loader = { path = "crates/re_data_loader", version = "=0.17.0-alpha.9", default-features = false }

crates/re_analytics/src/event.rs

@@ -75,7 +75,7 @@ pub struct OpenRecording {
     pub data_source: Option<&'static str>,
 }
 
-/// Basic information about a recording's data store.
+/// Basic information about a recording's chunk store.
 pub struct StoreInfo {
     /// Name of the application.
     ///

crates/re_chunk/Cargo.toml

@@ -22,6 +22,16 @@ all-features = true
 [features]
 default = []
 
+## Enable (de)serialization using serde.
+serde = [
+  "dep:serde",
+  "dep:serde_bytes",
+  "re_log_types/serde",
+  "re_string_interner/serde",
+  "re_tuid/serde",
+  "re_types_core/serde",
+]
+
 
 [dependencies]
 
@@ -40,10 +50,8 @@ re_types_core.workspace = true
 ahash.workspace = true
 anyhow.workspace = true
 arrow2 = { workspace = true, features = [
-  "io_ipc",
-  "io_print",
-  "compute_comparison",
   "compute_concatenate",
+  "compute_filter",
 ] }
 backtrace.workspace = true
 document-features.workspace = true
@@ -55,12 +63,17 @@ smallvec.workspace = true
 static_assertions.workspace = true
 thiserror.workspace = true
 
+# Optional dependencies:
+serde = { workspace = true, optional = true, features = ["derive", "rc"] }
+serde_bytes = { workspace = true, optional = true }
+
 # Native dependencies:
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 crossbeam.workspace = true
 
 
 [dev-dependencies]
+re_log = { workspace = true, features = ["setup"] }
 criterion.workspace = true
 mimalloc.workspace = true
 similar-asserts.workspace = true