From 42de80ca67b28f482cef257ea36c449da123498c Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 8 Mar 2023 17:34:30 +0100 Subject: [PATCH 1/3] disable compaction until we support batching --- crates/re_arrow_store/src/store_write.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs index 5b0e06bac623..2ecdb5d2e51f 100644 --- a/crates/re_arrow_store/src/store_write.rs +++ b/crates/re_arrow_store/src/store_write.rs @@ -1107,8 +1107,13 @@ impl ComponentTable { "allocating new component bucket, previous one overflowed" ); - // Archive currently active bucket. - active_bucket.archive(); + // TODO(cmc): Compaction is disabled until we implement batching. + // See for rationale. + // + // This has no noticeable impact on importance. + // + // // Archive currently active bucket. + // active_bucket.archive(); let row_offset = active_bucket.row_offset + len; self.buckets @@ -1215,6 +1220,7 @@ impl ComponentBucket { /// Archives the bucket as a new one is about to take its place. /// /// This is a good opportunity to run compaction and other maintenance related tasks. + #[allow(dead_code)] pub fn archive(&mut self) { crate::profile_function!(); From 8c2ebe163e444ed5a24ab68f3a5332c508a06cd5 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Wed, 8 Mar 2023 17:51:21 +0100 Subject: [PATCH 2/3] link --- crates/re_arrow_store/src/store_write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs index 2ecdb5d2e51f..ac869c0492c7 100644 --- a/crates/re_arrow_store/src/store_write.rs +++ b/crates/re_arrow_store/src/store_write.rs @@ -1108,7 +1108,7 @@ impl ComponentTable { ); // TODO(cmc): Compaction is disabled until we implement batching. - // See for rationale. + // See https://github.com/rerun-io/rerun/pull/1535 for rationale. // // This has no noticeable impact on importance. // From 90bb1decc9d5dc17bda94bed50cacefcd3fdf100 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Thu, 9 Mar 2023 23:27:18 +0100 Subject: [PATCH 3/3] make it configurable --- crates/re_arrow_store/src/store.rs | 8 ++++++++ crates/re_arrow_store/src/store_write.rs | 10 +++------- crates/re_arrow_store/src/test_util.rs | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/re_arrow_store/src/store.rs b/crates/re_arrow_store/src/store.rs index c31d3be17ae4..153849e46f0d 100644 --- a/crates/re_arrow_store/src/store.rs +++ b/crates/re_arrow_store/src/store.rs @@ -178,6 +178,9 @@ pub struct DataStoreConfig { /// /// See [`DataStore::insert_id_key`]. pub store_insert_ids: bool, + + /// Should soon-to-be inactive buckets be compacted before being archived? + pub enable_compaction: bool, } impl Default for DataStoreConfig { @@ -193,6 +196,11 @@ impl DataStoreConfig { index_bucket_size_bytes: 32 * 1024, // 32kiB index_bucket_nb_rows: 1024, store_insert_ids: cfg!(debug_assertions), + // TODO(cmc): Compaction is disabled until we implement batching. + // See https://github.com/rerun-io/rerun/pull/1535 for rationale. + // + // This has no noticeable impact on performance. + enable_compaction: false, }; } diff --git a/crates/re_arrow_store/src/store_write.rs b/crates/re_arrow_store/src/store_write.rs index ac869c0492c7..0521c8eb355e 100644 --- a/crates/re_arrow_store/src/store_write.rs +++ b/crates/re_arrow_store/src/store_write.rs @@ -1107,13 +1107,9 @@ impl ComponentTable { "allocating new component bucket, previous one overflowed" ); - // TODO(cmc): Compaction is disabled until we implement batching. - // See https://github.com/rerun-io/rerun/pull/1535 for rationale. - // - // This has no noticeable impact on importance. - // - // // Archive currently active bucket. - // active_bucket.archive(); + if config.enable_compaction { + active_bucket.archive(); + } let row_offset = active_bucket.row_offset + len; self.buckets diff --git a/crates/re_arrow_store/src/test_util.rs b/crates/re_arrow_store/src/test_util.rs index aebccc2b3243..b739d795fd50 100644 --- a/crates/re_arrow_store/src/test_util.rs +++ b/crates/re_arrow_store/src/test_util.rs @@ -104,6 +104,7 @@ pub fn all_configs() -> impl Iterator { index_bucket_size_bytes: idx.index_bucket_size_bytes, index_bucket_nb_rows: idx.index_bucket_nb_rows, store_insert_ids: comp.store_insert_ids || idx.store_insert_ids, + enable_compaction: comp.enable_compaction || idx.enable_compaction, }) }) }