Skip to content

Commit

Permalink
perf: Back arrow arrays with SharedStorage which can have non-refcoun…
Browse files Browse the repository at this point in the history
…ted static slices (#18666)
  • Loading branch information
orlp authored Sep 11, 2024
1 parent 9d01442 commit 43cc962
Show file tree
Hide file tree
Showing 13 changed files with 322 additions and 349 deletions.
6 changes: 3 additions & 3 deletions crates/polars-arrow/src/array/binview/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,10 +403,10 @@ impl<T: ViewType + ?Sized> BinaryViewArrayGeneric<T> {
self.buffers
.iter()
.map(|buf| {
if buf.shared_count_strong() == 1 {
buf.len()
} else {
if buf.storage_refcount() > 1 {
0
} else {
buf.len()
}
})
.sum()
Expand Down
10 changes: 5 additions & 5 deletions crates/polars-arrow/src/array/static_array_collect.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use std::borrow::Cow;
use std::sync::Arc;

use polars_utils::no_call_const;

Expand All @@ -15,6 +14,7 @@ use crate::datatypes::ArrowDataType;
use crate::legacy::prelude::fixed_size_list::AnonymousBuilder as AnonymousFixedSizeListArrayBuilder;
use crate::legacy::prelude::list::AnonymousBuilder as AnonymousListArrayBuilder;
use crate::legacy::trusted_len::TrustedLenPush;
use crate::storage::SharedStorage;
use crate::trusted_len::TrustedLen;
use crate::types::NativeType;

Expand Down Expand Up @@ -256,7 +256,7 @@ macro_rules! impl_collect_vec_validity {
unsafe {
// SAFETY: we made sure the null_count is correct.
Some(Bitmap::from_inner_unchecked(
Arc::new(bitmap.into()),
SharedStorage::from_vec(bitmap),
0,
buf.len(),
Some(null_count),
Expand Down Expand Up @@ -317,7 +317,7 @@ macro_rules! impl_trusted_collect_vec_validity {
unsafe {
// SAFETY: we made sure the null_count is correct.
Some(Bitmap::from_inner_unchecked(
Arc::new(bitmap.into()),
SharedStorage::from_vec(bitmap),
0,
buf.len(),
Some(null_count),
Expand Down Expand Up @@ -766,15 +766,15 @@ macro_rules! impl_collect_bool_validity {

let false_count = len - true_count;
let values = unsafe {
Bitmap::from_inner_unchecked(Arc::new(buf.into()), 0, len, Some(false_count))
Bitmap::from_inner_unchecked(SharedStorage::from_vec(buf), 0, len, Some(false_count))
};

let null_count = len - nonnull_count;
let validity_bitmap = if $with_valid && null_count > 0 {
unsafe {
// SAFETY: we made sure the null_count is correct.
Some(Bitmap::from_inner_unchecked(
Arc::new(validity.into()),
SharedStorage::from_vec(validity),
0,
len,
Some(null_count),
Expand Down
Loading

0 comments on commit 43cc962

Please sign in to comment.