feat: add architecture for polars-flavored IPC (#13734)

crates/polars-core/src/chunked_array/arithmetic/decimal.rs

@@ -104,8 +104,8 @@ impl Add for &DecimalChunked {
         self.arithmetic_helper(
             rhs,
             decimal::add,
-            |lhs, rhs_val| decimal::add_scalar(lhs, rhs_val, &rhs.dtype().to_arrow()),
-            |lhs_val, rhs| decimal::add_scalar(rhs, lhs_val, &self.dtype().to_arrow()),
+            |lhs, rhs_val| decimal::add_scalar(lhs, rhs_val, &rhs.dtype().to_arrow(true)),
+            |lhs_val, rhs| decimal::add_scalar(rhs, lhs_val, &self.dtype().to_arrow(true)),
         )
     }
 }
@@ -130,8 +130,8 @@ impl Mul for &DecimalChunked {
         self.arithmetic_helper(
             rhs,
             decimal::mul,
-            |lhs, rhs_val| decimal::mul_scalar(lhs, rhs_val, &rhs.dtype().to_arrow()),
-            |lhs_val, rhs| decimal::mul_scalar(rhs, lhs_val, &self.dtype().to_arrow()),
+            |lhs, rhs_val| decimal::mul_scalar(lhs, rhs_val, &rhs.dtype().to_arrow(true)),
+            |lhs_val, rhs| decimal::mul_scalar(rhs, lhs_val, &self.dtype().to_arrow(true)),
         )
     }
 }
@@ -143,8 +143,8 @@ impl Div for &DecimalChunked {
         self.arithmetic_helper(
             rhs,
             decimal::div,
-            |lhs, rhs_val| decimal::div_scalar(lhs, rhs_val, &rhs.dtype().to_arrow()),
-            |lhs_val, rhs| decimal::div_scalar_swapped(lhs_val, &self.dtype().to_arrow(), rhs),
+            |lhs, rhs_val| decimal::div_scalar(lhs, rhs_val, &rhs.dtype().to_arrow(true)),
+            |lhs_val, rhs| decimal::div_scalar_swapped(lhs_val, &self.dtype().to_arrow(true), rhs),
         )
     }
 }

crates/polars-core/src/chunked_array/array/mod.rs

@@ -32,7 +32,7 @@ impl ArrayChunked {
     /// Get the inner values as `Series`
     pub fn get_inner(&self) -> Series {
         let ca = self.rechunk();
-        let inner_dtype = self.inner_dtype().to_arrow();
+        let inner_dtype = self.inner_dtype().to_arrow(true);
         let arr = ca.downcast_iter().next().unwrap();
         unsafe {
             Series::_try_from_arrow_unchecked(
@@ -51,7 +51,7 @@ impl ArrayChunked {
     ) -> PolarsResult<ArrayChunked> {
         // Rechunk or the generated Series will have wrong length.
         let ca = self.rechunk();
-        let inner_dtype = self.inner_dtype().to_arrow();
+        let inner_dtype = self.inner_dtype().to_arrow(true);
 
         let chunks = ca.downcast_iter().map(|arr| {
             let elements = unsafe {
@@ -73,7 +73,7 @@ impl ArrayChunked {
             let values = out.chunks()[0].clone();
 
             let inner_dtype =
-                FixedSizeListArray::default_datatype(out.dtype().to_arrow(), ca.width());
+                FixedSizeListArray::default_datatype(out.dtype().to_arrow(true), ca.width());
             let arr = FixedSizeListArray::new(inner_dtype, values, arr.validity().cloned());
             Ok(arr)
         });

crates/polars-core/src/chunked_array/builder/fixed_size_list.rs

@@ -124,7 +124,12 @@ impl FixedSizeListBuilder for AnonymousOwnedFixedSizeListBuilder {
 
     fn finish(&mut self) -> ArrayChunked {
         let arr = std::mem::take(&mut self.inner)
-            .finish(self.inner_dtype.as_ref().map(|dt| dt.to_arrow()).as_ref())
+            .finish(
+                self.inner_dtype
+                    .as_ref()
+                    .map(|dt| dt.to_arrow(true))
+                    .as_ref(),
+            )
             .unwrap();
         ChunkedArray::with_chunk(self.name.as_str(), arr)
     }

crates/polars-core/src/chunked_array/builder/list/anonymous.rs

@@ -87,7 +87,9 @@ impl<'a> AnonymousListBuilder<'a> {
         } else {
             let inner_dtype = slf.inner_dtype.materialize();
 
-            let inner_dtype_physical = inner_dtype.as_ref().map(|dt| dt.to_physical().to_arrow());
+            let inner_dtype_physical = inner_dtype
+                .as_ref()
+                .map(|dt| dt.to_physical().to_arrow(true));
             let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
 
             let list_dtype_logical = match inner_dtype {
@@ -153,7 +155,9 @@ impl ListBuilderTrait for AnonymousOwnedListBuilder {
         let inner_dtype = std::mem::take(&mut self.inner_dtype).materialize();
         // Don't use self from here on out.
         let slf = std::mem::take(self);
-        let inner_dtype_physical = inner_dtype.as_ref().map(|dt| dt.to_physical().to_arrow());
+        let inner_dtype_physical = inner_dtype
+            .as_ref()
+            .map(|dt| dt.to_physical().to_arrow(true));
         let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
 
         let list_dtype_logical = match inner_dtype {

crates/polars-core/src/chunked_array/builder/mod.rs

@@ -71,7 +71,7 @@ where
     T: PolarsNumericType,
 {
     fn from_slice(name: &str, v: &[T::Native]) -> Self {
-        let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow());
+        let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow(true));
         ChunkedArray::with_chunk(name, arr)
     }
 

crates/polars-core/src/chunked_array/builder/primitive.rs

@@ -50,7 +50,7 @@ where
 {
     pub fn new(name: &str, capacity: usize) -> Self {
         let array_builder = MutablePrimitiveArray::<T::Native>::with_capacity(capacity)
-            .to(T::get_dtype().to_arrow());
+            .to(T::get_dtype().to_arrow(true));
 
         PrimitiveChunkedBuilder {
             array_builder,

crates/polars-core/src/chunked_array/cast.rs

@@ -25,7 +25,7 @@ pub(crate) fn cast_chunks(
         }
     };
 
-    let arrow_dtype = dtype.to_arrow();
+    let arrow_dtype = dtype.to_arrow(true);
     chunks
         .iter()
         .map(|arr| arrow::compute::cast::cast(arr.as_ref(), &arrow_dtype, options))

crates/polars-core/src/chunked_array/collect.rs

@@ -30,7 +30,7 @@ pub(crate) fn prepare_collect_dtype(dtype: &DataType) -> ArrowDataType {
                 registry::get_object_physical_type()
             },
         },
-        dt => dt.to_arrow(),
+        dt => dt.to_arrow(true),
     }
 }
 

crates/polars-core/src/chunked_array/from.rs

@@ -209,7 +209,7 @@ where
         #[cfg(debug_assertions)]
         {
             if !chunks.is_empty() && dtype.is_primitive() {
-                assert_eq!(chunks[0].data_type(), &dtype.to_physical().to_arrow())
+                assert_eq!(chunks[0].data_type(), &dtype.to_physical().to_arrow(true))
             }
         }
         let field = Arc::new(Field::new(name, dtype));

crates/polars-core/src/chunked_array/list/mod.rs

@@ -41,7 +41,7 @@ impl ListChunked {
     /// Get the inner values as [`Series`], ignoring the list offsets.
     pub fn get_inner(&self) -> Series {
         let ca = self.rechunk();
-        let inner_dtype = self.inner_dtype().to_arrow();
+        let inner_dtype = self.inner_dtype().to_arrow(true);
         let arr = ca.downcast_iter().next().unwrap();
         unsafe {
             Series::_try_from_arrow_unchecked(
@@ -60,7 +60,7 @@ impl ListChunked {
     ) -> PolarsResult<ListChunked> {
         // generated Series will have wrong length otherwise.
         let ca = self.rechunk();
-        let inner_dtype = self.inner_dtype().to_arrow();
+        let inner_dtype = self.inner_dtype().to_arrow(true);
         let arr = ca.downcast_iter().next().unwrap();
 
         let elements = unsafe {

crates/polars-core/src/chunked_array/logical/categorical/merge.rs

@@ -23,7 +23,7 @@ fn slots_to_mut(slots: &Utf8Array<i64>) -> MutableUtf8Array<i64> {
     // all offsets are valid and the u8 data is valid utf8
     unsafe {
         MutableUtf8Array::new_unchecked(
-            DataType::String.to_arrow(),
+            ArrowDataType::LargeUtf8,
             offset_buf,
             values_buf,
             validity_buf,

crates/polars-core/src/chunked_array/logical/decimal.rs

@@ -18,7 +18,7 @@ impl Int128Chunked {
             let (_, values, validity) = default.into_inner();
 
             *arr = PrimitiveArray::new(
-                DataType::Decimal(precision, Some(scale)).to_arrow(),
+                DataType::Decimal(precision, Some(scale)).to_arrow(true),
                 values,
                 validity,
             );

crates/polars-core/src/chunked_array/logical/struct_/mod.rs

@@ -46,12 +46,12 @@ fn fields_to_struct_array(fields: &[Series], physical: bool) -> (ArrayRef, Vec<S
             let s = s.rechunk();
             match s.dtype() {
                 #[cfg(feature = "object")]
-                DataType::Object(_, _) => s.to_arrow(0),
+                DataType::Object(_, _) => s.to_arrow(0, true),
                 _ => {
                     if physical {
                         s.chunks()[0].clone()
                     } else {
-                        s.to_arrow(0)
+                        s.to_arrow(0, true)
                     }
                 },
             }
@@ -143,7 +143,7 @@ impl StructChunked {
                 .iter()
                 .map(|s| match s.dtype() {
                     #[cfg(feature = "object")]
-                    DataType::Object(_, _) => s.to_arrow(i),
+                    DataType::Object(_, _) => s.to_arrow(i, true),
                     _ => s.chunks()[i].clone(),
                 })
                 .collect::<Vec<_>>();
@@ -297,7 +297,7 @@ impl StructChunked {
         let values = self
             .fields
             .iter()
-            .map(|s| s.to_arrow(i))
+            .map(|s| s.to_arrow(i, true))
             .collect::<Vec<_>>();
 
         // we determine fields from arrays as there might be object arrays

crates/polars-core/src/chunked_array/mod.rs

@@ -661,7 +661,7 @@ pub(crate) fn to_primitive<T: PolarsNumericType>(
     values: Vec<T::Native>,
     validity: Option<Bitmap>,
 ) -> PrimitiveArray<T::Native> {
-    PrimitiveArray::new(T::get_dtype().to_arrow(), values.into(), validity)
+    PrimitiveArray::new(T::get_dtype().to_arrow(true), values.into(), validity)
 }
 
 pub(crate) fn to_array<T: PolarsNumericType>(

crates/polars-core/src/chunked_array/ops/apply.rs

@@ -68,13 +68,13 @@ where
                 let out: U::Array = arr
                     .values_iter()
                     .map(&mut op)
-                    .collect_arr_with_dtype(dtype.to_arrow());
+                    .collect_arr_with_dtype(dtype.to_arrow(true));
                 out.with_validity_typed(arr.validity().cloned())
             } else {
                 let out: U::Array = arr
                     .iter()
                     .map(|opt| opt.map(&mut op))
-                    .collect_arr_with_dtype(dtype.to_arrow());
+                    .collect_arr_with_dtype(dtype.to_arrow(true));
                 out.with_validity_typed(arr.validity().cloned())
             }
         });
@@ -159,7 +159,7 @@ where
         drop(arr);
 
         let compute_immutable = |arr: &PrimitiveArray<S::Native>| {
-            arrow::compute::arity::unary(arr, f, S::get_dtype().to_arrow())
+            arrow::compute::arity::unary(arr, f, S::get_dtype().to_arrow(true))
         };
 
         if owned_arr.values().is_sliced() {

crates/polars-core/src/chunked_array/ops/arity.rs

@@ -78,7 +78,7 @@ where
     V::Array: ArrayFromIter<<F as UnaryFnMut<T::Physical<'a>>>::Ret>,
 {
     if ca.null_count() == ca.len() {
-        let arr = V::Array::full_null(ca.len(), V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(ca.len(), V::get_dtype().to_arrow(true));
         return ChunkedArray::with_chunk(ca.name(), arr);
     }
 
@@ -102,7 +102,7 @@ where
     V::Array: ArrayFromIter<K>,
 {
     if ca.null_count() == ca.len() {
-        let arr = V::Array::full_null(ca.len(), V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(ca.len(), V::get_dtype().to_arrow(true));
         return Ok(ChunkedArray::with_chunk(ca.name(), arr));
     }
 
@@ -280,7 +280,7 @@ where
 {
     if lhs.null_count() == lhs.len() || rhs.null_count() == rhs.len() {
         let len = lhs.len().min(rhs.len());
-        let arr = V::Array::full_null(len, V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(len, V::get_dtype().to_arrow(true));
 
         return ChunkedArray::with_chunk(lhs.name(), arr);
     }
@@ -319,7 +319,7 @@ where
 {
     if lhs.null_count() == lhs.len() || rhs.null_count() == rhs.len() {
         let len = lhs.len().min(rhs.len());
-        let arr = V::Array::full_null(len, V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(len, V::get_dtype().to_arrow(true));
 
         return Ok(ChunkedArray::with_chunk(lhs.name(), arr));
     }
@@ -678,7 +678,7 @@ where
         let min = lhs.len().min(rhs.len());
         let max = lhs.len().max(rhs.len());
         let len = if min == 1 { max } else { min };
-        let arr = V::Array::full_null(len, V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(len, V::get_dtype().to_arrow(true));
 
         return ChunkedArray::with_chunk(lhs.name(), arr);
     }
@@ -714,7 +714,7 @@ where
         let min = lhs.len().min(rhs.len());
         let max = lhs.len().max(rhs.len());
         let len = if min == 1 { max } else { min };
-        let arr = V::Array::full_null(len, V::get_dtype().to_arrow());
+        let arr = V::Array::full_null(len, V::get_dtype().to_arrow(true));
 
         return Ok(ChunkedArray::with_chunk(lhs.name(), arr));
     }

crates/polars-core/src/chunked_array/ops/bit_repr.rs

@@ -45,7 +45,7 @@ fn reinterpret_list_chunked<T: PolarsNumericType, U: PolarsNumericType>(
         let pa =
             PrimitiveArray::from_data_default(reinterpreted_buf, inner_arr.validity().cloned());
         LargeListArray::new(
-            DataType::List(Box::new(U::get_dtype())).to_arrow(),
+            DataType::List(Box::new(U::get_dtype())).to_arrow(true),
             array.offsets().clone(),
             pa.to_boxed(),
             array.validity().cloned(),

crates/polars-core/src/chunked_array/ops/explode.rs

@@ -155,7 +155,7 @@ where
             unsafe { unset_bit_raw(validity_slice, i) }
         }
         let arr = PrimitiveArray::new(
-            T::get_dtype().to_arrow(),
+            T::get_dtype().to_arrow(true),
             new_values.into(),
             Some(validity.into()),
         );
@@ -274,7 +274,7 @@ impl ExplodeByOffsets for ListChunked {
             last = o;
         }
         process_range(start, last, &mut builder);
-        let arr = builder.finish(Some(&inner_type.to_arrow())).unwrap();
+        let arr = builder.finish(Some(&inner_type.to_arrow(true))).unwrap();
         unsafe { self.copy_with_chunks(vec![Box::new(arr)], true, true) }.into_series()
     }
 }

crates/polars-core/src/chunked_array/ops/filter.rs

@@ -100,7 +100,7 @@ impl ChunkFilter<ListType> for ListChunked {
                 Some(true) => Ok(self.clone()),
                 _ => Ok(ListChunked::from_chunk_iter(
                     self.name(),
-                    [ListArray::new_empty(self.dtype().to_arrow())],
+                    [ListArray::new_empty(self.dtype().to_arrow(true))],
                 )),
             };
         }
@@ -126,7 +126,7 @@ impl ChunkFilter<FixedSizeListType> for ArrayChunked {
                 Some(true) => Ok(self.clone()),
                 _ => Ok(ArrayChunked::from_chunk_iter(
                     self.name(),
-                    [FixedSizeListArray::new_empty(self.dtype().to_arrow())],
+                    [FixedSizeListArray::new_empty(self.dtype().to_arrow(true))],
                 )),
             };
         }

crates/polars-core/src/chunked_array/ops/full.rs

@@ -22,7 +22,7 @@ where
     T: PolarsNumericType,
 {
     fn full_null(name: &str, length: usize) -> Self {
-        let arr = PrimitiveArray::new_null(T::get_dtype().to_arrow(), length);
+        let arr = PrimitiveArray::new_null(T::get_dtype().to_arrow(true), length);
         ChunkedArray::with_chunk(name, arr)
     }
 }
@@ -39,7 +39,7 @@ impl ChunkFull<bool> for BooleanChunked {
 
 impl ChunkFullNull for BooleanChunked {
     fn full_null(name: &str, length: usize) -> Self {
-        let arr = BooleanArray::new_null(DataType::Boolean.to_arrow(), length);
+        let arr = BooleanArray::new_null(ArrowDataType::Boolean, length);
         ChunkedArray::with_chunk(name, arr)
     }
 }
@@ -59,7 +59,7 @@ impl<'a> ChunkFull<&'a str> for StringChunked {
 
 impl ChunkFullNull for StringChunked {
     fn full_null(name: &str, length: usize) -> Self {
-        let arr = Utf8Array::new_null(DataType::String.to_arrow(), length);
+        let arr = Utf8Array::new_null(DataType::String.to_arrow(true), length);
         ChunkedArray::with_chunk(name, arr)
     }
 }
@@ -79,7 +79,7 @@ impl<'a> ChunkFull<&'a [u8]> for BinaryChunked {
 
 impl ChunkFullNull for BinaryChunked {
     fn full_null(name: &str, length: usize) -> Self {
-        let arr = BinaryArray::new_null(DataType::Binary.to_arrow(), length);
+        let arr = BinaryArray::new_null(DataType::Binary.to_arrow(true), length);
         ChunkedArray::with_chunk(name, arr)
     }
 }
@@ -111,7 +111,7 @@ impl ArrayChunked {
     ) -> ArrayChunked {
         let arr = FixedSizeListArray::new_null(
             ArrowDataType::FixedSizeList(
-                Box::new(ArrowField::new("item", inner_dtype.to_arrow(), true)),
+                Box::new(ArrowField::new("item", inner_dtype.to_arrow(true), true)),
                 width,
             ),
             length,
@@ -150,7 +150,7 @@ impl ListChunked {
         let arr: ListArray<i64> = ListArray::new_null(
             ArrowDataType::LargeList(Box::new(ArrowField::new(
                 "item",
-                inner_dtype.to_physical().to_arrow(),
+                inner_dtype.to_physical().to_arrow(true),
                 true,
             ))),
             length,