feat: add "future" versioning (#17421)

crates/polars-core/src/chunked_array/array/mod.rs

@@ -44,7 +44,9 @@ impl ArrayChunked {
     ) -> PolarsResult<ArrayChunked> {
         // Rechunk or the generated Series will have wrong length.
         let ca = self.rechunk();
-        let field = self.inner_dtype().to_arrow_field("item", true);
+        let field = self
+            .inner_dtype()
+            .to_arrow_field("item", CompatLevel::newest());
 
         let chunks = ca.downcast_iter().map(|arr| {
             let elements = unsafe {
@@ -66,8 +68,10 @@ impl ArrayChunked {
             let out = out.rechunk();
             let values = out.chunks()[0].clone();
 
-            let inner_dtype =
-                FixedSizeListArray::default_datatype(out.dtype().to_arrow(true), ca.width());
+            let inner_dtype = FixedSizeListArray::default_datatype(
+                out.dtype().to_arrow(CompatLevel::newest()),
+                ca.width(),
+            );
             let arr = FixedSizeListArray::new(inner_dtype, values, arr.validity().cloned());
             Ok(arr)
         });

crates/polars-core/src/chunked_array/builder/fixed_size_list.rs

@@ -124,7 +124,7 @@ impl FixedSizeListBuilder for AnonymousOwnedFixedSizeListBuilder {
             .finish(
                 self.inner_dtype
                     .as_ref()
-                    .map(|dt| dt.to_arrow(true))
+                    .map(|dt| dt.to_arrow(CompatLevel::newest()))
                     .as_ref(),
             )
             .unwrap();

crates/polars-core/src/chunked_array/builder/list/anonymous.rs

@@ -89,7 +89,7 @@ impl<'a> AnonymousListBuilder<'a> {
 
             let inner_dtype_physical = inner_dtype
                 .as_ref()
-                .map(|dt| dt.to_physical().to_arrow(true));
+                .map(|dt| dt.to_physical().to_arrow(CompatLevel::newest()));
             let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
 
             let list_dtype_logical = match inner_dtype {
@@ -157,7 +157,7 @@ impl ListBuilderTrait for AnonymousOwnedListBuilder {
         let slf = std::mem::take(self);
         let inner_dtype_physical = inner_dtype
             .as_ref()
-            .map(|dt| dt.to_physical().to_arrow(true));
+            .map(|dt| dt.to_physical().to_arrow(CompatLevel::newest()));
         let arr = slf.builder.finish(inner_dtype_physical.as_ref()).unwrap();
 
         let list_dtype_logical = match inner_dtype {

crates/polars-core/src/chunked_array/builder/list/primitive.rs

@@ -39,7 +39,7 @@ where
     ) -> Self {
         let values = MutablePrimitiveArray::<T::Native>::with_capacity_from(
             values_capacity,
-            values_type.to_arrow(true),
+            values_type.to_arrow(CompatLevel::newest()),
         );
         let builder = LargePrimitiveBuilder::<T::Native>::new_with_capacity(values, capacity);
         let field = Field::new(name, DataType::List(Box::new(logical_type)));

crates/polars-core/src/chunked_array/builder/mod.rs

@@ -66,7 +66,7 @@ where
     T: PolarsNumericType,
 {
     fn from_slice(name: &str, v: &[T::Native]) -> Self {
-        let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow(true));
+        let arr = PrimitiveArray::from_slice(v).to(T::get_dtype().to_arrow(CompatLevel::newest()));
         ChunkedArray::with_chunk(name, arr)
     }
 

crates/polars-core/src/chunked_array/builder/primitive.rs

@@ -41,7 +41,7 @@ where
 {
     pub fn new(name: &str, capacity: usize) -> Self {
         let array_builder = MutablePrimitiveArray::<T::Native>::with_capacity(capacity)
-            .to(T::get_dtype().to_arrow(true));
+            .to(T::get_dtype().to_arrow(CompatLevel::newest()));
 
         PrimitiveChunkedBuilder {
             array_builder,

crates/polars-core/src/chunked_array/cast.rs

@@ -51,7 +51,7 @@ pub(crate) fn cast_chunks(
     let check_nulls = matches!(options, CastOptions::Strict);
     let options = options.into();
 
-    let arrow_dtype = dtype.try_to_arrow(true)?;
+    let arrow_dtype = dtype.try_to_arrow(CompatLevel::newest())?;
     chunks
         .iter()
         .map(|arr| {

crates/polars-core/src/chunked_array/collect.rs

@@ -18,6 +18,7 @@ use crate::chunked_array::ChunkedArray;
 use crate::datatypes::{
     ArrayCollectIterExt, ArrayFromIter, ArrayFromIterDtype, DataType, Field, PolarsDataType,
 };
+use crate::prelude::CompatLevel;
 
 pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
     #[inline]
@@ -26,7 +27,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         T::Array: ArrayFromIterDtype<Self::Item>,
     {
         let field = Arc::new(Field::new(name, dtype.clone()));
-        let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(true));
+        let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
         ChunkedArray::from_chunk_iter_and_field(field, [arr])
     }
 
@@ -36,7 +37,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         T::Array: ArrayFromIterDtype<Self::Item>,
     {
         let field = Arc::clone(&name_dtype_src.field);
-        let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(true));
+        let arr = self.collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
         ChunkedArray::from_chunk_iter_and_field(field, [arr])
     }
 
@@ -47,7 +48,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: TrustedLen,
     {
         let field = Arc::new(Field::new(name, dtype.clone()));
-        let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(true));
+        let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
         ChunkedArray::from_chunk_iter_and_field(field, [arr])
     }
 
@@ -58,7 +59,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: TrustedLen,
     {
         let field = Arc::clone(&name_dtype_src.field);
-        let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(true));
+        let arr = self.collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()));
         ChunkedArray::from_chunk_iter_and_field(field, [arr])
     }
 
@@ -73,7 +74,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: Iterator<Item = Result<U, E>>,
     {
         let field = Arc::new(Field::new(name, dtype.clone()));
-        let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(true))?;
+        let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
         Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
     }
 
@@ -87,7 +88,7 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: Iterator<Item = Result<U, E>>,
     {
         let field = Arc::clone(&name_dtype_src.field);
-        let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(true))?;
+        let arr = self.try_collect_arr_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
         Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
     }
 
@@ -102,7 +103,8 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: Iterator<Item = Result<U, E>> + TrustedLen,
     {
         let field = Arc::new(Field::new(name, dtype.clone()));
-        let arr = self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(true))?;
+        let arr =
+            self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
         Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
     }
 
@@ -116,7 +118,8 @@ pub trait ChunkedCollectIterExt<T: PolarsDataType>: Iterator + Sized {
         Self: Iterator<Item = Result<U, E>> + TrustedLen,
     {
         let field = Arc::clone(&name_dtype_src.field);
-        let arr = self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(true))?;
+        let arr =
+            self.try_collect_arr_trusted_with_dtype(field.dtype.to_arrow(CompatLevel::newest()))?;
         Ok(ChunkedArray::from_chunk_iter_and_field(field, [arr]))
     }
 }

crates/polars-core/src/chunked_array/from.rs

@@ -217,7 +217,10 @@ where
         #[cfg(debug_assertions)]
         {
             if !chunks.is_empty() && !chunks[0].is_empty() && dtype.is_primitive() {
-                assert_eq!(chunks[0].data_type(), &dtype.to_arrow(true))
+                assert_eq!(
+                    chunks[0].data_type(),
+                    &dtype.to_arrow(CompatLevel::newest())
+                )
             }
         }
         let field = Arc::new(Field::new(name, dtype));
@@ -234,7 +237,10 @@ where
     }
 
     pub fn full_null_like(ca: &Self, length: usize) -> Self {
-        let chunks = std::iter::once(T::Array::full_null(length, ca.dtype().to_arrow(true)));
+        let chunks = std::iter::once(T::Array::full_null(
+            length,
+            ca.dtype().to_arrow(CompatLevel::newest()),
+        ));
         Self::from_chunk_iter_like(ca, chunks)
     }
 }

crates/polars-core/src/chunked_array/logical/categorical/from.rs

@@ -3,25 +3,25 @@ use arrow::datatypes::IntegerType;
 
 use super::*;
 
-fn convert_values(arr: &Utf8ViewArray, pl_flavor: bool) -> ArrayRef {
-    if pl_flavor {
+fn convert_values(arr: &Utf8ViewArray, compat_level: CompatLevel) -> ArrayRef {
+    if compat_level.0 >= 1 {
         arr.clone().boxed()
     } else {
         utf8view_to_utf8::<i64>(arr).boxed()
     }
 }
 
 impl CategoricalChunked {
-    pub fn to_arrow(&self, pl_flavor: bool, as_i64: bool) -> ArrayRef {
+    pub fn to_arrow(&self, compat_level: CompatLevel, as_i64: bool) -> ArrayRef {
         if as_i64 {
-            self.to_i64(pl_flavor).boxed()
+            self.to_i64(compat_level).boxed()
         } else {
-            self.to_u32(pl_flavor).boxed()
+            self.to_u32(compat_level).boxed()
         }
     }
 
-    fn to_u32(&self, pl_flavor: bool) -> DictionaryArray<u32> {
-        let values_dtype = if pl_flavor {
+    fn to_u32(&self, compat_level: CompatLevel) -> DictionaryArray<u32> {
+        let values_dtype = if compat_level.0 >= 1 {
             ArrowDataType::Utf8View
         } else {
             ArrowDataType::LargeUtf8
@@ -32,7 +32,7 @@ impl CategoricalChunked {
         let dtype = ArrowDataType::Dictionary(IntegerType::UInt32, Box::new(values_dtype), false);
         match map {
             RevMapping::Local(arr, _) => {
-                let values = convert_values(arr, pl_flavor);
+                let values = convert_values(arr, compat_level);
 
                 // SAFETY:
                 // the keys are in bounds
@@ -44,7 +44,7 @@ impl CategoricalChunked {
                     .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap()));
                 let keys = PrimitiveArray::from_trusted_len_iter(iter);
 
-                let values = convert_values(values, pl_flavor);
+                let values = convert_values(values, compat_level);
 
                 // SAFETY:
                 // the keys are in bounds
@@ -53,8 +53,8 @@ impl CategoricalChunked {
         }
     }
 
-    fn to_i64(&self, pl_flavor: bool) -> DictionaryArray<i64> {
-        let values_dtype = if pl_flavor {
+    fn to_i64(&self, compat_level: CompatLevel) -> DictionaryArray<i64> {
+        let values_dtype = if compat_level.0 >= 1 {
             ArrowDataType::Utf8View
         } else {
             ArrowDataType::LargeUtf8
@@ -65,7 +65,7 @@ impl CategoricalChunked {
         let dtype = ArrowDataType::Dictionary(IntegerType::Int64, Box::new(values_dtype), false);
         match map {
             RevMapping::Local(arr, _) => {
-                let values = convert_values(arr, pl_flavor);
+                let values = convert_values(arr, compat_level);
 
                 // SAFETY:
                 // the keys are in bounds
@@ -89,7 +89,7 @@ impl CategoricalChunked {
                     .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap() as i64));
                 let keys = PrimitiveArray::from_trusted_len_iter(iter);
 
-                let values = convert_values(values, pl_flavor);
+                let values = convert_values(values, compat_level);
 
                 // SAFETY:
                 // the keys are in bounds

crates/polars-core/src/chunked_array/logical/categorical/mod.rs

@@ -464,7 +464,7 @@ mod test {
         let ca = ca.cast(&DataType::Categorical(None, Default::default()))?;
         let ca = ca.categorical().unwrap();
 
-        let arr = ca.to_arrow(true, false);
+        let arr = ca.to_arrow(CompatLevel::newest(), false);
         let s = Series::try_from(("foo", arr))?;
         assert!(matches!(s.dtype(), &DataType::Categorical(_, _)));
         assert_eq!(s.null_count(), 1);

crates/polars-core/src/chunked_array/logical/decimal.rs

@@ -20,7 +20,7 @@ impl Int128Chunked {
             let (_, values, validity) = default.into_inner();
 
             *arr = PrimitiveArray::new(
-                DataType::Decimal(precision, Some(scale)).to_arrow(true),
+                DataType::Decimal(precision, Some(scale)).to_arrow(CompatLevel::newest()),
                 values,
                 validity,
             );

crates/polars-core/src/chunked_array/logical/struct_/mod.rs

@@ -48,12 +48,12 @@ fn fields_to_struct_array(fields: &[Series], physical: bool) -> (ArrayRef, Vec<S
             let s = s.rechunk();
             match s.dtype() {
                 #[cfg(feature = "object")]
-                DataType::Object(_, _) => s.to_arrow(0, true),
+                DataType::Object(_, _) => s.to_arrow(0, CompatLevel::newest()),
                 _ => {
                     if physical {
                         s.chunks()[0].clone()
                     } else {
-                        s.to_arrow(0, true)
+                        s.to_arrow(0, CompatLevel::newest())
                     }
                 },
             }
@@ -145,7 +145,7 @@ impl StructChunked {
                 .iter()
                 .map(|s| match s.dtype() {
                     #[cfg(feature = "object")]
-                    DataType::Object(_, _) => s.to_arrow(i, true),
+                    DataType::Object(_, _) => s.to_arrow(i, CompatLevel::newest()),
                     _ => s.chunks()[i].clone(),
                 })
                 .collect::<Vec<_>>();
@@ -295,11 +295,11 @@ impl StructChunked {
         self.into()
     }
 
-    pub(crate) fn to_arrow(&self, i: usize, pl_flavor: bool) -> ArrayRef {
+    pub(crate) fn to_arrow(&self, i: usize, compat_level: CompatLevel) -> ArrayRef {
         let values = self
             .fields
             .iter()
-            .map(|s| s.to_arrow(i, pl_flavor))
+            .map(|s| s.to_arrow(i, compat_level))
             .collect::<Vec<_>>();
 
         // we determine fields from arrays as there might be object arrays

crates/polars-core/src/chunked_array/mod.rs

@@ -933,7 +933,11 @@ pub(crate) fn to_primitive<T: PolarsNumericType>(
     values: Vec<T::Native>,
     validity: Option<Bitmap>,
 ) -> PrimitiveArray<T::Native> {
-    PrimitiveArray::new(T::get_dtype().to_arrow(true), values.into(), validity)
+    PrimitiveArray::new(
+        T::get_dtype().to_arrow(CompatLevel::newest()),
+        values.into(),
+        validity,
+    )
 }
 
 pub(crate) fn to_array<T: PolarsNumericType>(

crates/polars-core/src/chunked_array/ops/apply.rs

@@ -42,13 +42,13 @@ where
                 let out: U::Array = arr
                     .values_iter()
                     .map(&mut op)
-                    .collect_arr_with_dtype(dtype.to_arrow(true));
+                    .collect_arr_with_dtype(dtype.to_arrow(CompatLevel::newest()));
                 out.with_validity_typed(arr.validity().cloned())
             } else {
                 let out: U::Array = arr
                     .iter()
                     .map(|opt| opt.map(&mut op))
-                    .collect_arr_with_dtype(dtype.to_arrow(true));
+                    .collect_arr_with_dtype(dtype.to_arrow(CompatLevel::newest()));
                 out.with_validity_typed(arr.validity().cloned())
             }
         });
@@ -133,7 +133,7 @@ where
         drop(arr);
 
         let compute_immutable = |arr: &PrimitiveArray<S::Native>| {
-            arrow::compute::arity::unary(arr, f, S::get_dtype().to_arrow(true))
+            arrow::compute::arity::unary(arr, f, S::get_dtype().to_arrow(CompatLevel::newest()))
         };
 
         if owned_arr.values().is_sliced() {