refactor: Move from parquet-format-safe to polars-parquet-format

crates/polars-parquet/Cargo.toml

@@ -24,10 +24,10 @@ hashbrown = { workspace = true }
 num-traits = { workspace = true }
 polars-compute = { workspace = true, features = ["approx_unique"] }
 polars-error = { workspace = true }
+polars-parquet-format = { git = "https://github.com/pola-rs/parquet-format" }
 polars-utils = { workspace = true, features = ["mmap"] }
 simdutf8 = { workspace = true }
 
-parquet-format-safe = "0.2"
 streaming-decompression = "0.1"
 
 async-stream = { version = "0.3.3", optional = true }
@@ -61,6 +61,6 @@ gzip_zlib_ng = ["flate2/zlib-ng"]
 lz4 = ["dep:lz4"]
 lz4_flex = ["dep:lz4_flex"]
 
-async = ["async-stream", "futures", "parquet-format-safe/async"]
+async = ["async-stream", "futures", "polars-parquet-format/async"]
 bloom_filter = ["xxhash-rust"]
 serde_types = ["serde"]

crates/polars-parquet/src/parquet/bloom_filter/read.rs

@@ -1,7 +1,7 @@
 use std::io::{Read, Seek, SeekFrom};
 
-use parquet_format_safe::thrift::protocol::TCompactInputProtocol;
-use parquet_format_safe::{
+use polars_parquet_format::thrift::protocol::TCompactInputProtocol;
+use polars_parquet_format::{
     BloomFilterAlgorithm, BloomFilterCompression, BloomFilterHeader, SplitBlockAlgorithm,
     Uncompressed,
 };

crates/polars-parquet/src/parquet/error.rs

@@ -94,8 +94,8 @@ impl From<lz4_flex::block::CompressError> for ParquetError {
     }
 }
 
-impl From<parquet_format_safe::thrift::Error> for ParquetError {
-    fn from(e: parquet_format_safe::thrift::Error) -> ParquetError {
+impl From<polars_parquet_format::thrift::Error> for ParquetError {
+    fn from(e: polars_parquet_format::thrift::Error) -> ParquetError {
         ParquetError::OutOfSpec(format!("Invalid thrift: {}", e))
     }
 }

crates/polars-parquet/src/parquet/metadata/column_chunk_metadata.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::{ColumnChunk, ColumnMetaData, Encoding};
+use polars_parquet_format::{ColumnChunk, ColumnMetaData, Encoding};
 
 use super::column_descriptor::ColumnDescriptor;
 use crate::parquet::compression::Compression;
@@ -10,7 +10,7 @@ use crate::parquet::statistics::Statistics;
 mod serde_types {
     pub use std::io::Cursor;
 
-    pub use parquet_format_safe::thrift::protocol::{
+    pub use polars_parquet_format::thrift::protocol::{
         TCompactInputProtocol, TCompactOutputProtocol,
     };
     pub use serde::de::Error as DeserializeError;

crates/polars-parquet/src/parquet/metadata/file_metadata.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::ColumnOrder as TColumnOrder;
+use polars_parquet_format::ColumnOrder as TColumnOrder;
 
 use super::column_order::ColumnOrder;
 use super::schema_descriptor::SchemaDescriptor;
@@ -8,7 +8,7 @@ use crate::parquet::metadata::get_sort_order;
 pub use crate::parquet::thrift_format::KeyValue;
 
 /// Metadata for a Parquet file.
-// This is almost equal to [`parquet_format_safe::FileMetaData`] but contains the descriptors,
+// This is almost equal to [`polars_parquet_format::FileMetaData`] but contains the descriptors,
 // which are crucial to deserialize pages.
 #[derive(Debug, Clone)]
 pub struct FileMetadata {
@@ -65,7 +65,7 @@ impl FileMetadata {
 
     /// Deserializes [`crate::parquet::thrift_format::FileMetadata`] into this struct
     pub fn try_from_thrift(
-        metadata: parquet_format_safe::FileMetaData,
+        metadata: polars_parquet_format::FileMetaData,
     ) -> Result<Self, ParquetError> {
         let schema_descr = SchemaDescriptor::try_from_thrift(&metadata.schema)?;
 

crates/polars-parquet/src/parquet/metadata/row_metadata.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use hashbrown::hash_map::RawEntryMut;
-use parquet_format_safe::{RowGroup, SortingColumn};
+use polars_parquet_format::{RowGroup, SortingColumn};
 use polars_utils::aliases::{InitHashMaps, PlHashMap};
 use polars_utils::idx_vec::UnitVec;
 use polars_utils::pl_str::PlSmallStr;

crates/polars-parquet/src/parquet/metadata/schema_descriptor.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::SchemaElement;
+use polars_parquet_format::SchemaElement;
 use polars_utils::pl_str::PlSmallStr;
 #[cfg(feature = "serde_types")]
 use serde::{Deserialize, Serialize};

crates/polars-parquet/src/parquet/mod.rs

@@ -15,7 +15,7 @@ pub mod write;
 
 use std::ops::Deref;
 
-use parquet_format_safe as thrift_format;
+use polars_parquet_format as thrift_format;
 use polars_utils::mmap::MemSlice;
 pub use streaming_decompression::{fallible_streaming_iterator, FallibleStreamingIterator};
 

crates/polars-parquet/src/parquet/read/compression.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::DataPageHeaderV2;
+use polars_parquet_format::DataPageHeaderV2;
 
 use super::PageReader;
 use crate::parquet::compression::{self, Compression};

crates/polars-parquet/src/parquet/read/metadata.rs

@@ -1,8 +1,8 @@
 use std::cmp::min;
 use std::io::{Read, Seek, SeekFrom};
 
-use parquet_format_safe::thrift::protocol::TCompactInputProtocol;
-use parquet_format_safe::FileMetaData as TFileMetadata;
+use polars_parquet_format::thrift::protocol::TCompactInputProtocol;
+use polars_parquet_format::FileMetaData as TFileMetadata;
 
 use super::super::metadata::FileMetadata;
 use super::super::{DEFAULT_FOOTER_READ_SIZE, FOOTER_SIZE, HEADER_SIZE, PARQUET_MAGIC};

crates/polars-parquet/src/parquet/read/page/reader.rs

@@ -1,7 +1,7 @@
 use std::io::Seek;
 use std::sync::OnceLock;
 
-use parquet_format_safe::thrift::protocol::TCompactInputProtocol;
+use polars_parquet_format::thrift::protocol::TCompactInputProtocol;
 use polars_utils::mmap::{MemReader, MemSlice};
 
 use super::PageIterator;

crates/polars-parquet/src/parquet/read/page/stream.rs

@@ -2,7 +2,7 @@ use std::io::SeekFrom;
 
 use async_stream::try_stream;
 use futures::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, Stream};
-use parquet_format_safe::thrift::protocol::TCompactInputStreamProtocol;
+use polars_parquet_format::thrift::protocol::TCompactInputStreamProtocol;
 use polars_utils::mmap::MemSlice;
 
 use super::reader::{finish_page, PageMetaData};

crates/polars-parquet/src/parquet/schema/io_message/from_message.rs

@@ -42,7 +42,7 @@
 //! println!("{:?}", schema);
 //! ```
 
-use parquet_format_safe::Type;
+use polars_parquet_format::Type;
 use polars_utils::pl_str::PlSmallStr;
 use types::PrimitiveLogicalType;
 

crates/polars-parquet/src/parquet/schema/io_thrift/from_thrift.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::SchemaElement;
+use polars_parquet_format::SchemaElement;
 use polars_utils::pl_str::PlSmallStr;
 
 use super::super::types::ParquetType;

crates/polars-parquet/src/parquet/schema/io_thrift/to_thrift.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::{ConvertedType, SchemaElement};
+use polars_parquet_format::{ConvertedType, SchemaElement};
 
 use super::super::types::ParquetType;
 use crate::parquet::schema::types::PrimitiveType;

crates/polars-parquet/src/parquet/schema/types/converted_type.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::ConvertedType;
+use polars_parquet_format::ConvertedType;
 #[cfg(feature = "serde_types")]
 use serde::{Deserialize, Serialize};
 

crates/polars-parquet/src/parquet/schema/types/physical_type.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::Type;
+use polars_parquet_format::Type;
 #[cfg(feature = "serde_types")]
 use serde::{Deserialize, Serialize};
 

crates/polars-parquet/src/parquet/statistics/binary.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::Statistics as ParquetStatistics;
+use polars_parquet_format::Statistics as ParquetStatistics;
 
 use crate::parquet::error::ParquetResult;
 use crate::parquet::schema::types::PrimitiveType;
@@ -32,8 +32,10 @@ impl BinaryStatistics {
             distinct_count: self.distinct_count,
             max_value: self.max_value.clone(),
             min_value: self.min_value.clone(),
-            min: None,
             max: None,
+            min: None,
+            is_max_value_exact: None,
+            is_min_value_exact: None,
         }
     }
 }

crates/polars-parquet/src/parquet/statistics/boolean.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::Statistics as ParquetStatistics;
+use polars_parquet_format::Statistics as ParquetStatistics;
 
 use crate::parquet::error::{ParquetError, ParquetResult};
 
@@ -49,8 +49,10 @@ impl BooleanStatistics {
             distinct_count: self.distinct_count,
             max_value: self.max_value.map(|x| vec![x as u8]),
             min_value: self.min_value.map(|x| vec![x as u8]),
-            min: None,
             max: None,
+            min: None,
+            is_max_value_exact: None,
+            is_min_value_exact: None,
         }
     }
 }

crates/polars-parquet/src/parquet/statistics/fixed_len_binary.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::Statistics as ParquetStatistics;
+use polars_parquet_format::Statistics as ParquetStatistics;
 
 use crate::parquet::error::{ParquetError, ParquetResult};
 use crate::parquet::schema::types::PrimitiveType;
@@ -54,8 +54,10 @@ impl FixedLenStatistics {
             distinct_count: self.distinct_count,
             max_value: self.max_value.clone(),
             min_value: self.min_value.clone(),
-            min: None,
             max: None,
+            min: None,
+            is_max_value_exact: None,
+            is_min_value_exact: None,
         }
     }
 }

crates/polars-parquet/src/parquet/statistics/mod.rs

@@ -11,6 +11,7 @@ pub use primitive::PrimitiveStatistics;
 use crate::parquet::error::ParquetResult;
 use crate::parquet::schema::types::{PhysicalType, PrimitiveType};
 pub use crate::parquet::thrift_format::Statistics as ParquetStatistics;
+use crate::read::ParquetError;
 
 #[derive(Debug, PartialEq)]
 pub enum Statistics {
@@ -50,6 +51,17 @@ impl Statistics {
         statistics: &ParquetStatistics,
         primitive_type: PrimitiveType,
     ) -> ParquetResult<Self> {
+        if statistics.is_min_value_exact.is_some() {
+            return Err(ParquetError::not_supported(
+                "is_min_value_exact in statistics",
+            ));
+        }
+        if statistics.is_max_value_exact.is_some() {
+            return Err(ParquetError::not_supported(
+                "is_max_value_exact in statistics",
+            ));
+        }
+
         use {PhysicalType as T, PrimitiveStatistics as PrimStat};
         Ok(match primitive_type.physical_type {
             T::ByteArray => BinaryStatistics::deserialize(statistics, primitive_type)?.into(),

crates/polars-parquet/src/parquet/statistics/primitive.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::Statistics as ParquetStatistics;
+use polars_parquet_format::Statistics as ParquetStatistics;
 
 use crate::parquet::error::{ParquetError, ParquetResult};
 use crate::parquet::schema::types::PrimitiveType;
@@ -50,8 +50,10 @@ impl<T: types::NativeType> PrimitiveStatistics<T> {
             distinct_count: self.distinct_count,
             max_value: self.max_value.map(|x| x.to_le_bytes().as_ref().to_vec()),
             min_value: self.min_value.map(|x| x.to_le_bytes().as_ref().to_vec()),
-            min: None,
             max: None,
+            min: None,
+            is_max_value_exact: None,
+            is_min_value_exact: None,
         }
     }
 }

crates/polars-parquet/src/parquet/write/column_chunk.rs

@@ -2,10 +2,10 @@ use std::io::Write;
 
 #[cfg(feature = "async")]
 use futures::AsyncWrite;
-use parquet_format_safe::thrift::protocol::TCompactOutputProtocol;
+use polars_parquet_format::thrift::protocol::TCompactOutputProtocol;
 #[cfg(feature = "async")]
-use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol;
-use parquet_format_safe::{ColumnChunk, ColumnMetaData, Type};
+use polars_parquet_format::thrift::protocol::TCompactOutputStreamProtocol;
+use polars_parquet_format::{ColumnChunk, ColumnMetaData, Type};
 use polars_utils::aliases::PlHashSet;
 
 #[cfg(feature = "async")]
@@ -195,6 +195,8 @@ fn build_column_chunk(
         statistics,
         encoding_stats: None,
         bloom_filter_offset: None,
+        bloom_filter_length: None,
+        size_statistics: None,
     };
 
     Ok(ColumnChunk {

crates/polars-parquet/src/parquet/write/file.rs

@@ -1,7 +1,7 @@
 use std::io::Write;
 
-use parquet_format_safe::thrift::protocol::TCompactOutputProtocol;
-use parquet_format_safe::RowGroup;
+use polars_parquet_format::thrift::protocol::TCompactOutputProtocol;
+use polars_parquet_format::RowGroup;
 
 use super::indexes::{write_column_index, write_offset_index};
 use super::page::PageWriteSpec;
@@ -39,15 +39,17 @@ pub(super) fn end_file<W: Write>(
     Ok(metadata_len as u64 + FOOTER_SIZE)
 }
 
-fn create_column_orders(schema_desc: &SchemaDescriptor) -> Vec<parquet_format_safe::ColumnOrder> {
+fn create_column_orders(schema_desc: &SchemaDescriptor) -> Vec<polars_parquet_format::ColumnOrder> {
     // We only include ColumnOrder for leaf nodes.
     // Currently only supported ColumnOrder is TypeDefinedOrder so we set this
     // for all leaf nodes.
     // Even if the column has an undefined sort order, such as INTERVAL, this
     // is still technically the defined TYPEORDER so it should still be set.
     (0..schema_desc.columns().len())
         .map(|_| {
-            parquet_format_safe::ColumnOrder::TYPEORDER(parquet_format_safe::TypeDefinedOrder {})
+            polars_parquet_format::ColumnOrder::TYPEORDER(
+                polars_parquet_format::TypeDefinedOrder {},
+            )
         })
         .collect()
 }

crates/polars-parquet/src/parquet/write/indexes/serialize.rs

@@ -1,4 +1,4 @@
-use parquet_format_safe::{BoundaryOrder, ColumnIndex, OffsetIndex, PageLocation};
+use polars_parquet_format::{BoundaryOrder, ColumnIndex, OffsetIndex, PageLocation};
 
 use crate::parquet::error::{ParquetError, ParquetResult};
 use crate::parquet::write::page::{is_data_page, PageWriteSpec};
@@ -48,6 +48,8 @@ pub fn serialize_column_index(pages: &[PageWriteSpec]) -> ParquetResult<ColumnIn
         max_values,
         boundary_order: BoundaryOrder::UNORDERED,
         null_counts: Some(null_counts),
+        repetition_level_histograms: None,
+        definition_level_histograms: None,
     })
 }
 
@@ -68,5 +70,8 @@ pub fn serialize_offset_index(pages: &[PageWriteSpec]) -> ParquetResult<OffsetIn
         })
         .collect::<ParquetResult<Vec<_>>>()?;
 
-    Ok(OffsetIndex { page_locations })
+    Ok(OffsetIndex {
+        page_locations,
+        unencoded_byte_array_data_bytes: None,
+    })
 }