diff --git a/Cargo.lock b/Cargo.lock index 2a9759c0d46f..d551d67ff069 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2686,16 +2686,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "parquet-format-safe" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" -dependencies = [ - "async-trait", - "futures", -] - [[package]] name = "parse-zoneinfo" version = "0.3.1" @@ -3187,10 +3177,10 @@ dependencies = [ "lz4", "lz4_flex", "num-traits", - "parquet-format-safe", "polars-arrow", "polars-compute", "polars-error", + "polars-parquet-format", "polars-utils", "rand", "serde", @@ -3201,6 +3191,15 @@ dependencies = [ "zstd", ] +[[package]] +name = "polars-parquet-format" +version = "2.10.0" +source = "git+https://github.com/pola-rs/parquet-format#b96e00d2b054739ee02da06987bcd7f44b82a4ef" +dependencies = [ + "async-trait", + "futures", +] + [[package]] name = "polars-pipe" version = "0.43.1" diff --git a/crates/polars-parquet/Cargo.toml b/crates/polars-parquet/Cargo.toml index 881c9a477398..544e52388e58 100644 --- a/crates/polars-parquet/Cargo.toml +++ b/crates/polars-parquet/Cargo.toml @@ -24,10 +24,10 @@ hashbrown = { workspace = true } num-traits = { workspace = true } polars-compute = { workspace = true, features = ["approx_unique"] } polars-error = { workspace = true } +polars-parquet-format = { git = "https://github.com/pola-rs/parquet-format" } polars-utils = { workspace = true, features = ["mmap"] } simdutf8 = { workspace = true } -parquet-format-safe = "0.2" streaming-decompression = "0.1" async-stream = { version = "0.3.3", optional = true } @@ -61,6 +61,6 @@ gzip_zlib_ng = ["flate2/zlib-ng"] lz4 = ["dep:lz4"] lz4_flex = ["dep:lz4_flex"] -async = ["async-stream", "futures", "parquet-format-safe/async"] +async = ["async-stream", "futures", "polars-parquet-format/async"] bloom_filter = ["xxhash-rust"] serde_types = ["serde"] diff --git a/crates/polars-parquet/src/parquet/bloom_filter/read.rs b/crates/polars-parquet/src/parquet/bloom_filter/read.rs index deda00b36272..fe4ee718cb7e 100644 --- a/crates/polars-parquet/src/parquet/bloom_filter/read.rs +++ b/crates/polars-parquet/src/parquet/bloom_filter/read.rs @@ -1,7 +1,7 @@ use std::io::{Read, Seek, SeekFrom}; -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::{ +use polars_parquet_format::thrift::protocol::TCompactInputProtocol; +use polars_parquet_format::{ BloomFilterAlgorithm, BloomFilterCompression, BloomFilterHeader, SplitBlockAlgorithm, Uncompressed, }; diff --git a/crates/polars-parquet/src/parquet/error.rs b/crates/polars-parquet/src/parquet/error.rs index fdd7f8fcadfd..3d00cf3c647f 100644 --- a/crates/polars-parquet/src/parquet/error.rs +++ b/crates/polars-parquet/src/parquet/error.rs @@ -94,8 +94,8 @@ impl From for ParquetError { } } -impl From for ParquetError { - fn from(e: parquet_format_safe::thrift::Error) -> ParquetError { +impl From for ParquetError { + fn from(e: polars_parquet_format::thrift::Error) -> ParquetError { ParquetError::OutOfSpec(format!("Invalid thrift: {}", e)) } } diff --git a/crates/polars-parquet/src/parquet/metadata/column_chunk_metadata.rs b/crates/polars-parquet/src/parquet/metadata/column_chunk_metadata.rs index 30a606d6108a..dba897a8eeea 100644 --- a/crates/polars-parquet/src/parquet/metadata/column_chunk_metadata.rs +++ b/crates/polars-parquet/src/parquet/metadata/column_chunk_metadata.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::{ColumnChunk, ColumnMetaData, Encoding}; +use polars_parquet_format::{ColumnChunk, ColumnMetaData, Encoding}; use super::column_descriptor::ColumnDescriptor; use crate::parquet::compression::Compression; @@ -10,7 +10,7 @@ use crate::parquet::statistics::Statistics; mod serde_types { pub use std::io::Cursor; - pub use parquet_format_safe::thrift::protocol::{ + pub use polars_parquet_format::thrift::protocol::{ TCompactInputProtocol, TCompactOutputProtocol, }; pub use serde::de::Error as DeserializeError; diff --git a/crates/polars-parquet/src/parquet/metadata/file_metadata.rs b/crates/polars-parquet/src/parquet/metadata/file_metadata.rs index 2705c2a7b70d..ed14a1e130d6 100644 --- a/crates/polars-parquet/src/parquet/metadata/file_metadata.rs +++ b/crates/polars-parquet/src/parquet/metadata/file_metadata.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::ColumnOrder as TColumnOrder; +use polars_parquet_format::ColumnOrder as TColumnOrder; use super::column_order::ColumnOrder; use super::schema_descriptor::SchemaDescriptor; @@ -8,7 +8,7 @@ use crate::parquet::metadata::get_sort_order; pub use crate::parquet::thrift_format::KeyValue; /// Metadata for a Parquet file. -// This is almost equal to [`parquet_format_safe::FileMetaData`] but contains the descriptors, +// This is almost equal to [`polars_parquet_format::FileMetaData`] but contains the descriptors, // which are crucial to deserialize pages. #[derive(Debug, Clone)] pub struct FileMetadata { @@ -65,7 +65,7 @@ impl FileMetadata { /// Deserializes [`crate::parquet::thrift_format::FileMetadata`] into this struct pub fn try_from_thrift( - metadata: parquet_format_safe::FileMetaData, + metadata: polars_parquet_format::FileMetaData, ) -> Result { let schema_descr = SchemaDescriptor::try_from_thrift(&metadata.schema)?; diff --git a/crates/polars-parquet/src/parquet/metadata/row_metadata.rs b/crates/polars-parquet/src/parquet/metadata/row_metadata.rs index bf27bffb66ef..b02983a760ed 100644 --- a/crates/polars-parquet/src/parquet/metadata/row_metadata.rs +++ b/crates/polars-parquet/src/parquet/metadata/row_metadata.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use hashbrown::hash_map::RawEntryMut; -use parquet_format_safe::{RowGroup, SortingColumn}; +use polars_parquet_format::{RowGroup, SortingColumn}; use polars_utils::aliases::{InitHashMaps, PlHashMap}; use polars_utils::idx_vec::UnitVec; use polars_utils::pl_str::PlSmallStr; diff --git a/crates/polars-parquet/src/parquet/metadata/schema_descriptor.rs b/crates/polars-parquet/src/parquet/metadata/schema_descriptor.rs index 7c29f983ee1d..c40fcdd1309b 100644 --- a/crates/polars-parquet/src/parquet/metadata/schema_descriptor.rs +++ b/crates/polars-parquet/src/parquet/metadata/schema_descriptor.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::SchemaElement; +use polars_parquet_format::SchemaElement; use polars_utils::pl_str::PlSmallStr; #[cfg(feature = "serde_types")] use serde::{Deserialize, Serialize}; diff --git a/crates/polars-parquet/src/parquet/mod.rs b/crates/polars-parquet/src/parquet/mod.rs index ea6b5b2c8357..1926e641fd04 100644 --- a/crates/polars-parquet/src/parquet/mod.rs +++ b/crates/polars-parquet/src/parquet/mod.rs @@ -15,7 +15,7 @@ pub mod write; use std::ops::Deref; -use parquet_format_safe as thrift_format; +use polars_parquet_format as thrift_format; use polars_utils::mmap::MemSlice; pub use streaming_decompression::{fallible_streaming_iterator, FallibleStreamingIterator}; diff --git a/crates/polars-parquet/src/parquet/read/compression.rs b/crates/polars-parquet/src/parquet/read/compression.rs index a79989c39e26..1bd457474e06 100644 --- a/crates/polars-parquet/src/parquet/read/compression.rs +++ b/crates/polars-parquet/src/parquet/read/compression.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::DataPageHeaderV2; +use polars_parquet_format::DataPageHeaderV2; use super::PageReader; use crate::parquet::compression::{self, Compression}; diff --git a/crates/polars-parquet/src/parquet/read/metadata.rs b/crates/polars-parquet/src/parquet/read/metadata.rs index e14a2a60e997..a260fe71ff06 100644 --- a/crates/polars-parquet/src/parquet/read/metadata.rs +++ b/crates/polars-parquet/src/parquet/read/metadata.rs @@ -1,8 +1,8 @@ use std::cmp::min; use std::io::{Read, Seek, SeekFrom}; -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use parquet_format_safe::FileMetaData as TFileMetadata; +use polars_parquet_format::thrift::protocol::TCompactInputProtocol; +use polars_parquet_format::FileMetaData as TFileMetadata; use super::super::metadata::FileMetadata; use super::super::{DEFAULT_FOOTER_READ_SIZE, FOOTER_SIZE, HEADER_SIZE, PARQUET_MAGIC}; diff --git a/crates/polars-parquet/src/parquet/read/page/reader.rs b/crates/polars-parquet/src/parquet/read/page/reader.rs index ad453a0ff50a..811557aa6ccb 100644 --- a/crates/polars-parquet/src/parquet/read/page/reader.rs +++ b/crates/polars-parquet/src/parquet/read/page/reader.rs @@ -1,7 +1,7 @@ use std::io::Seek; use std::sync::OnceLock; -use parquet_format_safe::thrift::protocol::TCompactInputProtocol; +use polars_parquet_format::thrift::protocol::TCompactInputProtocol; use polars_utils::mmap::{MemReader, MemSlice}; use super::PageIterator; diff --git a/crates/polars-parquet/src/parquet/read/page/stream.rs b/crates/polars-parquet/src/parquet/read/page/stream.rs index fbd36b3ccfe1..7145689493fc 100644 --- a/crates/polars-parquet/src/parquet/read/page/stream.rs +++ b/crates/polars-parquet/src/parquet/read/page/stream.rs @@ -2,7 +2,7 @@ use std::io::SeekFrom; use async_stream::try_stream; use futures::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, Stream}; -use parquet_format_safe::thrift::protocol::TCompactInputStreamProtocol; +use polars_parquet_format::thrift::protocol::TCompactInputStreamProtocol; use polars_utils::mmap::MemSlice; use super::reader::{finish_page, PageMetaData}; diff --git a/crates/polars-parquet/src/parquet/schema/io_message/from_message.rs b/crates/polars-parquet/src/parquet/schema/io_message/from_message.rs index d4f2c692e95d..ae98fd694e64 100644 --- a/crates/polars-parquet/src/parquet/schema/io_message/from_message.rs +++ b/crates/polars-parquet/src/parquet/schema/io_message/from_message.rs @@ -42,7 +42,7 @@ //! println!("{:?}", schema); //! ``` -use parquet_format_safe::Type; +use polars_parquet_format::Type; use polars_utils::pl_str::PlSmallStr; use types::PrimitiveLogicalType; diff --git a/crates/polars-parquet/src/parquet/schema/io_thrift/from_thrift.rs b/crates/polars-parquet/src/parquet/schema/io_thrift/from_thrift.rs index b0bbe20999bc..7a874fb59e46 100644 --- a/crates/polars-parquet/src/parquet/schema/io_thrift/from_thrift.rs +++ b/crates/polars-parquet/src/parquet/schema/io_thrift/from_thrift.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::SchemaElement; +use polars_parquet_format::SchemaElement; use polars_utils::pl_str::PlSmallStr; use super::super::types::ParquetType; diff --git a/crates/polars-parquet/src/parquet/schema/io_thrift/to_thrift.rs b/crates/polars-parquet/src/parquet/schema/io_thrift/to_thrift.rs index 3aef1fe792fa..db372b733593 100644 --- a/crates/polars-parquet/src/parquet/schema/io_thrift/to_thrift.rs +++ b/crates/polars-parquet/src/parquet/schema/io_thrift/to_thrift.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::{ConvertedType, SchemaElement}; +use polars_parquet_format::{ConvertedType, SchemaElement}; use super::super::types::ParquetType; use crate::parquet::schema::types::PrimitiveType; diff --git a/crates/polars-parquet/src/parquet/schema/types/converted_type.rs b/crates/polars-parquet/src/parquet/schema/types/converted_type.rs index 8432167fcd3b..91b4b5ac78f9 100644 --- a/crates/polars-parquet/src/parquet/schema/types/converted_type.rs +++ b/crates/polars-parquet/src/parquet/schema/types/converted_type.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::ConvertedType; +use polars_parquet_format::ConvertedType; #[cfg(feature = "serde_types")] use serde::{Deserialize, Serialize}; diff --git a/crates/polars-parquet/src/parquet/schema/types/physical_type.rs b/crates/polars-parquet/src/parquet/schema/types/physical_type.rs index 01595134c6b3..ed7242adac71 100644 --- a/crates/polars-parquet/src/parquet/schema/types/physical_type.rs +++ b/crates/polars-parquet/src/parquet/schema/types/physical_type.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::Type; +use polars_parquet_format::Type; #[cfg(feature = "serde_types")] use serde::{Deserialize, Serialize}; diff --git a/crates/polars-parquet/src/parquet/statistics/binary.rs b/crates/polars-parquet/src/parquet/statistics/binary.rs index e9506c375a71..7f1dabf21ec8 100644 --- a/crates/polars-parquet/src/parquet/statistics/binary.rs +++ b/crates/polars-parquet/src/parquet/statistics/binary.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::Statistics as ParquetStatistics; +use polars_parquet_format::Statistics as ParquetStatistics; use crate::parquet::error::ParquetResult; use crate::parquet::schema::types::PrimitiveType; @@ -32,8 +32,10 @@ impl BinaryStatistics { distinct_count: self.distinct_count, max_value: self.max_value.clone(), min_value: self.min_value.clone(), - min: None, max: None, + min: None, + is_max_value_exact: None, + is_min_value_exact: None, } } } diff --git a/crates/polars-parquet/src/parquet/statistics/boolean.rs b/crates/polars-parquet/src/parquet/statistics/boolean.rs index 607897bdddf0..b171a2efdd51 100644 --- a/crates/polars-parquet/src/parquet/statistics/boolean.rs +++ b/crates/polars-parquet/src/parquet/statistics/boolean.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::Statistics as ParquetStatistics; +use polars_parquet_format::Statistics as ParquetStatistics; use crate::parquet::error::{ParquetError, ParquetResult}; @@ -49,8 +49,10 @@ impl BooleanStatistics { distinct_count: self.distinct_count, max_value: self.max_value.map(|x| vec![x as u8]), min_value: self.min_value.map(|x| vec![x as u8]), - min: None, max: None, + min: None, + is_max_value_exact: None, + is_min_value_exact: None, } } } diff --git a/crates/polars-parquet/src/parquet/statistics/fixed_len_binary.rs b/crates/polars-parquet/src/parquet/statistics/fixed_len_binary.rs index 8de2aef0a508..87642246907d 100644 --- a/crates/polars-parquet/src/parquet/statistics/fixed_len_binary.rs +++ b/crates/polars-parquet/src/parquet/statistics/fixed_len_binary.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::Statistics as ParquetStatistics; +use polars_parquet_format::Statistics as ParquetStatistics; use crate::parquet::error::{ParquetError, ParquetResult}; use crate::parquet::schema::types::PrimitiveType; @@ -54,8 +54,10 @@ impl FixedLenStatistics { distinct_count: self.distinct_count, max_value: self.max_value.clone(), min_value: self.min_value.clone(), - min: None, max: None, + min: None, + is_max_value_exact: None, + is_min_value_exact: None, } } } diff --git a/crates/polars-parquet/src/parquet/statistics/mod.rs b/crates/polars-parquet/src/parquet/statistics/mod.rs index 7501df2a3886..b72e2a7c94b1 100644 --- a/crates/polars-parquet/src/parquet/statistics/mod.rs +++ b/crates/polars-parquet/src/parquet/statistics/mod.rs @@ -11,6 +11,7 @@ pub use primitive::PrimitiveStatistics; use crate::parquet::error::ParquetResult; use crate::parquet::schema::types::{PhysicalType, PrimitiveType}; pub use crate::parquet::thrift_format::Statistics as ParquetStatistics; +use crate::read::ParquetError; #[derive(Debug, PartialEq)] pub enum Statistics { @@ -50,6 +51,17 @@ impl Statistics { statistics: &ParquetStatistics, primitive_type: PrimitiveType, ) -> ParquetResult { + if statistics.is_min_value_exact.is_some() { + return Err(ParquetError::not_supported( + "is_min_value_exact in statistics", + )); + } + if statistics.is_max_value_exact.is_some() { + return Err(ParquetError::not_supported( + "is_max_value_exact in statistics", + )); + } + use {PhysicalType as T, PrimitiveStatistics as PrimStat}; Ok(match primitive_type.physical_type { T::ByteArray => BinaryStatistics::deserialize(statistics, primitive_type)?.into(), diff --git a/crates/polars-parquet/src/parquet/statistics/primitive.rs b/crates/polars-parquet/src/parquet/statistics/primitive.rs index ed5ae71515b0..c4cd417b635c 100644 --- a/crates/polars-parquet/src/parquet/statistics/primitive.rs +++ b/crates/polars-parquet/src/parquet/statistics/primitive.rs @@ -1,4 +1,4 @@ -use parquet_format_safe::Statistics as ParquetStatistics; +use polars_parquet_format::Statistics as ParquetStatistics; use crate::parquet::error::{ParquetError, ParquetResult}; use crate::parquet::schema::types::PrimitiveType; @@ -50,8 +50,10 @@ impl PrimitiveStatistics { distinct_count: self.distinct_count, max_value: self.max_value.map(|x| x.to_le_bytes().as_ref().to_vec()), min_value: self.min_value.map(|x| x.to_le_bytes().as_ref().to_vec()), - min: None, max: None, + min: None, + is_max_value_exact: None, + is_min_value_exact: None, } } } diff --git a/crates/polars-parquet/src/parquet/write/column_chunk.rs b/crates/polars-parquet/src/parquet/write/column_chunk.rs index 6ae51a191dc5..8728f289eb65 100644 --- a/crates/polars-parquet/src/parquet/write/column_chunk.rs +++ b/crates/polars-parquet/src/parquet/write/column_chunk.rs @@ -2,10 +2,10 @@ use std::io::Write; #[cfg(feature = "async")] use futures::AsyncWrite; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; +use polars_parquet_format::thrift::protocol::TCompactOutputProtocol; #[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::{ColumnChunk, ColumnMetaData, Type}; +use polars_parquet_format::thrift::protocol::TCompactOutputStreamProtocol; +use polars_parquet_format::{ColumnChunk, ColumnMetaData, Type}; use polars_utils::aliases::PlHashSet; #[cfg(feature = "async")] @@ -195,6 +195,8 @@ fn build_column_chunk( statistics, encoding_stats: None, bloom_filter_offset: None, + bloom_filter_length: None, + size_statistics: None, }; Ok(ColumnChunk { diff --git a/crates/polars-parquet/src/parquet/write/file.rs b/crates/polars-parquet/src/parquet/write/file.rs index 8dd3212bb76a..d46f85dd3138 100644 --- a/crates/polars-parquet/src/parquet/write/file.rs +++ b/crates/polars-parquet/src/parquet/write/file.rs @@ -1,7 +1,7 @@ use std::io::Write; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; -use parquet_format_safe::RowGroup; +use polars_parquet_format::thrift::protocol::TCompactOutputProtocol; +use polars_parquet_format::RowGroup; use super::indexes::{write_column_index, write_offset_index}; use super::page::PageWriteSpec; @@ -39,7 +39,7 @@ pub(super) fn end_file( Ok(metadata_len as u64 + FOOTER_SIZE) } -fn create_column_orders(schema_desc: &SchemaDescriptor) -> Vec { +fn create_column_orders(schema_desc: &SchemaDescriptor) -> Vec { // We only include ColumnOrder for leaf nodes. // Currently only supported ColumnOrder is TypeDefinedOrder so we set this // for all leaf nodes. @@ -47,7 +47,9 @@ fn create_column_orders(schema_desc: &SchemaDescriptor) -> Vec ParquetResult ParquetResult>>()?; - Ok(OffsetIndex { page_locations }) + Ok(OffsetIndex { + page_locations, + unencoded_byte_array_data_bytes: None, + }) } diff --git a/crates/polars-parquet/src/parquet/write/indexes/write.rs b/crates/polars-parquet/src/parquet/write/indexes/write.rs index 7c82b1dcc9ae..73325654e518 100644 --- a/crates/polars-parquet/src/parquet/write/indexes/write.rs +++ b/crates/polars-parquet/src/parquet/write/indexes/write.rs @@ -2,9 +2,9 @@ use std::io::Write; #[cfg(feature = "async")] use futures::AsyncWrite; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; +use polars_parquet_format::thrift::protocol::TCompactOutputProtocol; #[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; +use polars_parquet_format::thrift::protocol::TCompactOutputStreamProtocol; use super::serialize::{serialize_column_index, serialize_offset_index}; use crate::parquet::error::ParquetResult; diff --git a/crates/polars-parquet/src/parquet/write/page.rs b/crates/polars-parquet/src/parquet/write/page.rs index f9e527d5a9db..8fb65c3daf12 100644 --- a/crates/polars-parquet/src/parquet/write/page.rs +++ b/crates/polars-parquet/src/parquet/write/page.rs @@ -2,10 +2,10 @@ use std::io::Write; #[cfg(feature = "async")] use futures::{AsyncWrite, AsyncWriteExt}; -use parquet_format_safe::thrift::protocol::TCompactOutputProtocol; +use polars_parquet_format::thrift::protocol::TCompactOutputProtocol; #[cfg(feature = "async")] -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::{DictionaryPageHeader, Encoding, PageType}; +use polars_parquet_format::thrift::protocol::TCompactOutputStreamProtocol; +use polars_parquet_format::{DictionaryPageHeader, Encoding, PageType}; use crate::parquet::compression::Compression; use crate::parquet::error::{ParquetError, ParquetResult}; diff --git a/crates/polars-parquet/src/parquet/write/row_group.rs b/crates/polars-parquet/src/parquet/write/row_group.rs index 43404dc32a89..dfca3d27f948 100644 --- a/crates/polars-parquet/src/parquet/write/row_group.rs +++ b/crates/polars-parquet/src/parquet/write/row_group.rs @@ -2,7 +2,7 @@ use std::io::Write; #[cfg(feature = "async")] use futures::AsyncWrite; -use parquet_format_safe::{ColumnChunk, RowGroup}; +use polars_parquet_format::{ColumnChunk, RowGroup}; use super::column_chunk::write_column_chunk; #[cfg(feature = "async")] diff --git a/crates/polars-parquet/src/parquet/write/stream.rs b/crates/polars-parquet/src/parquet/write/stream.rs index eca712db65dc..05c50e6e3a2c 100644 --- a/crates/polars-parquet/src/parquet/write/stream.rs +++ b/crates/polars-parquet/src/parquet/write/stream.rs @@ -1,8 +1,8 @@ use std::io::Write; use futures::{AsyncWrite, AsyncWriteExt}; -use parquet_format_safe::thrift::protocol::TCompactOutputStreamProtocol; -use parquet_format_safe::RowGroup; +use polars_parquet_format::thrift::protocol::TCompactOutputStreamProtocol; +use polars_parquet_format::RowGroup; use super::row_group::write_row_group_async; use super::{RowGroupIterColumns, WriteOptions}; @@ -20,7 +20,7 @@ async fn start_file(writer: &mut W) -> ParquetResult async fn end_file( mut writer: &mut W, - metadata: parquet_format_safe::FileMetaData, + metadata: polars_parquet_format::FileMetaData, ) -> ParquetResult { // Write file metadata let mut protocol = TCompactOutputStreamProtocol::new(&mut writer); @@ -169,7 +169,7 @@ impl FileStreamer { } } - let metadata = parquet_format_safe::FileMetaData::new( + let metadata = polars_parquet_format::FileMetaData::new( self.options.version.into(), self.schema.clone().into_thrift(), num_rows,