parquet Statistics - deprecate has_* APIs and add _opt functions that return Option<T>

parquet/src/arrow/arrow_reader/statistics.rs

@@ -116,7 +116,7 @@ macro_rules! make_stats_iterator {
                 let next = self.iter.next();
                 next.map(|x| {
                     x.and_then(|stats| match stats {
-                        $parquet_statistics_type(s) if stats.has_min_max_set() => Some(s.$func()),
+                        $parquet_statistics_type(s) => s.$func(),
                         _ => None,
                     })
                 })
@@ -131,45 +131,85 @@ macro_rules! make_stats_iterator {
 
 make_stats_iterator!(
     MinBooleanStatsIterator,
-    min,
+    min_opt,
     ParquetStatistics::Boolean,
     bool
 );
 make_stats_iterator!(
     MaxBooleanStatsIterator,
-    max,
+    max_opt,
     ParquetStatistics::Boolean,
     bool
 );
-make_stats_iterator!(MinInt32StatsIterator, min, ParquetStatistics::Int32, i32);
-make_stats_iterator!(MaxInt32StatsIterator, max, ParquetStatistics::Int32, i32);
-make_stats_iterator!(MinInt64StatsIterator, min, ParquetStatistics::Int64, i64);
-make_stats_iterator!(MaxInt64StatsIterator, max, ParquetStatistics::Int64, i64);
-make_stats_iterator!(MinFloatStatsIterator, min, ParquetStatistics::Float, f32);
-make_stats_iterator!(MaxFloatStatsIterator, max, ParquetStatistics::Float, f32);
-make_stats_iterator!(MinDoubleStatsIterator, min, ParquetStatistics::Double, f64);
-make_stats_iterator!(MaxDoubleStatsIterator, max, ParquetStatistics::Double, f64);
+make_stats_iterator!(
+    MinInt32StatsIterator,
+    min_opt,
+    ParquetStatistics::Int32,
+    i32
+);
+make_stats_iterator!(
+    MaxInt32StatsIterator,
+    max_opt,
+    ParquetStatistics::Int32,
+    i32
+);
+make_stats_iterator!(
+    MinInt64StatsIterator,
+    min_opt,
+    ParquetStatistics::Int64,
+    i64
+);
+make_stats_iterator!(
+    MaxInt64StatsIterator,
+    max_opt,
+    ParquetStatistics::Int64,
+    i64
+);
+make_stats_iterator!(
+    MinFloatStatsIterator,
+    min_opt,
+    ParquetStatistics::Float,
+    f32
+);
+make_stats_iterator!(
+    MaxFloatStatsIterator,
+    max_opt,
+    ParquetStatistics::Float,
+    f32
+);
+make_stats_iterator!(
+    MinDoubleStatsIterator,
+    min_opt,
+    ParquetStatistics::Double,
+    f64
+);
+make_stats_iterator!(
+    MaxDoubleStatsIterator,
+    max_opt,
+    ParquetStatistics::Double,
+    f64
+);
 make_stats_iterator!(
     MinByteArrayStatsIterator,
-    min_bytes,
+    min_bytes_opt,
     ParquetStatistics::ByteArray,
     [u8]
 );
 make_stats_iterator!(
     MaxByteArrayStatsIterator,
-    max_bytes,
+    max_bytes_opt,
     ParquetStatistics::ByteArray,
     [u8]
 );
 make_stats_iterator!(
     MinFixedLenByteArrayStatsIterator,
-    min_bytes,
+    min_bytes_opt,
     ParquetStatistics::FixedLenByteArray,
     [u8]
 );
 make_stats_iterator!(
     MaxFixedLenByteArrayStatsIterator,
-    max_bytes,
+    max_bytes_opt,
     ParquetStatistics::FixedLenByteArray,
     [u8]
 );
@@ -218,19 +258,18 @@ macro_rules! make_decimal_stats_iterator {
             fn next(&mut self) -> Option<Self::Item> {
                 let next = self.iter.next();
                 next.map(|x| {
-                    x.and_then(|stats| {
-                        if !stats.has_min_max_set() {
-                            return None;
+                    x.and_then(|stats| match stats {
+                        ParquetStatistics::Int32(s) => {
+                            s.$func().map(|x| $stat_value_type::from(*x))
                         }
-                        match stats {
-                            ParquetStatistics::Int32(s) => Some($stat_value_type::from(*s.$func())),
-                            ParquetStatistics::Int64(s) => Some($stat_value_type::from(*s.$func())),
-                            ParquetStatistics::ByteArray(s) => Some($convert_func(s.$bytes_func())),
-                            ParquetStatistics::FixedLenByteArray(s) => {
-                                Some($convert_func(s.$bytes_func()))
-                            }
-                            _ => None,
+                        ParquetStatistics::Int64(s) => {
+                            s.$func().map(|x| $stat_value_type::from(*x))
                         }
+                        ParquetStatistics::ByteArray(s) => s.$bytes_func().map($convert_func),
+                        ParquetStatistics::FixedLenByteArray(s) => {
+                            s.$bytes_func().map($convert_func)
+                        }
+                        _ => None,
                     })
                 })
             }
@@ -244,29 +283,29 @@ macro_rules! make_decimal_stats_iterator {
 
 make_decimal_stats_iterator!(
     MinDecimal128StatsIterator,
-    min,
-    min_bytes,
+    min_opt,
+    min_bytes_opt,
     i128,
     from_bytes_to_i128
 );
 make_decimal_stats_iterator!(
     MaxDecimal128StatsIterator,
-    max,
-    max_bytes,
+    max_opt,
+    max_bytes_opt,
     i128,
     from_bytes_to_i128
 );
 make_decimal_stats_iterator!(
     MinDecimal256StatsIterator,
-    min,
-    min_bytes,
+    min_opt,
+    min_bytes_opt,
     i256,
     from_bytes_to_i256
 );
 make_decimal_stats_iterator!(
     MaxDecimal256StatsIterator,
-    max,
-    max_bytes,
+    max_opt,
+    max_bytes_opt,
     i256,
     from_bytes_to_i256
 );
@@ -1347,7 +1386,7 @@ impl<'a> StatisticsConverter<'a> {
         let null_counts = metadatas
             .into_iter()
             .map(|x| x.column(parquet_index).statistics())
-            .map(|s| s.map(|s| s.null_count()));
+            .map(|s| s.and_then(|s| s.null_count_opt()));
         Ok(UInt64Array::from_iter(null_counts))
     }
 

parquet/src/arrow/arrow_writer/mod.rs

@@ -2541,10 +2541,15 @@ mod tests {
                 row_offset += column.num_values() as usize;
 
                 let stats = column.statistics().unwrap();
-                assert!(stats.has_min_max_set());
                 if let Statistics::Int32(stats) = stats {
-                    assert_eq!(*stats.min() as u32, *src_slice.iter().min().unwrap());
-                    assert_eq!(*stats.max() as u32, *src_slice.iter().max().unwrap());
+                    assert_eq!(
+                        *stats.min_opt().unwrap() as u32,
+                        *src_slice.iter().min().unwrap()
+                    );
+                    assert_eq!(
+                        *stats.max_opt().unwrap() as u32,
+                        *src_slice.iter().max().unwrap()
+                    );
                 } else {
                     panic!("Statistics::Int32 missing")
                 }
@@ -2582,10 +2587,15 @@ mod tests {
                 row_offset += column.num_values() as usize;
 
                 let stats = column.statistics().unwrap();
-                assert!(stats.has_min_max_set());
                 if let Statistics::Int64(stats) = stats {
-                    assert_eq!(*stats.min() as u64, *src_slice.iter().min().unwrap());
-                    assert_eq!(*stats.max() as u64, *src_slice.iter().max().unwrap());
+                    assert_eq!(
+                        *stats.min_opt().unwrap() as u64,
+                        *src_slice.iter().min().unwrap()
+                    );
+                    assert_eq!(
+                        *stats.max_opt().unwrap() as u64,
+                        *src_slice.iter().max().unwrap()
+                    );
                 } else {
                     panic!("Statistics::Int64 missing")
                 }
@@ -2608,7 +2618,7 @@ mod tests {
             assert_eq!(row_group.num_columns(), 1);
             let column = row_group.column(0);
             let stats = column.statistics().unwrap();
-            assert_eq!(stats.null_count(), 2);
+            assert_eq!(stats.null_count_opt(), Some(2));
         }
     }
 
@@ -3069,8 +3079,8 @@ mod tests {
 
         // Column chunk of column "a" should have chunk level statistics
         if let Statistics::ByteArray(byte_array_stats) = a_col.statistics().unwrap() {
-            let min = byte_array_stats.min();
-            let max = byte_array_stats.max();
+            let min = byte_array_stats.min_opt().unwrap();
+            let max = byte_array_stats.max_opt().unwrap();
 
             assert_eq!(min.as_bytes(), &[b'a']);
             assert_eq!(max.as_bytes(), &[b'd']);
@@ -3141,8 +3151,8 @@ mod tests {
 
         // Column chunk of column "a" should have chunk level statistics
         if let Statistics::ByteArray(byte_array_stats) = a_col.statistics().unwrap() {
-            let min = byte_array_stats.min();
-            let max = byte_array_stats.max();
+            let min = byte_array_stats.min_opt().unwrap();
+            let max = byte_array_stats.max_opt().unwrap();
 
             assert_eq!(min.as_bytes(), &[b'a']);
             assert_eq!(max.as_bytes(), &[b'd']);

parquet/src/column/page.rs

@@ -370,15 +370,15 @@ mod tests {
             encoding: Encoding::PLAIN,
             def_level_encoding: Encoding::RLE,
             rep_level_encoding: Encoding::RLE,
-            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
         };
         assert_eq!(data_page.page_type(), PageType::DATA_PAGE);
         assert_eq!(data_page.buffer(), vec![0, 1, 2].as_slice());
         assert_eq!(data_page.num_values(), 10);
         assert_eq!(data_page.encoding(), Encoding::PLAIN);
         assert_eq!(
             data_page.statistics(),
-            Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
+            Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
         );
 
         let data_page_v2 = Page::DataPageV2 {
@@ -390,15 +390,15 @@ mod tests {
             def_levels_byte_len: 30,
             rep_levels_byte_len: 40,
             is_compressed: false,
-            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
         };
         assert_eq!(data_page_v2.page_type(), PageType::DATA_PAGE_V2);
         assert_eq!(data_page_v2.buffer(), vec![0, 1, 2].as_slice());
         assert_eq!(data_page_v2.num_values(), 10);
         assert_eq!(data_page_v2.encoding(), Encoding::PLAIN);
         assert_eq!(
             data_page_v2.statistics(),
-            Some(&Statistics::int32(Some(1), Some(2), None, 1, true))
+            Some(&Statistics::int32(Some(1), Some(2), None, Some(1), true))
         );
 
         let dict_page = Page::DictionaryPage {
@@ -422,7 +422,7 @@ mod tests {
             encoding: Encoding::PLAIN,
             def_level_encoding: Encoding::RLE,
             rep_level_encoding: Encoding::RLE,
-            statistics: Some(Statistics::int32(Some(1), Some(2), None, 1, true)),
+            statistics: Some(Statistics::int32(Some(1), Some(2), None, Some(1), true)),
         };
 
         let cpage = CompressedPage::new(data_page, 5);