Merge pull request #6558 from dantengsky/fix-serde-null_count

fix: serde compatibility of null_count

query/src/storages/fuse/statistics/accumulator.rs

@@ -132,7 +132,7 @@ impl StatisticsAccumulator {
             let col_stats = ColumnStatistics {
                 min,
                 max,
-                unset_bits: unset_bits as u64,
+                null_count: unset_bits as u64,
                 in_memory_size,
             };
 
@@ -281,7 +281,7 @@ pub fn columns_statistics(data_block: &DataBlock) -> Result<StatisticsOfColumns>
         let col_stats = ColumnStatistics {
             min,
             max,
-            unset_bits: unset_bits as u64,
+            null_count: unset_bits as u64,
             in_memory_size,
         };
 

query/src/storages/fuse/statistics/reducers.rs

@@ -52,14 +52,14 @@ pub fn reduce_block_statistics<T: Borrow<StatisticsOfColumns>>(
         .try_fold(HashMap::with_capacity(len), |mut acc, (id, stats)| {
             let mut min_stats = Vec::with_capacity(stats.len());
             let mut max_stats = Vec::with_capacity(stats.len());
-            let mut unset_bits = 0;
+            let mut null_count = 0;
             let mut in_memory_size = 0;
 
             for col_stats in stats {
                 min_stats.push(col_stats.min.clone());
                 max_stats.push(col_stats.max.clone());
 
-                unset_bits += col_stats.unset_bits;
+                null_count += col_stats.null_count;
                 in_memory_size += col_stats.in_memory_size;
             }
 
@@ -89,7 +89,7 @@ pub fn reduce_block_statistics<T: Borrow<StatisticsOfColumns>>(
             acc.insert(*id, ColumnStatistics {
                 min,
                 max,
-                unset_bits,
+                null_count,
                 in_memory_size,
             });
             Ok(acc)

query/src/storages/index/range_filter.rs

@@ -35,11 +35,15 @@ use crate::sessions::QueryContext;
 
 pub type StatisticsOfColumns = HashMap<u32, ColumnStatistics>;
 
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Eq, PartialEq)]
 pub struct ColumnStatistics {
     pub min: DataValue,
     pub max: DataValue,
-    pub unset_bits: u64,
+    // A non-backward compatible change has been introduced by [PR#6067](https://github.com/datafuselabs/databend/pull/6067/files#diff-20030750809780d6492d2fe215a8eb80294aa6a8a5af2cf1bebe17eb740cae35)
+    // , please also see [issue#6556](https://github.com/datafuselabs/databend/issues/6556)
+    // therefore, we alias `null_count` with `unset_bits`, to make subsequent versions backward compatible again
+    #[serde(alias = "unset_bits")]
+    pub null_count: u64,
     pub in_memory_size: u64,
 }
 
@@ -238,7 +242,7 @@ impl StatColumn {
                     k
                 ))
             })?;
-            return Ok(Some(Series::from_data(vec![stat.unset_bits])));
+            return Ok(Some(Series::from_data(vec![stat.null_count])));
         }
 
         let mut single_point = true;

query/tests/it/storages/fuse/misc.rs

@@ -0,0 +1,47 @@
+//  Copyright 2022 Datafuse Labs.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+use common_datavalues::DataValue;
+use databend_query::storages::index::ColumnStatistics;
+use serde_json::Value;
+
+// A non-backward compatible change has been introduced by [PR#6067](https://github.com/datafuselabs/databend/pull/6067/files#diff-20030750809780d6492d2fe215a8eb80294aa6a8a5af2cf1bebe17eb740cae35)
+// , please also see [issue#6556](https://github.com/datafuselabs/databend/issues/6556)
+// therefore, we alias `null_count` with `unset_bits`, to make subsequent versions backward compatible again
+#[test]
+fn test_issue_6556_column_statistics_ser_de_compatability_null_count_alias(
+) -> common_exception::Result<()> {
+    let col_stats = ColumnStatistics {
+        min: DataValue::Null,
+        max: DataValue::Null,
+        null_count: 0,
+        in_memory_size: 0,
+    };
+
+    let mut json_value = serde_json::to_value(&col_stats)?;
+
+    // replace "field" null_count with "unset_bits"
+    if let Value::Object(ref mut object) = json_value {
+        let unset_bits = object.remove("null_count").unwrap();
+        object.insert("unset_bits".to_owned(), unset_bits);
+    } else {
+        panic!("should return json object");
+    }
+
+    // test that we can still de-ser it
+    let new_json = json_value.to_string();
+    let new_col_stats = serde_json::from_str(&new_json)?;
+    assert_eq!(col_stats, new_col_stats);
+    Ok(())
+}

query/tests/it/storages/fuse/mod.rs

@@ -14,6 +14,7 @@
 
 mod io;
 mod meta;
+mod misc;
 mod operations;
 mod pruning;
 mod statistics;

query/tests/it/storages/fuse/operations/read_plan.rs

@@ -39,7 +39,7 @@ fn test_to_partitions() -> Result<()> {
     let col_stats_gen = |col_size| ColumnStatistics {
         min: DataValue::Int64(1),
         max: DataValue::Int64(2),
-        unset_bits: 0,
+        null_count: 0,
         in_memory_size: col_size as u64,
     };
 

query/tests/it/storages/index/range_filter.rs

@@ -40,19 +40,19 @@ async fn test_range_filter() -> Result<()> {
     stats.insert(0u32, ColumnStatistics {
         min: DataValue::Int64(1),
         max: DataValue::Int64(20),
-        unset_bits: 1,
+        null_count: 1,
         in_memory_size: 0,
     });
     stats.insert(1u32, ColumnStatistics {
         min: DataValue::Int64(3),
         max: DataValue::Int64(10),
-        unset_bits: 0,
+        null_count: 0,
         in_memory_size: 0,
     });
     stats.insert(2u32, ColumnStatistics {
         min: DataValue::String("abc".as_bytes().to_vec()),
         max: DataValue::String("bcd".as_bytes().to_vec()),
-        unset_bits: 0,
+        null_count: 0,
         in_memory_size: 0,
     });