refactor(query): migrate deserializations to expression

src/common/arrow/Cargo.toml

@@ -34,7 +34,7 @@ simd = ["arrow/simd"]
 # Workspace dependencies
 
 # Crates.io dependencies
-arrow = { package = "arrow2", git = "https://github.com/RinChanNOWWW/arrow2", rev = "8bd6417", default-features = false, features = [
+arrow = { package = "arrow2", git = "https://github.com/jorgecarleitao/arrow2", rev = "562de6a", default-features = false, features = [
     "io_parquet",
     "io_parquet_compression",
 ] }

src/query/expression/src/deserializations/boolean.rs

@@ -0,0 +1,62 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_arrow::arrow::bitmap::MutableBitmap;
+use common_io::prelude::*;
+
+use crate::Column;
+use crate::Scalar;
+use crate::TypeDeserializer;
+
+pub struct BooleanDeserializer {
+    pub builder: MutableBitmap,
+}
+
+impl BooleanDeserializer {
+    pub fn create() -> Self {
+        Self {
+            builder: MutableBitmap::new(),
+        }
+    }
+}
+
+impl TypeDeserializer for BooleanDeserializer {
+    fn memory_size(&self) -> usize {
+        self.builder.as_slice().len()
+    }
+
+    fn de_default(&mut self, _format: &FormatSettings) {
+        self.builder.push(false);
+    }
+
+    fn append_data_value(&mut self, value: Scalar, _format: &FormatSettings) -> Result<(), String> {
+        let v = value
+            .as_boolean()
+            .ok_or_else(|| "Unable to get boolean value".to_string())?;
+        self.builder.push(*v);
+        Ok(())
+    }
+
+    fn pop_data_value(&mut self) -> Result<Scalar, String> {
+        match self.builder.pop() {
+            Some(v) => Ok(Scalar::Boolean(v)),
+            None => Err("Boolean column is empty when pop data value".to_string()),
+        }
+    }
+
+    fn finish_to_column(&mut self) -> Column {
+        self.builder.shrink_to_fit();
+        Column::Boolean(std::mem::take(&mut self.builder).into())
+    }
+}

src/query/expression/src/deserializations/date.rs

@@ -0,0 +1,63 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_io::prelude::*;
+
+use crate::types::date::check_date;
+use crate::Column;
+use crate::Scalar;
+use crate::TypeDeserializer;
+
+pub struct DateDeserializer {
+    pub builder: Vec<i32>,
+}
+
+impl DateDeserializer {
+    pub fn create() -> Self {
+        Self {
+            builder: Vec::new(),
+        }
+    }
+}
+
+impl TypeDeserializer for DateDeserializer {
+    fn memory_size(&self) -> usize {
+        self.builder.len() * std::mem::size_of::<i32>()
+    }
+
+    fn de_default(&mut self, _format: &FormatSettings) {
+        self.builder.push(i32::default());
+    }
+
+    fn append_data_value(&mut self, value: Scalar, _format: &FormatSettings) -> Result<(), String> {
+        let v = value
+            .as_date()
+            .ok_or_else(|| "Unable to get date value".to_string())?;
+        check_date(*v as i64)?;
+        self.builder.push(*v);
+        Ok(())
+    }
+
+    fn pop_data_value(&mut self) -> Result<Scalar, String> {
+        match self.builder.pop() {
+            Some(v) => Ok(Scalar::Date(v)),
+            None => Err("Date column is empty when pop data value".to_string()),
+        }
+    }
+
+    fn finish_to_column(&mut self) -> Column {
+        self.builder.shrink_to_fit();
+        Column::Date(std::mem::take(&mut self.builder).into())
+    }
+}

src/query/expression/src/deserializations/empty_array.rs

@@ -0,0 +1,62 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_io::prelude::*;
+
+use crate::Column;
+use crate::Scalar;
+use crate::TypeDeserializer;
+
+#[derive(Debug, Default)]
+pub struct EmptyArrayDeserializer {
+    pub len: usize,
+}
+
+impl EmptyArrayDeserializer {
+    pub fn create() -> Self {
+        Self { len: 0 }
+    }
+}
+
+impl TypeDeserializer for EmptyArrayDeserializer {
+    fn memory_size(&self) -> usize {
+        self.len
+    }
+
+    fn de_default(&mut self, _format: &FormatSettings) {
+        self.len += 1;
+    }
+
+    fn append_data_value(
+        &mut self,
+        _value: Scalar,
+        _format: &FormatSettings,
+    ) -> Result<(), String> {
+        self.len += 1;
+        Ok(())
+    }
+
+    fn pop_data_value(&mut self) -> Result<Scalar, String> {
+        if self.len > 0 {
+            self.len -= 1;
+            Ok(Scalar::EmptyArray)
+        } else {
+            Err("EmptyArray column is empty when pop data value".to_string())
+        }
+    }
+
+    fn finish_to_column(&mut self) -> Column {
+        Column::EmptyArray { len: self.len }
+    }
+}

src/query/expression/src/deserializations/mod.rs

@@ -0,0 +1,51 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_io::prelude::*;
+
+mod boolean;
+mod date;
+mod empty_array;
+mod null;
+mod nullable;
+mod number;
+mod string;
+mod timestamp;
+mod variant;
+
+pub use boolean::*;
+pub use date::*;
+pub use empty_array::*;
+pub use null::*;
+pub use nullable::*;
+pub use number::*;
+pub use string::*;
+pub use timestamp::*;
+pub use variant::*;
+
+use crate::Column;
+use crate::Scalar;
+
+pub trait TypeDeserializer: Send + Sync {
+    fn memory_size(&self) -> usize;
+
+    fn de_default(&mut self, format: &FormatSettings);
+
+    fn append_data_value(&mut self, value: Scalar, format: &FormatSettings) -> Result<(), String>;
+
+    /// Note this method will return err only when inner builder is empty.
+    fn pop_data_value(&mut self) -> Result<Scalar, String>;
+
+    fn finish_to_column(&mut self) -> Column;
+}

src/query/expression/src/deserializations/null.rs

@@ -0,0 +1,62 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_io::prelude::*;
+
+use crate::Column;
+use crate::Scalar;
+use crate::TypeDeserializer;
+
+#[derive(Debug, Default)]
+pub struct NullDeserializer {
+    pub len: usize,
+}
+
+impl NullDeserializer {
+    pub fn create() -> Self {
+        Self { len: 0 }
+    }
+}
+
+impl TypeDeserializer for NullDeserializer {
+    fn memory_size(&self) -> usize {
+        self.len
+    }
+
+    fn de_default(&mut self, _format: &FormatSettings) {
+        self.len += 1;
+    }
+
+    fn append_data_value(
+        &mut self,
+        _value: Scalar,
+        _format: &FormatSettings,
+    ) -> Result<(), String> {
+        self.len += 1;
+        Ok(())
+    }
+
+    fn pop_data_value(&mut self) -> Result<Scalar, String> {
+        if self.len > 0 {
+            self.len -= 1;
+            Ok(Scalar::Null)
+        } else {
+            Err("Null column is empty when pop data value".to_string())
+        }
+    }
+
+    fn finish_to_column(&mut self) -> Column {
+        Column::Null { len: self.len }
+    }
+}