Skip to content

Commit

Permalink
Merge pull request #8778 from youngsofun/ref
Browse files Browse the repository at this point in the history 
refactor(input format): refactor with FieldEncoder.
  • Loading branch information
@Xuanwo
Xuanwo authored Nov 14, 2022
2 parents 3560ed0 + cda3baf commit b24be85
Show file tree
Hide file tree
Showing 45 changed files with 1,083 additions and 490 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion src/common/io/src/cursor_ext/cursor_read_bytes_ext.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ where T: AsRef<[u8]>
fn ignore_bytes(&mut self, bs: &[u8]) -> bool {
let available = self.remaining_slice();
let len = bs.len();
if available.len() <= len {
if available.len() < len {
return false;
}
let eq = available[..len].iter().zip(bs).all(|(x, y)| x == y);
Expand Down
38 changes: 1 addition & 37 deletions src/query/datavalues/src/types/deserializations/array.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use std::io::Cursor;

use common_exception::ErrorCode;
use common_exception::Result;
use common_io::cursor_ext::*;
use common_io::prelude::BinaryRead;
use common_io::prelude::FormatSettings;

Expand Down Expand Up @@ -44,7 +41,7 @@ impl TypeDeserializer for ArrayDeserializer {
Ok(())
}

fn de_default(&mut self, _format: &FormatSettings) {
fn de_default(&mut self) {
self.builder.append_default();
}

Expand Down Expand Up @@ -87,39 +84,6 @@ impl TypeDeserializer for ArrayDeserializer {
}
}

fn de_text<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
reader.must_ignore_byte(b'[')?;
let mut idx = 0;
loop {
let _ = reader.ignore_white_spaces();
if reader.ignore_byte(b']') {
break;
}
if idx != 0 {
reader.must_ignore_byte(b',')?;
}
let _ = reader.ignore_white_spaces();
self.inner.de_text_quoted(reader, format)?;
idx += 1;
}
let mut values = Vec::with_capacity(idx);
for _ in 0..idx {
values.push(self.inner.pop_data_value()?);
}
values.reverse();
self.builder.append_value(ArrayValue::new(values));
Ok(())
}

fn de_whole_text(&mut self, reader: &[u8], format: &FormatSettings) -> Result<()> {
let mut reader = Cursor::new(reader);
self.de_text(&mut reader, format)
}

fn append_data_value(&mut self, value: DataValue, _format: &FormatSettings) -> Result<()> {
self.builder.append_data_value(value)
}
Expand Down
33 changes: 1 addition & 32 deletions src/query/datavalues/src/types/deserializations/boolean.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use std::io::Cursor;

use common_exception::ErrorCode;
use common_exception::Result;
use common_io::cursor_ext::*;
use common_io::prelude::BinaryRead;
use common_io::prelude::FormatSettings;

Expand All @@ -37,7 +34,7 @@ impl TypeDeserializer for BooleanDeserializer {
Ok(())
}

fn de_default(&mut self, _format: &FormatSettings) {
fn de_default(&mut self) {
self.builder.append_value(false);
}

Expand Down Expand Up @@ -65,34 +62,6 @@ impl TypeDeserializer for BooleanDeserializer {
Ok(())
}

fn de_whole_text(&mut self, reader: &[u8], _format: &FormatSettings) -> Result<()> {
if reader.eq_ignore_ascii_case(b"true") {
self.builder.append_value(true);
} else if reader.eq_ignore_ascii_case(b"false") {
self.builder.append_value(false);
} else {
return Err(ErrorCode::BadBytes("Incorrect boolean value"));
}
Ok(())
}

fn de_text<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
_format: &FormatSettings,
) -> Result<()> {
let v = if reader.ignore_insensitive_bytes(b"true") {
Ok(true)
} else if reader.ignore_insensitive_bytes(b"false") {
Ok(false)
} else {
Err(ErrorCode::BadBytes("Incorrect boolean value"))
}?;

self.builder.append_value(v);
Ok(())
}

fn append_data_value(&mut self, value: DataValue, _format: &FormatSettings) -> Result<()> {
self.builder.append_value(value.as_bool()?);
Ok(())
Expand Down
62 changes: 4 additions & 58 deletions src/query/datavalues/src/types/deserializations/date.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use num::cast::AsPrimitive;
use crate::prelude::*;

pub struct DateDeserializer<T: PrimitiveType> {
pub buffer: Vec<u8>,
pub builder: MutablePrimitiveColumn<T>,
}

Expand All @@ -50,7 +51,7 @@ where
Ok(())
}

fn de_default(&mut self, _format: &FormatSettings) {
fn de_default(&mut self) {
self.builder.append_value(T::default());
}

Expand All @@ -75,7 +76,7 @@ where
serde_json::Value::String(v) => {
let mut reader = Cursor::new(v.as_bytes());
let date = reader.read_date_text(&format.timezone)?;
let days = uniform(date);
let days = uniform_date(date);
check_date(days.as_i32())?;
self.builder.append_value(days);
Ok(())
Expand All @@ -84,61 +85,6 @@ where
}
}

fn de_whole_text(&mut self, reader: &[u8], format: &FormatSettings) -> Result<()> {
let mut reader = Cursor::new(reader);
let date = reader.read_date_text(&format.timezone)?;
let days = uniform(date);
check_date(days.as_i32())?;
reader.must_eof()?;
self.builder.append_value(days);
Ok(())
}

fn de_text_quoted<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
reader.must_ignore_byte(b'\'')?;
let date = reader.read_date_text(&format.timezone);
reader.must_ignore_byte(b'\'')?;
if date.is_err() {
return Err(date.err().unwrap());
}
let days = uniform(date.unwrap());
check_date(days.as_i32())?;

self.builder.append_value(days);
Ok(())
}

fn de_text<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
let date = reader.read_date_text(&format.timezone)?;
let days = uniform(date);
check_date(days.as_i32())?;
self.builder.append_value(days);
Ok(())
}

fn de_text_json<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
reader.must_ignore_byte(b'"')?;
let date = reader.read_date_text(&format.timezone)?;
let days = uniform(date);
check_date(days.as_i32())?;
reader.must_ignore_byte(b'"')?;

self.builder.append_value(days);
Ok(())
}

fn append_data_value(&mut self, value: DataValue, _format: &FormatSettings) -> Result<()> {
let v = value.as_i64()? as i32;
check_date(v)?;
Expand All @@ -156,7 +102,7 @@ where
}

#[inline]
fn uniform<T>(date: NaiveDate) -> T
pub fn uniform_date<T>(date: NaiveDate) -> T
where
i32: AsPrimitive<T>,
T: PrimitiveType,
Expand Down
27 changes: 1 addition & 26 deletions src/query/datavalues/src/types/deserializations/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ mod string;
mod struct_;
mod timestamp;
mod variant;
use std::io::Cursor;

pub use array::*;
pub use boolean::*;
Expand All @@ -48,7 +47,7 @@ pub trait TypeDeserializer: Send + Sync {

fn de_binary(&mut self, reader: &mut &[u8], format: &FormatSettings) -> Result<()>;

fn de_default(&mut self, format: &FormatSettings);
fn de_default(&mut self);

fn de_fixed_binary_batch(
&mut self,
Expand All @@ -64,30 +63,6 @@ pub trait TypeDeserializer: Send + Sync {
false
}

fn de_whole_text(&mut self, reader: &[u8], format: &FormatSettings) -> Result<()>;

fn de_text<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()>;

fn de_text_json<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
self.de_text(reader, format)
}

fn de_text_quoted<R: AsRef<[u8]>>(
&mut self,
reader: &mut Cursor<R>,
format: &FormatSettings,
) -> Result<()> {
self.de_text(reader, format)
}

fn append_data_value(&mut self, value: DataValue, format: &FormatSettings) -> Result<()>;

/// Note this method will return err only when inner builder is empty.
Expand Down
18 changes: 1 addition & 17 deletions src/query/datavalues/src/types/deserializations/null.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use std::io::Cursor;

use common_exception::Result;
use common_io::prelude::FormatSettings;

Expand All @@ -37,7 +35,7 @@ impl TypeDeserializer for NullDeserializer {
Ok(())
}

fn de_default(&mut self, _format: &FormatSettings) {
fn de_default(&mut self) {
self.builder.append_default();
}

Expand All @@ -59,20 +57,6 @@ impl TypeDeserializer for NullDeserializer {
Ok(())
}

fn de_whole_text(&mut self, _reader: &[u8], _format: &FormatSettings) -> Result<()> {
self.builder.append_default();
Ok(())
}

fn de_text<R: AsRef<[u8]>>(
&mut self,
_reader: &mut Cursor<R>,
_format: &FormatSettings,
) -> Result<()> {
self.builder.append_default();
Ok(())
}

fn append_data_value(&mut self, _value: DataValue, _format: &FormatSettings) -> Result<()> {
self.builder.append_default();
Ok(())
Expand Down
Loading

1 comment on commit b24be85

@vercel
Copy link

@vercel vercel bot commented on b24be85 Nov 14, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

databend – ./

databend-git-main-databend.vercel.app
databend.vercel.app
databend-databend.vercel.app
databend.rs

Please sign in to comment.