Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ISSUE-4558: Add check_json function #4606

Merged
merged 7 commits into from
Mar 30, 2022
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions common/functions/src/scalars/semi_structureds/check_json.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
// Copyright 2021 Datafuse Labs.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::fmt;
use std::sync::Arc;

use common_arrow::arrow::bitmap::Bitmap;
use common_datavalues::prelude::*;
use common_exception::Result;
use serde_json::Value as JsonValue;

use crate::scalars::Function;
use crate::scalars::FunctionDescription;
use crate::scalars::FunctionFeatures;

#[derive(Clone)]
pub struct CheckJsonFunction {
display_name: String,
}

impl CheckJsonFunction {
pub fn try_create(display_name: &str) -> Result<Box<dyn Function>> {
Ok(Box::new(CheckJsonFunction {
display_name: display_name.to_string(),
}))
}

pub fn desc() -> FunctionDescription {
FunctionDescription::creator(Box::new(Self::try_create)).features(
FunctionFeatures::default()
.deterministic()
.monotonicity()
.num_arguments(1),
)
}
}

impl Function for CheckJsonFunction {
fn name(&self) -> &str {
&*self.display_name
}

fn return_type(&self, args: &[&DataTypePtr]) -> Result<DataTypePtr> {
if args[0].data_type_id() == TypeID::Null {
return Ok(NullType::arc());
}

if args[0].is_nullable() {
return Ok(Arc::new(NullableType::create(VariantType::arc())));
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
}

Ok(VariantType::arc())
}

fn eval(&self, columns: &ColumnsWithField, input_rows: usize) -> Result<ColumnRef> {
let mut column = columns[0].column();
let mut _all_null: bool = false;
let mut source_valids: Option<&Bitmap> = None;
if column.is_nullable() {
(_all_null, source_valids) = column.validity();
let nullable_column: &NullableColumn = Series::check_get(column)?;
column = nullable_column.inner();
}

if column.data_type_id() == TypeID::Null {
return NullType::arc().create_constant_column(&DataValue::Null, input_rows);
}

let mut builder = NullableColumnBuilder::<JsonValue>::with_capacity(input_rows);
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved

match column.data_type_id() {
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
TypeID::Array => {
for _i in 0..input_rows {
builder.append(
&JsonValue::String(
"Invalid argument types for function 'CHECK_JSON': (ARRAY)".to_string(),
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
),
true,
)
}
}
TypeID::Struct => {
for _i in 0..input_rows {
builder.append(
&JsonValue::String(
"Invalid argument types for function 'CHECK_JSON': (STRUCT)"
.to_string(),
),
true,
)
}
}
TypeID::String => {
let c: &StringColumn = Series::check_get(column)?;
for (i, v) in c.iter().enumerate() {
if let Some(source_valids) = source_valids {
if !source_valids.get_bit(i) {
builder.append_null();
continue;
}
}

match std::str::from_utf8(v) {
Ok(v) => match serde_json::from_str::<JsonValue>(v) {
Ok(_v) => builder.append_null(),
Err(e) => builder.append(&JsonValue::String(e.to_string()), true),
},
Err(e) => builder.append(&JsonValue::String(e.to_string()), true),
}
}
}
TypeID::Variant => {
let c: &ObjectColumn<JsonValue> = Series::check_get(column)?;
for v in c.iter() {
if let JsonValue::String(s) = v {
match serde_json::from_str::<JsonValue>(s.as_str()) {
Ok(_v) => builder.append_null(),
Err(e) => builder.append(&JsonValue::String(e.to_string()), true),
}
} else {
builder.append_null()
}
}
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
}
_ => {
for _i in 0..input_rows {
builder.append_null()
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
}
}
}

return Ok(builder.build(input_rows));
}
}

impl fmt::Display for CheckJsonFunction {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.display_name.to_uppercase())
}
}
2 changes: 2 additions & 0 deletions common/functions/src/scalars/semi_structureds/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.

mod check_json;
mod parse_json;
mod semi_structured;

pub use check_json::CheckJsonFunction;
pub use parse_json::ParseJsonFunction;
pub use parse_json::TryParseJsonFunction;
pub use semi_structured::SemiStructuredFunction;
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

use super::parse_json::ParseJsonFunction;
use super::parse_json::TryParseJsonFunction;
use crate::scalars::CheckJsonFunction;
use crate::scalars::FunctionFactory;

pub struct SemiStructuredFunction;
Expand All @@ -22,5 +23,6 @@ impl SemiStructuredFunction {
pub fn register(factory: &mut FunctionFactory) {
factory.register("parse_json", ParseJsonFunction::desc());
factory.register("try_parse_json", TryParseJsonFunction::desc());
factory.register("check_json", CheckJsonFunction::desc());
}
}
2 changes: 1 addition & 1 deletion common/functions/tests/it/scalars/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ mod maths;
mod nullables;
mod others;
mod scalar_function2_test;
mod semi_structureds;
mod semi_structures;
kevinw66 marked this conversation as resolved.
Show resolved Hide resolved
mod strings;
mod tuples;
mod udfs;
Expand Down
Loading