Skip to content

Commit

Permalink
[FIX](type) fix matchExactType for complex type (apache#28233)
Browse files Browse the repository at this point in the history
fe matchExactType function should call type.matchTypes for its own logic, do not switch case to do special logic otherwise we may meet core in be like this.
 ```
F20231208 18:54:39.359673 680131 block.h:535] Check failed: _data_types[i]->is_nullable()  target type: Struct(l_info:Nullable(Array(Nullable(String)))) src type: Struct(col:Nullable(Array(Nullable(UInt8))))
*** Check failure stack trace: ***
    @     0x5584e952b926  google::LogMessage::SendToLog()
    @     0x5584e9527ef0  google::LogMessage::Flush()
    @     0x5584e952c169  google::LogMessageFatal::~LogMessageFatal()
    @     0x5584cf17201e  doris::vectorized::MutableBlock::merge_impl<>()
    @     0x5584ceac4b1d  doris::vectorized::MutableBlock::merge<>()
    @     0x5584d4dd7de3  doris::vectorized::VUnionNode::get_next_const()
    @     0x5584d4dd9a45  doris::vectorized::VUnionNode::get_next()
    @     0x5584bce469bd  std::__invoke_impl<>()
    @     0x5584bce466d0  std::__invoke<>()
    @     0x5584bce465c7  _ZNSt5_BindIFMN5doris8ExecNodeEFNS0_6StatusEPNS0_12RuntimeStateEPNS0_10vectorized5BlockEPbEPS1_St12_PlaceholderILi1EESC_ILi2EESC_ILi3EEEE6__callIS2_JOS4_OS7_OS8_EJLm0ELm1ELm2ELm3EEEET_OSt5tupleIJDpT0_EESt12_Index_tupleIJXspT1_EEE
    @     0x5584bce46358  std::_Bind<>::operator()<>()
    @     0x5584bce46208  std::__invoke_impl<>()
    @     0x5584bce46178  _ZSt10__invoke_rIN5doris6StatusERSt5_BindIFMNS0_8ExecNodeEFS1_PNS0_12RuntimeStateEPNS0_10vectorized5BlockEPbEPS3_St12_PlaceholderILi1EESD_ILi2EESD_ILi3EEEEJS5_S8_S9_EENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESL_E4typeEOSM_DpOSN_
    @     0x5584bce45c18  std::_Function_handler<>::_M_invoke()
    @     0x5584bce6412f  std::function<>::operator()()
    @     0x5584bce56382  doris::ExecNode::get_next_after_projects()
    @     0x5584bce26218  doris::PlanFragmentExecutor::get_vectorized_internal()
    @     0x5584bce2431b  doris::PlanFragmentExecutor::open_vectorized_internal()
    @     0x5584bce22a96  doris::PlanFragmentExecutor::open()
    @     0x5584bce27c9d  doris::PlanFragmentExecutor::execute()
    @     0x5584bcbdb3f8  doris::FragmentMgr::_exec_actual()
    @     0x5584bcbf982f  doris::FragmentMgr::exec_plan_fragment()::$_0::operator()()
    @     0x5584bcbf9715  std::__invoke_impl<>()
    @     0x5584bcbf96b5  _ZSt10__invoke_rIvRZN5doris11FragmentMgr18exec_plan_fragmentERKNS0_23TExecPlanFragmentParamsERKSt8functionIFvPNS0_12RuntimeStateEPNS0_6StatusEEEE3$_0JEENSt9enable_ifIX16is_invocable_r_vIT_T0_DpT1_EESH_E4typeEOSI_DpOSJ_
    @     0x5584bcbf942d  std::_Function_handler<>::_M_invoke()
    @     0x5584b9dfd883  std::function<>::operator()()
    @     0x5584bd6e3929  doris::FunctionRunnable::run()
    @     0x5584bd6cf8ce  doris::ThreadPool::dispatch_thread()
```
  • Loading branch information
amorynan authored and stephen committed Dec 28, 2023
1 parent 7b8da5b commit 0eb2f52
Show file tree
Hide file tree
Showing 12 changed files with 204 additions and 68 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,11 @@ public boolean matchesType(Type t) {
return false;
}

// Array(Null) is a virtual Array type, can match any Array(...) type
if (itemType.isNull() || ((ArrayType) t).getItemType().isNull()) {
return true;
if (((ArrayType) t).getContainsNull() != getContainsNull()) {
return false;
}

return itemType.matchesType(((ArrayType) t).itemType)
&& (((ArrayType) t).containsNull || !containsNull);
return itemType.matchesType(((ArrayType) t).itemType);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,11 @@ public boolean matchesType(Type t) {
return false;
}

if ((keyType.isNull() || ((MapType) t).getKeyType().isNull())
&& (valueType.isNull() || ((MapType) t).getKeyType().isNull())) {
return true;
if (((MapType) t).getIsKeyContainsNull() != getIsKeyContainsNull()) {
return false;
}
if (((MapType) t).getIsValueContainsNull() != getIsValueContainsNull()) {
return false;
}

return keyType.matchesType(((MapType) t).keyType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
}

if (specializedType != null
&& !specializedType.isNull()
&& !specificType.equals(specializedType)
&& !specificType.matchesType(specializedType)
&& !Type.isImplicitlyCastable(specificType, specializedType, true, enableDecimal256)
Expand All @@ -104,7 +105,7 @@ public Type specializeTemplateType(Type specificType, Map<String, Type> speciali
name, specificType, specializedType));
}

if (specializedType == null) {
if (specializedType == null || specializedType.isNull()) {
specializedTypeMap.put(name, specificType);
}
return specializedTypeMap.get(name);
Expand Down
17 changes: 1 addition & 16 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
Original file line number Diff line number Diff line change
Expand Up @@ -2241,6 +2241,7 @@ public static boolean matchExactType(Type type1, Type type2) {
}

public static boolean matchExactType(Type type1, Type type2, boolean ignorePrecision) {
// we should make type decide to match other for itself to impl matchesType instead of switch case types
if (type1.matchesType(type2)) {
if (PrimitiveType.typeWithPrecision.contains(type2.getPrimitiveType())) {
// For types which has precision and scale, we also need to check quality between precisions and scales
Expand All @@ -2253,22 +2254,6 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci
return isSameDecimalTypeWithDifferentPrecision(((ScalarType) type2).decimalPrecision(),
((ScalarType) type1).decimalPrecision());
}
} else if (type2.isArrayType()) {
// For types array, we also need to check contains null for case like
// cast(array<not_null(int)> as array<int>)
if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) {
return false;
}
return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType());
} else if (type2.isMapType()) {
if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) {
return false;
}
if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) {
return false;
}
return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType())
&& matchExactType(((MapType) type2).getValueType(), ((MapType) type1).getValueType());
} else {
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1875,6 +1875,14 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
&& fnName.getFunction().equalsIgnoreCase("map")) {
ix = i % 2 == 0 ? 0 : 1;
}
// array_zip varargs special case array_zip(array1, array2, ...)
// we only specialize array_zip with first array type, next type we same with custom type
if (i >= args.length && (fnName.getFunction().equalsIgnoreCase("array_zip"))) {
if (argTypes[i].isNull()) {
uncheckedCastChild(args[i - 1], i);
}
continue;
}

if (i == 0 && (fnName.getFunction().equalsIgnoreCase("char"))) {
continue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,17 +47,14 @@ public static void initBuiltins(FunctionSet functionSet) {

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(t), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

// for array type
for (Type complexType : Lists.newArrayList(Type.ARRAY, Type.MAP, Type.GENERIC_STRUCT)) {
functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL,
null, Lists.newArrayList(complexType), Type.BOOLEAN,
NullableMode.ALWAYS_NOT_NULLABLE));
}

functionSet.addBuiltinBothScalaAndVectorized(ScalarFunction.createBuiltinOperator(IS_NOT_NULL, null,
Lists.newArrayList(complexType), Type.BOOLEAN, NullableMode.ALWAYS_NOT_NULLABLE));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
import org.apache.doris.thrift.TExprNodeType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -106,6 +108,11 @@ public String getStringValueInFe() {
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.STRUCT_LITERAL;
((StructType) type).getFields().forEach(v -> msg.setChildType(v.getType().getPrimitiveType().toThrift()));
TTypeDesc container = new TTypeDesc();
container.setTypes(new ArrayList<TTypeNode>());
type.toThrift(container);
msg.setType(container);
}

@Override
Expand Down
42 changes: 35 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
Original file line number Diff line number Diff line change
Expand Up @@ -386,16 +386,31 @@ public Function specializeTemplateFunction(Function templateFunction, Function r
throw new TypeException(templateFunction
+ " is not support for template since it's not a ScalarFunction");
}
Type[] args = specializedFunction.getArgs();
ArrayList<Type> args = new ArrayList<>();
Collections.addAll(args, specializedFunction.getArgs());
Map<String, Type> specializedTypeMap = Maps.newHashMap();
boolean enableDecimal256 = SessionVariable.getEnableDecimal256();
for (int i = 0; i < args.length; i++) {
if (args[i].hasTemplateType()) {
int i = 0;
for (; i < args.size(); i++) {
if (args.get(i).hasTemplateType()) {
hasTemplateType = true;
args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false,
enableDecimal256);
// if args[i] is template type, and requestFunction.getArgs()[i] NULL_TYPE, we need call function
// deduce to get the specific type
Type deduceType = requestFunction.getArgs()[i];
if (requestFunction.getArgs()[i].isNull()
|| (requestFunction.getArgs()[i] instanceof ArrayType
&& ((ArrayType) requestFunction.getArgs()[i]).getItemType().isNull())
&& FunctionTypeDeducers.DEDUCERS.containsKey(specializedFunction.functionName())) {
deduceType = FunctionTypeDeducers.deduce(specializedFunction.functionName(), i, requestFunction.getArgs());
args.set(i, args.get(i).specializeTemplateType(deduceType == null ? requestFunction.getArgs()[i]
: deduceType, specializedTypeMap, false, enableDecimal256));
} else {
args.set(i, args.get(i).specializeTemplateType(requestFunction.getArgs()[i],
specializedTypeMap, false, enableDecimal256));
}
}
}
specializedFunction.setArgs(args);
if (specializedFunction.getReturnType().hasTemplateType()) {
hasTemplateType = true;
specializedFunction.setReturnType(
Expand Down Expand Up @@ -426,7 +441,7 @@ public Function resolveInferenceFunction(Function inferenceFunction, Function re
newTypes[i] = inputType;
}
}
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), newTypes);
Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), 0, newTypes);
if (newRetType != null && inferenceFunction instanceof ScalarFunction) {
ScalarFunction f = (ScalarFunction) inferenceFunction;
return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(newTypes), newRetType, f.hasVarArgs(),
Expand All @@ -448,7 +463,20 @@ public static boolean isCastMatchAllowed(Function desc, Function candicate) {
final Type[] candicateArgTypes = candicate.getArgs();
if (!(descArgTypes[0] instanceof ScalarType)
|| !(candicateArgTypes[0] instanceof ScalarType)) {
if (candicateArgTypes[0] instanceof ArrayType || candicateArgTypes[0] instanceof MapType) {
if (candicateArgTypes[0] instanceof ArrayType) {
// match is exactly type. but for null type , with in array|map elem can not return true, because for
// be will make null_type to uint8
// so here meet null_type just make true as allowed, descArgTypes[0]).getItemType().isNull() is for
// empty literal like: []|{}
if (descArgTypes[0] instanceof ArrayType && ((ArrayType) descArgTypes[0]).getItemType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
} else if (candicateArgTypes[0] instanceof MapType) {
if (descArgTypes[0] instanceof MapType && ((MapType) descArgTypes[0]).getKeyType().isNull()
&& ((MapType) descArgTypes[0]).getValueType().isNull()) {
return true;
}
return descArgTypes[0].matchesType(candicateArgTypes[0]);
}
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,26 +23,48 @@
import java.util.List;

public class FunctionTypeDeducers {

public interface TypeDeducer {
public Type deduce(Type[] args);
public Type deduce(int argIdx, Type[] args);
}

public static final ImmutableMap<String, TypeDeducer> DEDUCERS = ImmutableMap.<String, TypeDeducer>builder()
.put("named_struct", new NamedStructDeducer())
.put("struct_element", new StructElementDeducer())
.put("array_contains", new ArrayElemFuncDeducer())
.put("array_pushback", new ArrayElemFuncDeducer())
.put("element_at", new ArrayElemFuncDeducer())
.build();

public static Type deduce(String fnName, Type[] args) {
public static Type deduce(String fnName, int argIdx, Type[] args) {
if (DEDUCERS.containsKey(fnName)) {
return DEDUCERS.get(fnName).deduce(args);
return DEDUCERS.get(fnName).deduce(argIdx, args);
}
return null;
}

public static class ArrayElemFuncDeducer implements TypeDeducer {
@Override
public Type deduce(int argIdx, Type[] args) {
if (args.length >= 2) {
if (argIdx == 0) {
// first args should only to be array or null
return args[0] instanceof ArrayType || args[0].isNull() ? new ArrayType(args[1]) : args[0];
} else if (args[0].isNull()) {
// first arg is null, later element is not contains
return args[argIdx];
} else if (Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false, true)) {
return args[argIdx];
} else {
return null;
}
}
return null;
}
}

public static class NamedStructDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
List<Type> evenArgs = Lists.newArrayList();
for (int i = 0; i < args.length; i++) {
if ((i & 1) == 1) {
Expand All @@ -55,7 +77,7 @@ public Type deduce(Type[] args) {

public static class StructElementDeducer implements TypeDeducer {
@Override
public Type deduce(Type[] args) {
public Type deduce(int argIdx, Type[] args) {
if (args[0] instanceof StructType) {
return Type.ANY_ELEMENT_TYPE;
}
Expand Down
27 changes: 7 additions & 20 deletions gensrc/script/doris_builtins_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,24 +165,7 @@
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_VARCHAR', 'ARRAY_VARCHAR'], 'ALWAYS_NULLABLE'],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_STRING', 'ARRAY_STRING'], 'ALWAYS_NULLABLE'],

[['array_contains'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'BOOLEAN'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_TINYINT', 'TINYINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_SMALLINT', 'SMALLINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_INT', 'INT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_BIGINT', 'BIGINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_LARGEINT', 'LARGEINT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATETIME', 'DATETIME'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATE', 'DATE'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATETIMEV2', 'DATETIMEV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DATEV2', 'DATEV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_FLOAT', 'FLOAT'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DOUBLE', 'DOUBLE'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMALV2', 'DECIMALV2'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL32', 'DECIMAL32'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL64', 'DECIMAL64'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_DECIMAL128', 'DECIMAL128'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_VARCHAR', 'VARCHAR'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY_STRING', 'STRING'], 'CUSTOM'],
[['array_contains'], 'BOOLEAN', ['ARRAY<T>', 'T'], 'CUSTOM', ['T']],

[['array_cum_sum'], 'ARRAY_BIGINT', ['ARRAY_TINYINT'], ''],
[['array_cum_sum'], 'ARRAY_BIGINT', ['ARRAY_SMALLINT'], ''],
Expand Down Expand Up @@ -272,7 +255,7 @@
[['array_position'], 'BIGINT', ['ARRAY_VARCHAR', 'VARCHAR'], 'CUSTOM'],
[['array_position'], 'BIGINT', ['ARRAY_STRING', 'STRING'], 'CUSTOM'],

[['cardinality', 'size', 'array_size'], 'BIGINT', ['ARRAY'], ''],
[['cardinality', 'size', 'array_size'], 'BIGINT', ['ARRAY<T>'], '', ['T']],
[['array_distinct'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN'], ''],
[['array_distinct'], 'ARRAY_TINYINT', ['ARRAY_TINYINT'], ''],
[['array_distinct'], 'ARRAY_SMALLINT', ['ARRAY_SMALLINT'], ''],
Expand Down Expand Up @@ -772,6 +755,7 @@
[['array_pushfront'], 'ARRAY_VARCHAR', ['ARRAY_VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['array_pushfront'], 'ARRAY_STRING', ['ARRAY_STRING', 'STRING'], 'ALWAYS_NULLABLE'],

[['array_pushback'], 'ARRAY<T>', ['ARRAY<T>', 'T'], 'ALWAYS_NULLABLE', ['T']],
[['array_pushback'], 'ARRAY_BOOLEAN', ['ARRAY_BOOLEAN', 'BOOLEAN'], 'ALWAYS_NULLABLE'],
[['array_pushback'], 'ARRAY_TINYINT', ['ARRAY_TINYINT', 'TINYINT'], 'ALWAYS_NULLABLE'],
[['array_pushback'], 'ARRAY_SMALLINT', ['ARRAY_SMALLINT', 'SMALLINT'], 'ALWAYS_NULLABLE'],
Expand Down Expand Up @@ -833,7 +817,7 @@
[['array_range'], 'ARRAY_INT', ['INT', 'INT'], 'ALWAYS_NULLABLE'],
[['array_range'], 'ARRAY_INT', ['INT', 'INT', 'INT'], 'ALWAYS_NULLABLE'],

[['array_zip'], 'ARRAY', ['ARRAY', '...'], ''],
[['array_zip'], 'ARRAY', ['ARRAY<T>', '...'], '', ['T']],


# reverse function for string builtin
Expand Down Expand Up @@ -1469,6 +1453,9 @@
[['nullif'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['nullif'], 'STRING', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],

[['is_null_pred'], "BOOLEAN", ['T', '...'], 'ALWAYS_NULLABLE', ['T']],
[['is_not_null_pred'], "BOOLEAN", ['T', '...'], 'ALWAYS_NULLABLE', ['T']],

[['ifnull', 'nvl'], 'BOOLEAN', ['BOOLEAN', 'BOOLEAN'], 'CUSTOM'],
[['ifnull', 'nvl'], 'TINYINT', ['TINYINT', 'TINYINT'], 'CUSTOM'],
[['ifnull', 'nvl'], 'SMALLINT', ['SMALLINT', 'SMALLINT'], 'CUSTOM'],
Expand Down
Loading

0 comments on commit 0eb2f52

Please sign in to comment.