Skip to content

Commit

Permalink
[fix](function) fix str_to_date default return type scale for nereids (…
Browse files Browse the repository at this point in the history 
…apache#24932)

fix str_to_date default return type scale for nereids
  • Loading branch information
@zclllyybb
zclllyybb authored Oct 20, 2023
1 parent 0510d54 commit dc47087
Show file tree
Hide file tree
Showing 8 changed files with 40 additions and 11 deletions.
3 changes: 3 additions & 0 deletions be/src/vec/functions/function_timestamp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ struct StrToDate {
static DataTypes get_variadic_argument_types() { return {}; }

static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
//TODO: it doesn't matter now. maybe sometime we should find the function signature with return_type together
return make_nullable(std::make_shared<DataTypeDateTime>());
}

Expand Down Expand Up @@ -115,6 +116,8 @@ struct StrToDate {
auto& rdata = specific_char_column->get_chars();
auto& roffsets = specific_char_column->get_offsets();

// Because of we cant distinguish by return_type when we find function. so the return_type may NOT be same with real return type
// which decided by FE. that's found by which.
ColumnPtr res = nullptr;
WhichDataType which(remove_nullable(block.get_by_position(result).type));
if (which.is_date_time_v2()) {
Expand Down
24 changes: 19 additions & 5 deletions be/src/vec/runtime/vdatetime_value.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2300,14 +2300,28 @@ bool DateV2Value<T>::from_date_format_str(const char* format, int format_len, co
break;
// Micro second
case 'f':
tmp = val + min(6, val_end - val);
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
tmp = val;
// when there's still something to the end, fix the scale of ms.
while (tmp < val_end && isdigit(*tmp)) {
tmp++;
}

if (tmp - val > 6) {
const char* tmp2 = val + 6;
if (!str_to_int64(val, &tmp2, &int_value)) {
return false;
}
} else {
if (!str_to_int64(val, &tmp, &int_value)) {
return false;
}
}
if constexpr (is_datetime) {
microsecond = int_value * int_exp10(6 - min(6, tmp - val));
frac_part_used = true;
}
microsecond = int_value * int_exp10(6 - min(6, val_end - val));
val = tmp;
time_part_used = true;
frac_part_used = true;
break;
// AM/PM
case 'p':
Expand Down
10 changes: 7 additions & 3 deletions .../src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeType;
import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;

Expand All @@ -44,8 +44,9 @@ public class StrToDate extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable {

public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateTimeType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT),
FunctionSignature.ret(DateTimeType.INSTANCE).args(StringType.INSTANCE, StringType.INSTANCE)
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
VarcharType.SYSTEM_DEFAULT),
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(StringType.INSTANCE, StringType.INSTANCE)
);

/**
Expand Down Expand Up @@ -86,6 +87,9 @@ public FunctionSignature computeSignature(FunctionSignature signature) {
if (getArgument(1) instanceof StringLikeLiteral) {
if (DateLiteral.hasTimePart(((StringLikeLiteral) getArgument(1)).getStringValue())) {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
if (returnType.isDateTimeV2Type()) {
returnType = DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR);
}
} else {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATE));
}
Expand Down
4 changes: 2 additions & 2 deletions gensrc/script/doris_builtins_functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -939,8 +939,6 @@
[['datediff'], 'INT', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],
[['timediff'], 'TIME', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],

[['str_to_date'], 'DATETIME', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['str_to_date'], 'DATETIME', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIME', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATE', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date'], 'DATE', ['DATETIME'], 'ALWAYS_NULLABLE'],
Expand Down Expand Up @@ -1104,6 +1102,8 @@
[['datediff'], 'INT', ['DATEV2', 'DATEV2'], ''],
[['timediff'], 'TIMEV2', ['DATEV2', 'DATEV2'], ''],

[['str_to_date'], 'DATETIMEV2', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['str_to_date'], 'DATETIMEV2', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIMEV2', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATEV2', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date', 'datev2', 'to_datev2'], 'DATEV2', ['DATETIMEV2'], 'ALWAYS_NULLABLE'],
Expand Down
5 changes: 4 additions & 1 deletion regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,10 @@ February
2014-12-21T12:34:56

-- !sql --
2014-12-21T12:34:56
2014-12-21T12:34:56.789

-- !sql --
2023-07-05T02:09:55.880

-- !sql --
2004-10-18
Expand Down
3 changes: 3 additions & 0 deletions regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,9 @@ February
-- !sql --
2014-12-21T12:34:56

-- !sql --
2023-07-05T02:09:55

-- !sql --
2004-10-18

Expand Down
1 change: 1 addition & 0 deletions regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from ${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d %H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d %h:%i:%s.%f %p'); """
qt_sql """ select str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """
Expand Down
1 change: 1 addition & 0 deletions regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from ${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d %H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d %h:%i:%s.%f %p'); """
qt_sql """ select str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """
Expand Down

0 comments on commit dc47087

Please sign in to comment.