Skip to content

Commit

Permalink
[fix](date_function) fix str_to_date function return wrong microsecon…
Browse files Browse the repository at this point in the history
…d issue (#47252)
  • Loading branch information
Yulei-Yang authored Jan 24, 2025
1 parent ffb9c86 commit 8925a39
Show file tree
Hide file tree
Showing 7 changed files with 106 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public class DateLiteral extends LiteralExpr {
private static Map<String, Integer> WEEK_DAY_NAME_DICT = Maps.newHashMap();
private static Set<Character> TIME_PART_SET = Sets.newHashSet();
private static final int[] DAYS_IN_MONTH = new int[] {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
private static String MICRO_SECOND_FORMATTER = "%f";
private static final WeekFields weekFields = WeekFields.of(DayOfWeek.SUNDAY, 7);

static {
Expand Down Expand Up @@ -717,7 +718,7 @@ public String getStringValue() {
int scale = ((ScalarType) type).getScalarScale();
long scaledMicroseconds = (long) (microsecond / SCALE_FACTORS[scale]);

if (scaledMicroseconds != 0) {
if (scale > 0) {
dateTimeChars[19] = '.';
fillPaddedValue(dateTimeChars, 20, (int) scaledMicroseconds, scale);
return new String(dateTimeChars, 0, 20 + scale);
Expand Down Expand Up @@ -1062,6 +1063,10 @@ public static boolean hasTimePart(String format) {
return format.chars().anyMatch(c -> TIME_PART_SET.contains((char) c));
}

public static boolean hasMicroSecondPart(String format) {
return format.indexOf(MICRO_SECOND_FORMATTER) != -1;
}

// Return the date stored in the dateliteral as pattern format.
// eg : "%Y-%m-%d" or "%Y-%m-%d %H:%i:%s"
public String dateFormat(String pattern) throws AnalysisException {
Expand Down Expand Up @@ -1604,6 +1609,9 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr
case 'T':
partUsed |= timePart;
break;
case 'f':
partUsed |= fracPart;
break;
default:
break;
}
Expand Down Expand Up @@ -1665,7 +1673,7 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr

// Compute timestamp type
if ((partUsed & datePart) != 0) { // Ymd part only
if ((partUsed & fracPart) != 0) {
if (hasMicroSecondPart(format)) {
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
} else if ((partUsed & timePart) != 0) {
this.type = ScalarType.getDefaultDateType(Type.DATETIME);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1993,7 +1993,11 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
Expr child1Result = getChild(1).getResultValue(false);
if (child1Result instanceof StringLiteral) {
if (DateLiteral.hasTimePart(child1Result.getStringValue())) {
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
if (DateLiteral.hasMicroSecondPart(child1Result.getStringValue())) {
this.type = Type.DATETIMEV2_WITH_MAX_SCALAR;
} else {
this.type = Type.DEFAULT_DATETIMEV2;
}
} else {
this.type = Type.DATEV2;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -628,8 +628,10 @@ public static Expression strToDate(StringLikeLiteral str, StringLikeLiteral form
if (org.apache.doris.analysis.DateLiteral.hasTimePart(format.getStringValue())) {
DataType returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
if (returnType instanceof DateTimeV2Type) {
boolean hasMicroPart = org.apache.doris.analysis.DateLiteral
.hasMicroSecondPart(format.getStringValue());
return DateTimeV2Literal.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue())
.toFormatter(), str.getValue()));
.toFormatter(), str.getValue()), hasMicroPart ? 6 : 0);
} else {
return DateTimeLiteral.fromJavaDateType(DateUtils.getTime(DateUtils.formatBuilder(format.getValue())
.toFormatter(), str.getValue()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ public FunctionSignature computeSignature(FunctionSignature signature) {
if (getArgument(1) instanceof StringLikeLiteral) {
if (DateLiteral.hasTimePart(((StringLikeLiteral) getArgument(1)).getStringValue())) {
returnType = DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
if (returnType.isDateTimeV2Type()) {
if (returnType.isDateTimeV2Type()
&& DateLiteral.hasMicroSecondPart(((StringLikeLiteral) getArgument(1)).getStringValue())) {
returnType = DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR);
}
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]

-- !select --
["2022-12-02 22:23:25", "2022-12-02 22:23:23.998"]
["2022-12-02 22:23:25.000", "2022-12-02 22:23:23.998"]

-- !select --
[]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
order_qt_date4 """ select * from TEST_DATE where (T1 > '2022-01-21 00:00:00' and T1 < '2022-01-22 00:00:00') or (T1 > '2022-01-20 00:00:00' and T1 < '2022-01-23 00:00:00'); """
order_qt_date5 """ select * from TEST_DATE where T1 < '2022-01-22 00:00:00' or T1 = '2022-01-21 05:23:01'; """
order_qt_date6 """ select * from TEST_DATE where (T1 < '2022-01-22 00:00:00' or T1 > '2022-01-20 00:00:00') and (T1 < '2022-01-23 00:00:00' or T1 > '2022-01-19 00:00:00'); """
order_qt_date7 """select * from TEST_TIMESTAMP where T2 < str_to_date('2020-12-21 12:34:56', '%Y-%m-%d %H:%i:%s');"""
order_qt_date7 """select * from TEST_TIMESTAMP where T2 < str_to_date('2020-12-21 12:34:56', '%Y-%m-%d %H:%i:%s.%f');"""

// test nvl
explain {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_date_function_v2") {
sql """
admin set frontend config ("enable_date_conversion"="true");
"""
sql """SET enable_nereids_planner=true"""

def tableName = "test_date_function_v2"

sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName} (
`id` INT,
`name` varchar(32),
`dt` varchar(32)
) ENGINE=OLAP
UNIQUE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
)
"""
sql """ insert into ${tableName} values (3, 'Carl','2024-12-29 10:11:12') """
def result1 = try_sql """
select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s') as string) from ${tableName};
"""
assertEquals(result1[0][0], "2024-12-29 10:11:12");

def result2 = try_sql """
select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s.%f') as string) from ${tableName};
"""
assertEquals(result2[0][0], "2024-12-29 10:11:12.000000");

def result3 = try_sql """
select cast(str_to_date("2025-01-17 11:59:30", '%Y-%m-%d %H:%i:%s') as string);
"""
assertEquals(result3[0][0], "2025-01-17 11:59:30");

def result4 = try_sql """
select cast(str_to_date("2025-01-17 11:59:30", '%Y-%m-%d %H:%i:%s.%f') as string);
"""
assertEquals(result4[0][0], "2025-01-17 11:59:30.000000");

// test legacy planner
sql """SET enable_nereids_planner=false"""
def result5 = try_sql """
select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s') as string) from ${tableName};
"""
assertEquals(result5[0][0], "2024-12-29 10:11:12");

result5 = try_sql """
select cast(str_to_date(dt, '%Y-%m-%d %H:%i:%s.%f') as string) from ${tableName};
"""
assertEquals(result5[0][0], "2024-12-29 10:11:12.000000");

result5 = try_sql """
select cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s') as string);
"""
assertEquals(result5[0][0], "2025-01-17 11:59:30");

result5 = try_sql """
select cast(str_to_date('2025-01-17 11:59:30', '%Y-%m-%d %H:%i:%s.%f') as string);
"""
assertEquals(result5[0][0], "2025-01-17 11:59:30.000000");


sql """ drop table ${tableName} """
}

0 comments on commit 8925a39

Please sign in to comment.