From 142d0a21f30ba73361df5a3a0a023c4b77fd00a3 Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Thu, 1 Jun 2023 11:29:29 -0700 Subject: [PATCH 1/8] Check custom formats for characters Signed-off-by: Guian Gumpac --- .../data/type/OpenSearchDateType.java | 84 +++++++++++++--- .../value/OpenSearchExprValueFactory.java | 96 ++++++++++++------- 2 files changed, 134 insertions(+), 46 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 3554a5b2b4..1314e02aae 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import java.util.List; +import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; @@ -27,6 +28,10 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); + public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of( + FormatNames.EPOCH_MILLIS, + FormatNames.EPOCH_SECOND + ); public static final List SUPPORTED_NAMED_DATETIME_FORMATS = List.of( FormatNames.ISO8601, FormatNames.EPOCH_MILLIS, @@ -165,6 +170,20 @@ public List getAllNamedFormatters() { .map(DateFormatter::forPattern).collect(Collectors.toList()); } + /** + * Retrieves a list of numeric formatters that format for dates. + * + * @return a list of DateFormatters that can be used to parse a Date. + */ + public List getNumericNamedFormatters() { + return getFormatList().stream() + .filter(formatString -> { + FormatNames namedFormat = FormatNames.forName(formatString); + return namedFormat == null ? false : SUPPORTED_NAMED_NUMERIC_FORMATS.contains(namedFormat); + }) + .map(DateFormatter::forPattern).collect(Collectors.toList()); + } + /** * Retrieves a list of custom formatters defined by the user. * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. @@ -184,7 +203,7 @@ public List getDateNamedFormatters() { return getFormatList().stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); + return namedFormat == null ? false : SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } @@ -198,38 +217,75 @@ public List getTimeNamedFormatters() { return getFormatList().stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); + return namedFormat == null ? false : SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } - private ExprCoreType getExprTypeFromFormatString(String formatString) { - if (formatString.isEmpty()) { - // FOLLOW-UP: check the default formatter - and set it here instead - // of assuming that the default is always a timestamp - return TIMESTAMP; + private ExprCoreType getExprTypeFromCustomFormatString(List formatters) { + // Characters from + // https://opensearch.org/docs/latest/field-types/supported-field-types/date/#built-in-formats + Pattern timeFormatChars = Pattern.compile("[HmsSZ]"); + Pattern dateFormatChars = Pattern.compile("[yYMwdDe]"); + boolean isTime = false; + boolean isDate = false; + + for (DateFormatter formatter: formatters) { + if (timeFormatChars.matcher(formatter.pattern()).find()) { + isTime = true; + } + if (dateFormatChars.matcher(formatter.pattern()).find()) { + isDate = true; + } + if ((isDate && isTime)) { + return TIMESTAMP; + } + } + if (isTime) { + return TIME; + } + if (isDate) { + return DATE; } - List namedFormatters = getAllNamedFormatters(); + // Default type + return TIMESTAMP; + } - if (namedFormatters.isEmpty()) { + private ExprCoreType getExprTypeFromFormatString(String formatString) { + List timeFormatters = getTimeNamedFormatters(); + List dateFormatters = getDateNamedFormatters(); + List customFormatters = getAllCustomFormatters(); + List numericFormatters = getNumericNamedFormatters(); + + if (formatString.isEmpty() + || (!timeFormatters.isEmpty() && !dateFormatters.isEmpty()) + || !numericFormatters.isEmpty()) { + // FOLLOW-UP: check the default formatter - and set it here instead + // of assuming that the default is always a timestamp return TIMESTAMP; } - if (!getAllCustomFormatters().isEmpty()) { - // FOLLOW-UP: support custom format in - return TIMESTAMP; + if (!customFormatters.isEmpty()) { + ExprCoreType customFormatType = getExprTypeFromCustomFormatString(customFormatters); + if (!timeFormatters.isEmpty() && customFormatType == TIME) { + return TIME; + } + if (!dateFormatters.isEmpty() && customFormatType == DATE) { + return DATE; + } + return customFormatType; } // if there is nothing in the dateformatter that accepts a year/month/day, then // we can assume the type is strictly a Time object - if (namedFormatters.size() == getTimeNamedFormatters().size()) { + if (!timeFormatters.isEmpty() && dateFormatters.isEmpty()) { return TIME; } // if there is nothing in the dateformatter that accepts a hour/minute/second, then // we can assume the type is strictly a Date object - if (namedFormatters.size() == getDateNamedFormatters().size()) { + if (!dateFormatters.isEmpty() && timeFormatters.isEmpty() ) { return DATE; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index abad197bd4..847826b4f9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -33,6 +33,7 @@ import java.time.LocalTime; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; @@ -225,38 +226,48 @@ private Optional type(String field) { */ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType) { Instant parsed = null; - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { + TemporalAccessor accessor = null; + ZonedDateTime zonedDateTime = null; + List formatters = dateType.getAllNamedFormatters(); + formatters.addAll(dateType.getAllCustomFormatters()); + + for (DateFormatter formatter : formatters) { try { - TemporalAccessor accessor = formatter.parse(value); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + accessor = formatter.parse(value); + zonedDateTime = DateFormatters.from(accessor); // remove the Zone parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); + return new ExprTimestampValue(parsed); } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } - // FOLLOW-UP PR: Check custom formatters too - - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0 - || dateType.getAllCustomFormatters().size() > 0) { - try { - parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); - } catch (DateTimeParseException e) { - // ignored - } - } - - if (parsed == null) { - // otherwise, throw an error that no formatters worked - throw new IllegalArgumentException( - String.format( - "Construct ExprTimestampValue from \"%s\" failed, unsupported date format.", value) - ); +// for (DateFormatter formatter : dateType.getAllCustomFormatters()) { +// try { +// accessor = formatter.parse(value); +// zonedDateTime = DateFormatters.from(accessor); +// // remove the Zone +// parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); +// return new ExprTimestampValue(parsed); +// } catch (DateTimeParseException ignored) { +// // nothing to do, try another format +// } +// } + + // if no formatters are available, try the default formatter + try { + parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); + return new ExprTimestampValue(parsed); + } catch (DateTimeParseException e) { + // ignored } - return new ExprTimestampValue(parsed); + // otherwise, throw an error that no formatters worked + throw new IllegalArgumentException( + String.format( + "Construct ExprTimestampValue from \"%s\" failed, unsupported date format.", value) + ); } /** @@ -278,15 +289,25 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { } } - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0) { + for (DateFormatter formatter : dateType.getAllCustomFormatters()) { try { + TemporalAccessor accessor = formatter.parse(value); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); return new ExprTimeValue( - DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); - } catch (DateTimeParseException e) { - // ignored + zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalTime()); + } catch (DateTimeParseException ignored) { + // nothing to do, try another format } } + + // if no formatters are available, try the default formatter + try { + return new ExprTimeValue( + DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); + } catch (DateTimeParseException e) { + // ignored + } + throw new IllegalArgumentException("Construct ExprTimeValue from \"" + value + "\" failed, unsupported time format."); } @@ -311,15 +332,26 @@ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { } } - // if no named formatters are available, use the default - if (dateType.getAllNamedFormatters().size() == 0) { + for (DateFormatter formatter : dateType.getAllCustomFormatters()) { try { + TemporalAccessor accessor = formatter.parse(value); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); + // return the first matching formatter as a date without timezone return new ExprDateValue( - DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); - } catch (DateTimeParseException e) { - // ignored + zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalDate()); + } catch (DateTimeParseException ignored) { + // nothing to do, try another format } } + + // if no formatters are available, use the default formatter + try { + return new ExprDateValue( + DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); + } catch (DateTimeParseException e) { + // ignored + } + throw new IllegalArgumentException("Construct ExprDateValue from \"" + value + "\" failed, unsupported date format."); } From 819e4467d261747980e872cbeee6595e37b36ad8 Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Thu, 1 Jun 2023 14:30:10 -0700 Subject: [PATCH 2/8] Removed duplicated code Signed-off-by: Guian Gumpac --- .../value/OpenSearchExprValueFactory.java | 51 ++++--------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 847826b4f9..247d151556 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -226,15 +226,13 @@ private Optional type(String field) { */ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType) { Instant parsed = null; - TemporalAccessor accessor = null; - ZonedDateTime zonedDateTime = null; List formatters = dateType.getAllNamedFormatters(); formatters.addAll(dateType.getAllCustomFormatters()); for (DateFormatter formatter : formatters) { try { - accessor = formatter.parse(value); - zonedDateTime = DateFormatters.from(accessor); + TemporalAccessor accessor = formatter.parse(value); + ZonedDateTime zonedDateTime = DateFormatters.from(accessor); // remove the Zone parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); return new ExprTimestampValue(parsed); @@ -243,18 +241,6 @@ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType } } -// for (DateFormatter formatter : dateType.getAllCustomFormatters()) { -// try { -// accessor = formatter.parse(value); -// zonedDateTime = DateFormatters.from(accessor); -// // remove the Zone -// parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); -// return new ExprTimestampValue(parsed); -// } catch (DateTimeParseException ignored) { -// // nothing to do, try another format -// } -// } - // if no formatters are available, try the default formatter try { parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); @@ -278,24 +264,16 @@ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType * @return time without timezone */ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { - try { - TemporalAccessor accessor = formatter.parse(value); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - return new ExprTimeValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalTime()); - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } + List formatters = dateType.getAllNamedFormatters(); + formatters.addAll(dateType.getAllCustomFormatters()); - for (DateFormatter formatter : dateType.getAllCustomFormatters()) { + for (DateFormatter formatter : formatters) { try { TemporalAccessor accessor = formatter.parse(value); ZonedDateTime zonedDateTime = DateFormatters.from(accessor); return new ExprTimeValue( zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalTime()); - } catch (DateTimeParseException ignored) { + } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } @@ -320,26 +298,17 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { * @return date without timezone */ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { - for (DateFormatter formatter : dateType.getAllNamedFormatters()) { - try { - TemporalAccessor accessor = formatter.parse(value); - ZonedDateTime zonedDateTime = DateFormatters.from(accessor); - // return the first matching formatter as a date without timezone - return new ExprDateValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalDate()); - } catch (IllegalArgumentException ignored) { - // nothing to do, try another format - } - } + List formatters = dateType.getAllNamedFormatters(); + formatters.addAll(dateType.getAllCustomFormatters()); - for (DateFormatter formatter : dateType.getAllCustomFormatters()) { + for (DateFormatter formatter : formatters) { try { TemporalAccessor accessor = formatter.parse(value); ZonedDateTime zonedDateTime = DateFormatters.from(accessor); // return the first matching formatter as a date without timezone return new ExprDateValue( zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalDate()); - } catch (DateTimeParseException ignored) { + } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } From 4130109ddba87102545b04bcafa710a51efc51c6 Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Fri, 2 Jun 2023 17:09:33 -0700 Subject: [PATCH 3/8] Reworked checking for exprcoretype Signed-off-by: Guian Gumpac --- .../src/test/resources/date_formats.json | 4 +- .../date_formats_index_mapping.json | 14 +++++-- .../data/type/OpenSearchDateType.java | 41 +++++++++---------- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index cc694930e9..85059a7b3f 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12"} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12"} diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 938f598d0b..37bd62545d 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -289,9 +289,9 @@ "type" : "date", "format": "yyyy-MM-dd" }, - "HH:mm:ss" : { + "custom_time" : { "type" : "date", - "format": "HH:mm:ss" + "format": "hh:mm:ss a" }, "yyyy-MM-dd_OR_epoch_millis" : { "type" : "date", @@ -300,7 +300,15 @@ "hour_minute_second_OR_t_time" : { "type" : "date", "format": "hour_minute_second||t_time" + }, + "custom_timestamp" : { + "type" : "date", + "format": "yyyy-MM-dd hh:mm:ss ---- a" + }, + "custom_date_or_date" : { + "type" : "date", + "format": "yyyy-MM-dd||date" } } } -} \ No newline at end of file +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 1314e02aae..cf2c0191bd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,8 +11,9 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; import java.util.List; -import java.util.regex.Pattern; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; @@ -222,33 +223,29 @@ public List getTimeNamedFormatters() { .map(DateFormatter::forPattern).collect(Collectors.toList()); } - private ExprCoreType getExprTypeFromCustomFormatString(List formatters) { - // Characters from - // https://opensearch.org/docs/latest/field-types/supported-field-types/date/#built-in-formats - Pattern timeFormatChars = Pattern.compile("[HmsSZ]"); - Pattern dateFormatChars = Pattern.compile("[yYMwdDe]"); - boolean isTime = false; + private ExprCoreType getExprTypeFromCustomFormatString(String formatString) { + List formats = Arrays.asList(formatString.split("\\|\\|")); boolean isDate = false; + boolean isTime = false; - for (DateFormatter formatter: formatters) { - if (timeFormatChars.matcher(formatter.pattern()).find()) { - isTime = true; - } - if (dateFormatChars.matcher(formatter.pattern()).find()) { - isDate = true; - } - if ((isDate && isTime)) { - return TIMESTAMP; - } + try { + isDate = formats.stream() + .anyMatch(format -> DateTimeFormatter.ofPattern(format).toString().toLowerCase().contains("year")); + } catch (IllegalArgumentException ignore) { } - if (isTime) { - return TIME; + try { + isTime = formats.stream() + .anyMatch(format -> DateTimeFormatter.ofPattern(formatString).toString().toLowerCase().contains("minute")); + } catch (IllegalArgumentException ignore) { } - if (isDate) { + + if (isDate && !isTime) { return DATE; } + if (isTime && !isDate) { + return TIME; + } - // Default type return TIMESTAMP; } @@ -267,7 +264,7 @@ private ExprCoreType getExprTypeFromFormatString(String formatString) { } if (!customFormatters.isEmpty()) { - ExprCoreType customFormatType = getExprTypeFromCustomFormatString(customFormatters); + ExprCoreType customFormatType = getExprTypeFromCustomFormatString(formatString); if (!timeFormatters.isEmpty() && customFormatType == TIME) { return TIME; } From 69e9cf619cbf7437aefc8df9c5493146b0e98ece Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Mon, 5 Jun 2023 08:53:50 -0700 Subject: [PATCH 4/8] Changed check for time Signed-off-by: Guian Gumpac --- .../sql/opensearch/data/type/OpenSearchDateType.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index cf2c0191bd..65801ac13e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -229,13 +229,15 @@ private ExprCoreType getExprTypeFromCustomFormatString(String formatString) { boolean isTime = false; try { + // Dates will always have "year" in the formatter string isDate = formats.stream() .anyMatch(format -> DateTimeFormatter.ofPattern(format).toString().toLowerCase().contains("year")); } catch (IllegalArgumentException ignore) { } try { + // Times will always have "hour" in the formatter string isTime = formats.stream() - .anyMatch(format -> DateTimeFormatter.ofPattern(formatString).toString().toLowerCase().contains("minute")); + .anyMatch(format -> DateTimeFormatter.ofPattern(format).toString().toLowerCase().contains("hour")); } catch (IllegalArgumentException ignore) { } From 3b2527c30f608b9c93314ea359ea8eda5bf915b2 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 28 Jun 2023 21:28:09 -0700 Subject: [PATCH 5/8] Rework processing custom and incomplete formats and add tests. Signed-off-by: Yury-Fridlyand --- .../opensearch/sql/sql/DateTimeFormatsIT.java | 33 +++ .../src/test/resources/date_formats.json | 4 +- .../date_formats_index_mapping.json | 20 ++ .../data/type/OpenSearchDateType.java | 163 ++++++++----- .../value/OpenSearchExprValueFactory.java | 20 +- .../data/type/OpenSearchDateTypeTest.java | 219 +++++++++++------- .../value/OpenSearchExprValueFactoryTest.java | 36 +-- 7 files changed, 326 insertions(+), 169 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 7cd95fb509..274cb4a413 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.util.Locale; +import lombok.SneakyThrows; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -56,6 +57,38 @@ public void testDateFormatsWithOr() throws IOException { rows("1984-04-12 09:07:42.000123456")); } + @Test + @SneakyThrows + public void testCustomFormats() { + String query = String.format("SELECT custom_time, custom_timestamp, custom_date_or_date," + + "custom_date_or_custom_time, custom_time_parser_check FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("custom_time", null, "time"), + schema("custom_timestamp", null, "timestamp"), + schema("custom_date_or_date", null, "date"), + schema("custom_date_or_custom_time", null, "timestamp"), + schema("custom_time_parser_check", null, "time")); + verifyDataRows(result, + rows("09:07:42", "1984-04-12 09:07:42", "1984-04-12", "1961-04-12 00:00:00", "23:44:36.321"), + rows("21:07:42", "1984-04-12 22:07:42", "1984-04-12", "1970-01-01 09:07:00", "09:01:16.542")); + } + + @Test + @SneakyThrows + public void testIncompleteFormats() { + String query = String.format("SELECT incomplete_1, incomplete_2, incorrect" + + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("incomplete_1", null, "keyword"), + schema("incomplete_2", null, "keyword"), + schema("incorrect", null, "keyword")); + verifyDataRows(result, + rows("1984", null, null), + rows("2012", null, null)); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index 85059a7b3f..c7a26d3f74 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incorrect" : null} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incorrect" : null} diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 37bd62545d..7902f9f24f 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -308,6 +308,26 @@ "custom_date_or_date" : { "type" : "date", "format": "yyyy-MM-dd||date" + }, + "custom_date_or_custom_time" : { + "type" : "date", + "format" : "yyyy-MM-dd || HH:mm:ss" + }, + "custom_time_parser_check" : { + "type" : "date", + "format" : "::: k-A || A " + }, + "incomplete_1" : { + "type" : "date", + "format" : "year" + }, + "incomplete_2" : { + "type" : "date", + "format" : "E-w" + }, + "incorrect" : { + "type" : "date", + "format" : "___" } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 65801ac13e..cde9b3472f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -8,15 +8,19 @@ import static org.opensearch.common.time.DateFormatter.splitCombinedPatterns; import static org.opensearch.common.time.DateFormatter.strip8Prefix; import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; +import java.time.LocalDateTime; +import java.time.temporal.TemporalAccessor; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -29,14 +33,15 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); + // numeric formats which support full datetime public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of( FormatNames.EPOCH_MILLIS, FormatNames.EPOCH_SECOND ); + + // list of named formats which support full datetime public static final List SUPPORTED_NAMED_DATETIME_FORMATS = List.of( FormatNames.ISO8601, - FormatNames.EPOCH_MILLIS, - FormatNames.EPOCH_SECOND, FormatNames.BASIC_DATE_TIME, FormatNames.BASIC_DATE_TIME_NO_MILLIS, FormatNames.BASIC_ORDINAL_DATE_TIME, @@ -83,16 +88,21 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_DATE, FormatNames.YEAR_MONTH_DAY, FormatNames.STRICT_YEAR_MONTH_DAY, - FormatNames.YEAR_MONTH, - FormatNames.STRICT_YEAR_MONTH, - FormatNames.YEAR, - FormatNames.STRICT_YEAR, FormatNames.ORDINAL_DATE, FormatNames.STRICT_ORDINAL_DATE, FormatNames.WEEK_DATE, FormatNames.STRICT_WEEK_DATE, FormatNames.WEEKYEAR_WEEK_DAY, - FormatNames.STRICT_WEEKYEAR_WEEK_DAY, + FormatNames.STRICT_WEEKYEAR_WEEK_DAY + ); + + // list of named formats which produce incomplete date, + // e.g. 1 or 2 are missing from tuple year/month/day + public static final List SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS = List.of( + FormatNames.YEAR_MONTH, + FormatNames.STRICT_YEAR_MONTH, + FormatNames.YEAR, + FormatNames.STRICT_YEAR, FormatNames.WEEK_YEAR, FormatNames.WEEK_YEAR_WEEK, FormatNames.STRICT_WEEKYEAR_WEEK, @@ -144,10 +154,10 @@ private OpenSearchDateType(ExprType exprType) { this.exprCoreType = (ExprCoreType) exprType; } - private OpenSearchDateType(String formatStringArg) { + private OpenSearchDateType(String format) { super(MappingType.Date); - this.formatString = formatStringArg; - this.exprCoreType = getExprTypeFromFormatString(formatStringArg); + this.formatString = format; + this.exprCoreType = getExprTypeFromFormatString(format); } /** @@ -156,11 +166,9 @@ private OpenSearchDateType(String formatStringArg) { */ private List getFormatList() { String format = strip8Prefix(formatString); - List patterns = splitCombinedPatterns(format); - return patterns; + return splitCombinedPatterns(format).stream().map(String::trim).collect(Collectors.toList()); } - /** * Retrieves a list of named OpenSearch formatters given by user mapping. * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. @@ -180,7 +188,7 @@ public List getNumericNamedFormatters() { return getFormatList().stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return namedFormat == null ? false : SUPPORTED_NAMED_NUMERIC_FORMATS.contains(namedFormat); + return namedFormat != null && SUPPORTED_NAMED_NUMERIC_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } @@ -191,8 +199,17 @@ public List getNumericNamedFormatters() { */ public List getAllCustomFormatters() { return getFormatList().stream() - .filter(formatString -> FormatNames.forName(formatString) == null) - .map(DateFormatter::forPattern).collect(Collectors.toList()); + .filter(format -> FormatNames.forName(format) == null) + .map(format -> { + try { + return DateFormatter.forPattern(format); + } catch (Exception ignored) { + // parsing failed + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } /** @@ -204,7 +221,7 @@ public List getDateNamedFormatters() { return getFormatList().stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return namedFormat == null ? false : SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); + return namedFormat != null && SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } @@ -218,78 +235,116 @@ public List getTimeNamedFormatters() { return getFormatList().stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); - return namedFormat == null ? false : SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); + return namedFormat != null && SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); + }) + .map(DateFormatter::forPattern).collect(Collectors.toList()); + } + + /** + * Retrieves a list of named formatters that format for DateTimes. + * + * @return a list of DateFormatters that can be used to parse a DateTime. + */ + public List getDateTimeNamedFormatters() { + return getFormatList().stream() + .filter(formatString -> { + FormatNames namedFormat = FormatNames.forName(formatString); + return namedFormat != null && SUPPORTED_NAMED_DATETIME_FORMATS.contains(namedFormat); }) .map(DateFormatter::forPattern).collect(Collectors.toList()); } - private ExprCoreType getExprTypeFromCustomFormatString(String formatString) { - List formats = Arrays.asList(formatString.split("\\|\\|")); + private ExprCoreType getExprTypeFromCustomFormats(List formats) { boolean isDate = false; boolean isTime = false; - try { - // Dates will always have "year" in the formatter string - isDate = formats.stream() - .anyMatch(format -> DateTimeFormatter.ofPattern(format).toString().toLowerCase().contains("year")); - } catch (IllegalArgumentException ignore) { - } - try { - // Times will always have "hour" in the formatter string - isTime = formats.stream() - .anyMatch(format -> DateTimeFormatter.ofPattern(format).toString().toLowerCase().contains("hour")); - } catch (IllegalArgumentException ignore) { + LocalDateTime sampleDateTime = LocalDateTime.now(); + // Unfortunately, there is no public API to get info from the formatter object, + // whether it parses date or time or datetime. The workaround is: + // Converting a sample DateTime object by each formatter to string and back. + // Double-converted sample will be also DateTime, but if formatter parses + // time part only, date part would be lost. And vice versa. + // This trick allows us to get matching data type for every custom formatter. + // Unfortunately, this involves parsing a string, once per query, per column, per format. + // Overhead performance is equal to parsing extra row of result set (extra doc). + // Could be cached in scope of #1783 https://github.com/opensearch-project/sql/issues/1783. + + for (var format : formats) { + TemporalAccessor ta = format.parse(format.format(sampleDateTime)); + LocalDateTime parsedSample = sampleDateTime; + try { + // TODO do we need withZoneSameInstant or withZoneSameLocal? + parsedSample = DateFormatters.from(ta).toLocalDateTime(); + } catch (Exception ignored) { + // Can't convert to a DateTime - format does not represent a complete date or time + continue; + } + + if (!isDate) { + isDate = parsedSample.toLocalDate().equals(sampleDateTime.toLocalDate()); + } + if (!isTime) { + // Second and Second fraction part are optional and may miss in some formats, trim it. + isTime = parsedSample.toLocalTime().withSecond(0).withNano(0).equals( + sampleDateTime.toLocalTime().withSecond(0).withNano(0)); + } + if (isDate && isTime) { + return TIMESTAMP; + } } - if (isDate && !isTime) { + if (isDate) { return DATE; } - if (isTime && !isDate) { + if (isTime) { return TIME; } - return TIMESTAMP; + // Incomplete formats: can't be converted to DATE nor TIME, for example `year` (year only) + return STRING; } private ExprCoreType getExprTypeFromFormatString(String formatString) { - List timeFormatters = getTimeNamedFormatters(); - List dateFormatters = getDateNamedFormatters(); - List customFormatters = getAllCustomFormatters(); + List datetimeFormatters = getDateTimeNamedFormatters(); List numericFormatters = getNumericNamedFormatters(); - if (formatString.isEmpty() - || (!timeFormatters.isEmpty() && !dateFormatters.isEmpty()) - || !numericFormatters.isEmpty()) { - // FOLLOW-UP: check the default formatter - and set it here instead - // of assuming that the default is always a timestamp + if (formatString.isEmpty() || !datetimeFormatters.isEmpty() || !numericFormatters.isEmpty()) { + return TIMESTAMP; + } + + List timeFormatters = getTimeNamedFormatters(); + List dateFormatters = getDateNamedFormatters(); + if (!timeFormatters.isEmpty() && !dateFormatters.isEmpty()) { return TIMESTAMP; } + List customFormatters = getAllCustomFormatters(); if (!customFormatters.isEmpty()) { - ExprCoreType customFormatType = getExprTypeFromCustomFormatString(formatString); - if (!timeFormatters.isEmpty() && customFormatType == TIME) { - return TIME; + ExprCoreType customFormatType = getExprTypeFromCustomFormats(customFormatters); + ExprCoreType combinedByDefaultFormats = customFormatType; + if (!dateFormatters.isEmpty()) { + combinedByDefaultFormats = DATE; } - if (!dateFormatters.isEmpty() && customFormatType == DATE) { - return DATE; + if (!timeFormatters.isEmpty()) { + combinedByDefaultFormats = TIME; } - return customFormatType; + return customFormatType == combinedByDefaultFormats ? customFormatType : TIMESTAMP; } // if there is nothing in the dateformatter that accepts a year/month/day, then // we can assume the type is strictly a Time object - if (!timeFormatters.isEmpty() && dateFormatters.isEmpty()) { + if (!timeFormatters.isEmpty()) { return TIME; } // if there is nothing in the dateformatter that accepts a hour/minute/second, then // we can assume the type is strictly a Date object - if (!dateFormatters.isEmpty() && timeFormatters.isEmpty() ) { + if (!dateFormatters.isEmpty()) { return DATE; } - // According to the user mapping, this field may contain a DATE or a TIME - return TIMESTAMP; + // Unknown or incorrect format provided + return STRING; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 247d151556..c66eda9e53 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -33,7 +33,6 @@ import java.time.LocalTime; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; @@ -125,14 +124,13 @@ public void extendTypeMapping(Map typeMapping) { .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Boolean), (c, dt) -> ExprBooleanValue.of(c.booleanValue())) //Handles the creation of DATE, TIME & DATETIME - .put(OpenSearchDateType.of(TIME), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(DATE), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(TIMESTAMP), - this::createOpenSearchDateType) - .put(OpenSearchDateType.of(DATETIME), - this::createOpenSearchDateType) + .put(OpenSearchDateType.of(TIME), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(DATE), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(TIMESTAMP), this::createOpenSearchDateType) + .put(OpenSearchDateType.of(DATETIME), this::createOpenSearchDateType) + // if mapping has a format which can't produce a DATE, TIME or DATETIME object, + // it returned as a string. + .put(OpenSearchDateType.of(STRING), this::createOpenSearchDateType) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), @@ -329,6 +327,10 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; ExprType returnFormat = dt.getExprType(); + if (returnFormat == STRING) { + // mapping has a format which can't be converted to DATE or TIME or DATETIME + return new ExprStringValue(value.stringValue()); + } if (value.isNumber()) { Instant epochMillis = Instant.ofEpochMilli(value.longValue()); if (returnFormat == TIME) { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index f0add5bcd9..744c51a2cc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -3,34 +3,37 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.opensearch.data.type; -import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNotSame; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assertions.fail; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_DATETIME_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_DATE_FORMATS; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS; +import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_NUMERIC_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_TIME_FORMATS; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.isDateTypeCompatible; +import com.google.common.collect.Lists; import java.util.EnumSet; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.common.time.FormatNames; -import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.data.type.ExprCoreType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchDateTypeTest { @@ -107,96 +110,138 @@ public void check_exprTypeName() { assertEquals(TIMESTAMP, datetimeDateType.getExprType()); } - @Test - public void checkSupportedFormatNamesCoverage() { - EnumSet allFormatNames = EnumSet.allOf(FormatNames.class); - allFormatNames.stream().forEach(formatName -> { - assertTrue( - SUPPORTED_NAMED_DATETIME_FORMATS.contains(formatName) - || SUPPORTED_NAMED_DATE_FORMATS.contains(formatName) - || SUPPORTED_NAMED_TIME_FORMATS.contains(formatName), - formatName + " not supported"); - }); + private static Stream getAllSupportedFormats() { + return EnumSet.allOf(FormatNames.class).stream().map(Arguments::of); } - @Test - public void checkTimestampFormatNames() { - SUPPORTED_NAMED_DATETIME_FORMATS.stream().forEach( - datetimeFormat -> { - String camelCaseName = datetimeFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == TIMESTAMP, camelCaseName - + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = datetimeFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == TIMESTAMP, snakeCaseName - + " does not format to a TIMESTAMP type, instead got " - + dateType.getExprType()); - } - } - ); + @ParameterizedTest + @MethodSource("getAllSupportedFormats") + public void check_supported_format_names_coverage(FormatNames formatName) { + assertTrue(SUPPORTED_NAMED_NUMERIC_FORMATS.contains(formatName) + || SUPPORTED_NAMED_DATETIME_FORMATS.contains(formatName) + || SUPPORTED_NAMED_DATE_FORMATS.contains(formatName) + || SUPPORTED_NAMED_TIME_FORMATS.contains(formatName) + || SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS.contains(formatName), + formatName + " not supported"); + } - // check the default format case - OpenSearchDateType dateType = OpenSearchDateType.of(""); - assertTrue(dateType.getExprType() == TIMESTAMP); + private static Stream getSupportedDatetimeFormats() { + return SUPPORTED_NAMED_DATETIME_FORMATS.stream().map(Arguments::of); } - @Test - public void checkDateFormatNames() { - SUPPORTED_NAMED_DATE_FORMATS.stream().forEach( - dateFormat -> { - String camelCaseName = dateFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == DATE, camelCaseName - + " does not format to a DATE type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = dateFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == DATE, snakeCaseName - + " does not format to a DATE type, instead got " - + dateType.getExprType()); - } - } - ); + @ParameterizedTest + @MethodSource("getSupportedDatetimeFormats") + public void check_datetime_format_names(FormatNames datetimeFormat) { + String camelCaseName = datetimeFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = + OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), TIMESTAMP, camelCaseName + + " does not format to a TIMESTAMP type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = datetimeFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), TIMESTAMP, snakeCaseName + + " does not format to a TIMESTAMP type, instead got " + dateType.getExprType()); + } else { + fail(); + } } - @Test - public void checkTimeFormatNames() { - SUPPORTED_NAMED_TIME_FORMATS.stream().forEach( - timeFormat -> { - String camelCaseName = timeFormat.getCamelCaseName(); - if (camelCaseName != null && !camelCaseName.isEmpty()) { - OpenSearchDateType dateType = - OpenSearchDateType.of(camelCaseName); - assertTrue(dateType.getExprType() == TIME, camelCaseName - + " does not format to a TIME type, instead got " - + dateType.getExprType()); - } - - String snakeCaseName = timeFormat.getSnakeCaseName(); - if (snakeCaseName != null && !snakeCaseName.isEmpty()) { - OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); - assertTrue(dateType.getExprType() == TIME, snakeCaseName - + " does not format to a TIME type, instead got " - + dateType.getExprType()); - } - } + private static Stream getSupportedDateFormats() { + return SUPPORTED_NAMED_DATE_FORMATS.stream().map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("getSupportedDateFormats") + public void check_date_format_names(FormatNames dateFormat) { + String camelCaseName = dateFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), DATE, camelCaseName + + " does not format to a DATE type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = dateFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), DATE, snakeCaseName + + " does not format to a DATE type, instead got " + dateType.getExprType()); + } else { + fail(); + } + } + + private static Stream getSupportedTimeFormats() { + return SUPPORTED_NAMED_TIME_FORMATS.stream().map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("getSupportedTimeFormats") + public void check_time_format_names(FormatNames timeFormat) { + String camelCaseName = timeFormat.getCamelCaseName(); + if (camelCaseName != null && !camelCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(camelCaseName); + assertSame(dateType.getExprType(), TIME, camelCaseName + + " does not format to a TIME type, instead got " + dateType.getExprType()); + } + + String snakeCaseName = timeFormat.getSnakeCaseName(); + if (snakeCaseName != null && !snakeCaseName.isEmpty()) { + OpenSearchDateType dateType = OpenSearchDateType.of(snakeCaseName); + assertSame(dateType.getExprType(), TIME, snakeCaseName + + " does not format to a TIME type, instead got " + dateType.getExprType()); + } else { + fail(); + } + } + + private static Stream get_format_combinations_for_test() { + return Stream.of( + Arguments.of(DATE, List.of("dd.MM.yyyy", "date"), "d && custom date"), + Arguments.of(TIME, List.of("time", "HH:mm"), "t && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "time"), "t && custom date"), + Arguments.of(TIMESTAMP, List.of("date", "HH:mm"), "d && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy HH:mm", "date_time"), "dt && custom datetime"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "date_time"), "dt && custom date"), + Arguments.of(TIMESTAMP, List.of("HH:mm", "date_time"), "dt && custom time"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "epoch_second"), "custom date && num"), + Arguments.of(TIMESTAMP, List.of("HH:mm", "epoch_second"), "custom time && num"), + Arguments.of(TIMESTAMP, List.of("date_time", "epoch_second"), "dt && num"), + Arguments.of(TIMESTAMP, List.of("date", "epoch_second"), "d && num"), + Arguments.of(TIMESTAMP, List.of("time", "epoch_second"), "t && num"), + Arguments.of(TIMESTAMP, List.of(""), "no formats given"), + Arguments.of(TIMESTAMP, List.of("time", "date"), "t && d"), + Arguments.of(TIMESTAMP, List.of("epoch_second"), "numeric"), + Arguments.of(TIME, List.of("time"), "t"), + Arguments.of(DATE, List.of("date"), "d"), + Arguments.of(TIMESTAMP, List.of("date_time"), "dt"), + Arguments.of(STRING, List.of("unknown"), "unknown/incorrect"), + Arguments.of(STRING, List.of("uuuu"), "incomplete"), + Arguments.of(STRING, List.of("E-w"), "incomplete"), + // E - day of week, w - week of year + Arguments.of(STRING, List.of("uuuu", "E-w"), "incomplete"), + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "HH:mm"), "custom date and time"), + // D - day of year, N - nano of day + Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy N", "uuuu:D:HH:mm"), "custom datetime"), + Arguments.of(DATE, List.of("dd.MM.yyyy", "uuuu:D"), "custom date"), + Arguments.of(TIME, List.of("HH:mm", "N"), "custom time") ); } + @ParameterizedTest(name = "[{index}] {2}") + @MethodSource("get_format_combinations_for_test") + public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( + ExprCoreType expected, List formats, String testName) { + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + formats = Lists.reverse(formats); + assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); + } + @Test - public void checkIfDateTypeCompatible() { + public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); assertFalse(isDateTypeCompatible(OpenSearchDataType.of( OpenSearchDataType.MappingType.Text))); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index a7e3531e8b..e8d0e98a30 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; @@ -85,6 +86,7 @@ class OpenSearchExprValueFactoryTest { .put("customFormatV", OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss")) .put("customAndEpochMillisV", OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss||epoch_millis")) + .put("incompleteFormatV", OpenSearchDateType.of("year")) .put("boolV", OpenSearchDataType.of(BOOLEAN)) .put("structV", OpenSearchDataType.of(STRUCT)) .put("structV.id", OpenSearchDataType.of(INTEGER)) @@ -125,17 +127,17 @@ public void constructNullValue() { public void iterateArrayValue() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\",\"bb\"]")).array(); - assertTrue(arrayIt.next().stringValue().equals("zz")); - assertTrue(arrayIt.next().stringValue().equals("bb")); - assertTrue(!arrayIt.hasNext()); + assertEquals("zz", arrayIt.next().stringValue()); + assertEquals("bb", arrayIt.next().stringValue()); + assertFalse(arrayIt.hasNext()); } @Test public void iterateArrayValueWithOneElement() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\"]")).array(); - assertTrue(arrayIt.next().stringValue().equals("zz")); - assertTrue(!arrayIt.hasNext()); + assertEquals("zz", arrayIt.next().stringValue()); + assertFalse(arrayIt.hasNext()); } @Test @@ -307,17 +309,15 @@ public void constructDatetime() { @Test public void constructDatetime_fromCustomFormat() { - // this is not the desirable behaviour - instead if accepts the default formatter assertEquals( new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("customFormatV", "2015-01-01 12:10:30")); + constructFromObject("customFormatV", "2015-01-01-12-10-30")); - // this should pass when custom formats are supported IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> constructFromObject("customFormatV", "2015-01-01-12-10-30")); + () -> constructFromObject("customFormatV", "2015-01-01 12-10-30")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01-12-10-30\" failed, " + "Construct ExprTimestampValue from \"2015-01-01 12-10-30\" failed, " + "unsupported date format.", exception.getMessage()); @@ -325,14 +325,9 @@ public void constructDatetime_fromCustomFormat() { new ExprDatetimeValue("2015-01-01 12:10:30"), constructFromObject("customAndEpochMillisV", "2015-01-01 12:10:30")); - // this should pass when custom formats are supported - exception = - assertThrows(IllegalArgumentException.class, - () -> constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30")); assertEquals( - "Construct ExprTimestampValue from \"2015-01-01-12-10-30\" failed, " - + "unsupported date format.", - exception.getMessage()); + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30")); } @Test @@ -389,6 +384,13 @@ public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { exception.getMessage()); } + @Test + public void constructDateFromIncompleteFormat() { + assertEquals( + new ExprStringValue("1984"), + constructFromObject("incompleteFormatV", "1984")); + } + @Test public void constructArray() { assertEquals( From a22122f8488251413c0f46f3b5199d3b0c0af0f4 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Wed, 5 Jul 2023 16:26:33 -0700 Subject: [PATCH 6/8] Values of incomplete and incorrect formats to be returned as `TIMESTAMP` instead of `STRING`. Signed-off-by: Yury-Fridlyand --- .../opensearch/sql/sql/DateTimeFormatsIT.java | 39 ++++++++++++++--- .../src/test/resources/date_formats.json | 4 +- .../date_formats_index_mapping.json | 30 ++++++++++++- .../data/type/OpenSearchDateType.java | 6 +-- .../value/OpenSearchExprValueFactory.java | 43 +++++++++++-------- 5 files changed, 92 insertions(+), 30 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index 274cb4a413..b131651228 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -74,6 +74,21 @@ public void testCustomFormats() { rows("21:07:42", "1984-04-12 22:07:42", "1984-04-12", "1970-01-01 09:07:00", "09:01:16.542")); } + @Test + @SneakyThrows + public void testCustomFormats2() { + String query = String.format("SELECT custom_no_delimiter_date, custom_no_delimiter_time," + + "custom_no_delimiter_ts FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("custom_no_delimiter_date", null, "date"), + schema("custom_no_delimiter_time", null, "time"), + schema("custom_no_delimiter_ts", null, "timestamp")); + verifyDataRows(result, + rows("1984-10-20", "10:20:30", "1984-10-20 15:35:48"), + rows("1961-04-12", "09:07:00", "1961-04-12 09:07:00")); + } + @Test @SneakyThrows public void testIncompleteFormats() { @@ -81,12 +96,26 @@ public void testIncompleteFormats() { + " FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); verifySchema(result, - schema("incomplete_1", null, "keyword"), - schema("incomplete_2", null, "keyword"), - schema("incorrect", null, "keyword")); + schema("incomplete_1", null, "timestamp"), + schema("incomplete_2", null, "timestamp"), + schema("incorrect", null, "timestamp")); + verifyDataRows(result, + rows("1984-01-01 00:00:00", null, null), + rows("2012-01-01 00:00:00", null, null)); + } + + @Test + @SneakyThrows + public void testNumericFormats() { + String query = String.format("SELECT epoch_sec, epoch_milli" + + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, + schema("epoch_sec", null, "timestamp"), + schema("epoch_milli", null, "timestamp")); verifyDataRows(result, - rows("1984", null, null), - rows("2012", null, null)); + rows("1970-01-01 00:00:42", "1970-01-01 00:00:00.042"), + rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5")); } protected JSONObject executeQuery(String query) throws IOException { diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index c7a26d3f74..13d46a0e8c 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incorrect" : null} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incorrect" : null} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 7902f9f24f..65811f8d9e 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -325,9 +325,37 @@ "type" : "date", "format" : "E-w" }, + "incomplete_custom_date" : { + "type" : "date", + "format" : "uuuu" + }, + "incomplete_custom_time" : { + "type" : "date", + "format" : "HH" + }, "incorrect" : { "type" : "date", - "format" : "___" + "format" : "'___'" + }, + "epoch_sec" : { + "type" : "date", + "format" : "epoch_second" + }, + "epoch_milli" : { + "type" : "date", + "format" : "epoch_millis" + }, + "custom_no_delimiter_date" : { + "type" : "date", + "format" : "uuuuMMdd" + }, + "custom_no_delimiter_time" : { + "type" : "date", + "format" : "HHmmss" + }, + "custom_no_delimiter_ts" : { + "type" : "date", + "format" : "uuuuMMddHHmmss" } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index cde9b3472f..1ff94c698a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -8,10 +8,8 @@ import static org.opensearch.common.time.DateFormatter.splitCombinedPatterns; import static org.opensearch.common.time.DateFormatter.strip8Prefix; import static org.opensearch.sql.data.type.ExprCoreType.DATE; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import java.time.LocalDateTime; import java.time.temporal.TemporalAccessor; @@ -301,7 +299,7 @@ private ExprCoreType getExprTypeFromCustomFormats(List formats) { } // Incomplete formats: can't be converted to DATE nor TIME, for example `year` (year only) - return STRING; + return TIMESTAMP; } private ExprCoreType getExprTypeFromFormatString(String formatString) { @@ -344,7 +342,7 @@ private ExprCoreType getExprTypeFromFormatString(String formatString) { } // Unknown or incorrect format provided - return STRING; + return TIMESTAMP; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index c66eda9e53..a415e2692d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -45,6 +45,7 @@ import lombok.Setter; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; +import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; import org.opensearch.sql.data.model.ExprCollectionValue; @@ -82,7 +83,7 @@ public class OpenSearchExprValueFactory { /** * Extend existing mapping by new data without overwrite. - * Called from aggregation only {@link AggregationQueryBuilder#buildTypeMapping}. + * Called from aggregation only {@see AggregationQueryBuilder#buildTypeMapping}. * @param typeMapping A data type mapping produced by aggregation. */ public void extendTypeMapping(Map typeMapping) { @@ -128,9 +129,6 @@ public void extendTypeMapping(Map typeMapping) { .put(OpenSearchDateType.of(DATE), this::createOpenSearchDateType) .put(OpenSearchDateType.of(TIMESTAMP), this::createOpenSearchDateType) .put(OpenSearchDateType.of(DATETIME), this::createOpenSearchDateType) - // if mapping has a format which can't produce a DATE, TIME or DATETIME object, - // it returned as a string. - .put(OpenSearchDateType.of(STRING), this::createOpenSearchDateType) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), @@ -232,7 +230,7 @@ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType TemporalAccessor accessor = formatter.parse(value); ZonedDateTime zonedDateTime = DateFormatters.from(accessor); // remove the Zone - parsed = zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toInstant(); + parsed = zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toInstant(); return new ExprTimestampValue(parsed); } catch (IllegalArgumentException ignored) { // nothing to do, try another format @@ -270,7 +268,7 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { TemporalAccessor accessor = formatter.parse(value); ZonedDateTime zonedDateTime = DateFormatters.from(accessor); return new ExprTimeValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalTime()); + zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalTime()); } catch (IllegalArgumentException ignored) { // nothing to do, try another format } @@ -305,7 +303,7 @@ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { ZonedDateTime zonedDateTime = DateFormatters.from(accessor); // return the first matching formatter as a date without timezone return new ExprDateValue( - zonedDateTime.withZoneSameLocal(ZoneId.of("Z")).toLocalDate()); + zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalDate()); } catch (IllegalArgumentException ignored) { // nothing to do, try another format } @@ -327,19 +325,28 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; ExprType returnFormat = dt.getExprType(); - if (returnFormat == STRING) { - // mapping has a format which can't be converted to DATE or TIME or DATETIME - return new ExprStringValue(value.stringValue()); - } if (value.isNumber()) { - Instant epochMillis = Instant.ofEpochMilli(value.longValue()); - if (returnFormat == TIME) { - return new ExprTimeValue(LocalTime.from(epochMillis.atZone(UTC_ZONE_ID))); - } - if (returnFormat == DATE) { - return new ExprDateValue(LocalDate.ofInstant(epochMillis, UTC_ZONE_ID)); + var numFormatters = dt.getNumericNamedFormatters(); + if (numFormatters.size() > 0) { + long epochMillis = 0; + if (numFormatters.contains(DateFormatter.forPattern( + FormatNames.EPOCH_MILLIS.getSnakeCaseName()))) { + // no CamelCase for `EPOCH_*` formats + epochMillis = value.longValue(); + } else /* EPOCH_SECOND */ { + epochMillis = value.longValue() * 1000; + } + Instant instant = Instant.ofEpochMilli(epochMillis); + if (returnFormat == TIME) { + return new ExprTimeValue(LocalTime.from(instant.atZone(UTC_ZONE_ID))); + } + if (returnFormat == DATE) { + return new ExprDateValue(LocalDate.ofInstant(instant, UTC_ZONE_ID)); + } + return new ExprTimestampValue(instant); + } else { + return parseTimestampString(value.stringValue(), dt); } - return new ExprTimestampValue(Instant.ofEpochMilli(value.longValue())); } if (value.isString()) { From 6d214a53bc925fe63fcefb7c3479ec141f4daa76 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 6 Jul 2023 18:48:35 -0700 Subject: [PATCH 7/8] Complete fix and update tests. Signed-off-by: Yury-Fridlyand --- .../opensearch/sql/sql/DateTimeFormatsIT.java | 14 +- .../data/type/OpenSearchDateType.java | 89 +++--- .../value/OpenSearchExprValueFactory.java | 32 +-- .../data/type/OpenSearchDateTypeTest.java | 101 ++++--- .../value/OpenSearchExprValueFactoryTest.java | 269 ++++++++++-------- 5 files changed, 273 insertions(+), 232 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index b131651228..fc05e502c5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -92,16 +92,18 @@ public void testCustomFormats2() { @Test @SneakyThrows public void testIncompleteFormats() { - String query = String.format("SELECT incomplete_1, incomplete_2, incorrect" - + " FROM %s", TEST_INDEX_DATE_FORMATS); + String query = String.format("SELECT incomplete_1, incomplete_2, incorrect," + + "incomplete_custom_time, incomplete_custom_date FROM %s", TEST_INDEX_DATE_FORMATS); JSONObject result = executeQuery(query); verifySchema(result, schema("incomplete_1", null, "timestamp"), - schema("incomplete_2", null, "timestamp"), - schema("incorrect", null, "timestamp")); + schema("incomplete_2", null, "date"), + schema("incorrect", null, "timestamp"), + schema("incomplete_custom_time", null, "time"), + schema("incomplete_custom_date", null, "date")); verifyDataRows(result, - rows("1984-01-01 00:00:00", null, null), - rows("2012-01-01 00:00:00", null, null)); + rows("1984-01-01 00:00:00", null, null, "10:00:00", "1999-01-01"), + rows("2012-01-01 00:00:00", null, null, "20:00:00", "3021-01-01")); } @Test diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 1ff94c698a..86d95348f4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -11,14 +11,11 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import java.time.LocalDateTime; -import java.time.temporal.TemporalAccessor; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; import org.opensearch.common.time.FormatNames; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -31,13 +28,13 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final OpenSearchDateType instance = new OpenSearchDateType(); - // numeric formats which support full datetime + /** Numeric formats which support full datetime. */ public static final List SUPPORTED_NAMED_NUMERIC_FORMATS = List.of( FormatNames.EPOCH_MILLIS, FormatNames.EPOCH_SECOND ); - // list of named formats which support full datetime + /** List of named formats which support full datetime. */ public static final List SUPPORTED_NAMED_DATETIME_FORMATS = List.of( FormatNames.ISO8601, FormatNames.BASIC_DATE_TIME, @@ -78,7 +75,7 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_WEEK_DATE_TIME_NO_MILLIS ); - // list of named formats that only support year/month/day + /** List of named formats that only support year/month/day. */ public static final List SUPPORTED_NAMED_DATE_FORMATS = List.of( FormatNames.BASIC_DATE, FormatNames.BASIC_ORDINAL_DATE, @@ -94,8 +91,8 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_WEEKYEAR_WEEK_DAY ); - // list of named formats which produce incomplete date, - // e.g. 1 or 2 are missing from tuple year/month/day + /** list of named formats which produce incomplete date, + * e.g. 1 or 2 are missing from tuple year/month/day. */ public static final List SUPPORTED_NAMED_INCOMPLETE_DATE_FORMATS = List.of( FormatNames.YEAR_MONTH, FormatNames.STRICT_YEAR_MONTH, @@ -108,7 +105,7 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_WEEKYEAR ); - // list of named formats that only support hour/minute/second + /** List of named formats that only support hour/minute/second. */ public static final List SUPPORTED_NAMED_TIME_FORMATS = List.of( FormatNames.BASIC_TIME, FormatNames.BASIC_TIME_NO_MILLIS, @@ -134,6 +131,11 @@ public class OpenSearchDateType extends OpenSearchDataType { FormatNames.STRICT_T_TIME_NO_MILLIS ); + /** Formatter symbols which used to format time or date correspondingly. + * {@link java.time.format.DateTimeFormatter}. */ + private static final String CUSTOM_FORMAT_TIME_SYMBOLS = "nNASsmHkKha"; + private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; + @EqualsAndHashCode.Exclude String formatString; @@ -179,7 +181,6 @@ public List getAllNamedFormatters() { /** * Retrieves a list of numeric formatters that format for dates. - * * @return a list of DateFormatters that can be used to parse a Date. */ public List getNumericNamedFormatters() { @@ -191,6 +192,26 @@ public List getNumericNamedFormatters() { .map(DateFormatter::forPattern).collect(Collectors.toList()); } + /** + * Retrieves a list of custom formats defined by the user. + * @return a list of formats as strings that can be used to parse a Date/Time/Timestamp. + */ + public List getAllCustomFormats() { + return getFormatList().stream() + .filter(format -> FormatNames.forName(format) == null) + .map(format -> { + try { + DateFormatter.forPattern(format); + return format; + } catch (Exception ignored) { + // parsing failed + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + /** * Retrieves a list of custom formatters defined by the user. * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. @@ -212,7 +233,6 @@ public List getAllCustomFormatters() { /** * Retrieves a list of named formatters that format for dates. - * * @return a list of DateFormatters that can be used to parse a Date. */ public List getDateNamedFormatters() { @@ -226,7 +246,6 @@ public List getDateNamedFormatters() { /** * Retrieves a list of named formatters that format for Times. - * * @return a list of DateFormatters that can be used to parse a Time. */ public List getTimeNamedFormatters() { @@ -240,7 +259,6 @@ public List getTimeNamedFormatters() { /** * Retrieves a list of named formatters that format for DateTimes. - * * @return a list of DateFormatters that can be used to parse a DateTime. */ public List getDateTimeNamedFormatters() { @@ -252,39 +270,26 @@ public List getDateTimeNamedFormatters() { .map(DateFormatter::forPattern).collect(Collectors.toList()); } - private ExprCoreType getExprTypeFromCustomFormats(List formats) { + private ExprCoreType getExprTypeFromCustomFormats(List formats) { boolean isDate = false; boolean isTime = false; - LocalDateTime sampleDateTime = LocalDateTime.now(); - // Unfortunately, there is no public API to get info from the formatter object, - // whether it parses date or time or datetime. The workaround is: - // Converting a sample DateTime object by each formatter to string and back. - // Double-converted sample will be also DateTime, but if formatter parses - // time part only, date part would be lost. And vice versa. - // This trick allows us to get matching data type for every custom formatter. - // Unfortunately, this involves parsing a string, once per query, per column, per format. - // Overhead performance is equal to parsing extra row of result set (extra doc). - // Could be cached in scope of #1783 https://github.com/opensearch-project/sql/issues/1783. - for (var format : formats) { - TemporalAccessor ta = format.parse(format.format(sampleDateTime)); - LocalDateTime parsedSample = sampleDateTime; - try { - // TODO do we need withZoneSameInstant or withZoneSameLocal? - parsedSample = DateFormatters.from(ta).toLocalDateTime(); - } catch (Exception ignored) { - // Can't convert to a DateTime - format does not represent a complete date or time - continue; + if (!isTime) { + for (var symbol : CUSTOM_FORMAT_TIME_SYMBOLS.toCharArray()) { + if (format.contains(String.valueOf(symbol))) { + isTime = true; + break; + } + } } - if (!isDate) { - isDate = parsedSample.toLocalDate().equals(sampleDateTime.toLocalDate()); - } - if (!isTime) { - // Second and Second fraction part are optional and may miss in some formats, trim it. - isTime = parsedSample.toLocalTime().withSecond(0).withNano(0).equals( - sampleDateTime.toLocalTime().withSecond(0).withNano(0)); + for (var symbol : CUSTOM_FORMAT_DATE_SYMBOLS.toCharArray()) { + if (format.contains(String.valueOf(symbol))) { + isDate = true; + break; + } + } } if (isDate && isTime) { return TIMESTAMP; @@ -298,7 +303,7 @@ private ExprCoreType getExprTypeFromCustomFormats(List formats) { return TIME; } - // Incomplete formats: can't be converted to DATE nor TIME, for example `year` (year only) + // Incomplete or incorrect formats: can't be converted to DATE nor TIME, for example `year` return TIMESTAMP; } @@ -316,7 +321,7 @@ private ExprCoreType getExprTypeFromFormatString(String formatString) { return TIMESTAMP; } - List customFormatters = getAllCustomFormatters(); + List customFormatters = getAllCustomFormats(); if (!customFormatters.isEmpty()) { ExprCoreType customFormatType = getExprTypeFromCustomFormats(customFormatters); ExprCoreType combinedByDefaultFormats = customFormatType; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index a415e2692d..e79bd37e66 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -61,6 +61,7 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; @@ -241,8 +242,7 @@ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType try { parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); return new ExprTimestampValue(parsed); - } catch (DateTimeParseException e) { - // ignored + } catch (DateTimeParseException ignored) { } // otherwise, throw an error that no formatters worked @@ -269,7 +269,7 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { ZonedDateTime zonedDateTime = DateFormatters.from(accessor); return new ExprTimeValue( zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalTime()); - } catch (IllegalArgumentException ignored) { + } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } @@ -278,8 +278,7 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { try { return new ExprTimeValue( DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); - } catch (DateTimeParseException e) { - // ignored + } catch (DateTimeParseException ignored) { } throw new IllegalArgumentException("Construct ExprTimeValue from \"" + value @@ -304,7 +303,7 @@ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { // return the first matching formatter as a date without timezone return new ExprDateValue( zonedDateTime.withZoneSameLocal(UTC_ZONE_ID).toLocalDate()); - } catch (IllegalArgumentException ignored) { + } catch (IllegalArgumentException ignored) { // nothing to do, try another format } } @@ -313,8 +312,7 @@ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { try { return new ExprDateValue( DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); - } catch (DateTimeParseException e) { - // ignored + } catch (DateTimeParseException ignored) { } throw new IllegalArgumentException("Construct ExprDateValue from \"" + value @@ -325,7 +323,7 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { OpenSearchDateType dt = (OpenSearchDateType) type; ExprType returnFormat = dt.getExprType(); - if (value.isNumber()) { + if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); if (numFormatters.size() > 0) { long epochMillis = 0; @@ -336,16 +334,14 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { } else /* EPOCH_SECOND */ { epochMillis = value.longValue() * 1000; } - Instant instant = Instant.ofEpochMilli(epochMillis); - if (returnFormat == TIME) { - return new ExprTimeValue(LocalTime.from(instant.atZone(UTC_ZONE_ID))); - } - if (returnFormat == DATE) { - return new ExprDateValue(LocalDate.ofInstant(instant, UTC_ZONE_ID)); - } - return new ExprTimestampValue(instant); + return new ExprTimestampValue(Instant.ofEpochMilli(epochMillis)); } else { - return parseTimestampString(value.stringValue(), dt); + // custom format + switch ((ExprCoreType) dt.getExprType()) { + case TIME: return parseTimeString(value.stringValue(), dt); + case DATE: return parseDateString(value.stringValue(), dt); + default: return parseTimestampString(value.stringValue(), dt); + } } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java index 744c51a2cc..13393da732 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateTypeTest.java @@ -5,6 +5,7 @@ package org.opensearch.sql.opensearch.data.type; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; @@ -12,7 +13,6 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.opensearch.data.type.OpenSearchDateType.SUPPORTED_NAMED_DATETIME_FORMATS; @@ -56,58 +56,66 @@ class OpenSearchDateTypeTest { @Test public void isCompatible() { - // timestamp types is compatible with all date-types - assertTrue(TIMESTAMP.isCompatible(defaultDateType)); - assertTrue(TIMESTAMP.isCompatible(dateDateType)); - assertTrue(TIMESTAMP.isCompatible(timeDateType)); - assertTrue(TIMESTAMP.isCompatible(datetimeDateType)); - - // datetime - assertFalse(DATETIME.isCompatible(defaultDateType)); - assertTrue(DATETIME.isCompatible(dateDateType)); - assertTrue(DATETIME.isCompatible(timeDateType)); - assertFalse(DATETIME.isCompatible(datetimeDateType)); - - // time type - assertFalse(TIME.isCompatible(defaultDateType)); - assertFalse(TIME.isCompatible(dateDateType)); - assertTrue(TIME.isCompatible(timeDateType)); - assertFalse(TIME.isCompatible(datetimeDateType)); - - // date type - assertFalse(DATE.isCompatible(defaultDateType)); - assertTrue(DATE.isCompatible(dateDateType)); - assertFalse(DATE.isCompatible(timeDateType)); - assertFalse(DATE.isCompatible(datetimeDateType)); + assertAll( + // timestamp types is compatible with all date-types + () -> assertTrue(TIMESTAMP.isCompatible(defaultDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(dateDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(timeDateType)), + () -> assertTrue(TIMESTAMP.isCompatible(datetimeDateType)), + + // datetime + () -> assertFalse(DATETIME.isCompatible(defaultDateType)), + () -> assertTrue(DATETIME.isCompatible(dateDateType)), + () -> assertTrue(DATETIME.isCompatible(timeDateType)), + () -> assertFalse(DATETIME.isCompatible(datetimeDateType)), + + // time type + () -> assertFalse(TIME.isCompatible(defaultDateType)), + () -> assertFalse(TIME.isCompatible(dateDateType)), + () -> assertTrue(TIME.isCompatible(timeDateType)), + () -> assertFalse(TIME.isCompatible(datetimeDateType)), + + // date type + () -> assertFalse(DATE.isCompatible(defaultDateType)), + () -> assertTrue(DATE.isCompatible(dateDateType)), + () -> assertFalse(DATE.isCompatible(timeDateType)), + () -> assertFalse(DATE.isCompatible(datetimeDateType)) + ); } // `typeName` and `legacyTypeName` return the same thing for date objects: // https://github.com/opensearch-project/sql/issues/1296 @Test public void check_typeName() { - // always use the MappingType of "DATE" - assertEquals("DATE", defaultDateType.typeName()); - assertEquals("DATE", timeDateType.typeName()); - assertEquals("DATE", dateDateType.typeName()); - assertEquals("DATE", datetimeDateType.typeName()); + assertAll( + // always use the MappingType of "DATE" + () -> assertEquals("DATE", defaultDateType.typeName()), + () -> assertEquals("DATE", timeDateType.typeName()), + () -> assertEquals("DATE", dateDateType.typeName()), + () -> assertEquals("DATE", datetimeDateType.typeName()) + ); } @Test public void check_legacyTypeName() { - // always use the legacy "DATE" type - assertEquals("DATE", defaultDateType.legacyTypeName()); - assertEquals("DATE", timeDateType.legacyTypeName()); - assertEquals("DATE", dateDateType.legacyTypeName()); - assertEquals("DATE", datetimeDateType.legacyTypeName()); + assertAll( + // always use the legacy "DATE" type + () -> assertEquals("DATE", defaultDateType.legacyTypeName()), + () -> assertEquals("DATE", timeDateType.legacyTypeName()), + () -> assertEquals("DATE", dateDateType.legacyTypeName()), + () -> assertEquals("DATE", datetimeDateType.legacyTypeName()) + ); } @Test public void check_exprTypeName() { - // exprType changes based on type (no datetime): - assertEquals(TIMESTAMP, defaultDateType.getExprType()); - assertEquals(TIME, timeDateType.getExprType()); - assertEquals(DATE, dateDateType.getExprType()); - assertEquals(TIMESTAMP, datetimeDateType.getExprType()); + assertAll( + // exprType changes based on type (no datetime): + () -> assertEquals(TIMESTAMP, defaultDateType.getExprType()), + () -> assertEquals(TIME, timeDateType.getExprType()), + () -> assertEquals(DATE, dateDateType.getExprType()), + () -> assertEquals(TIMESTAMP, datetimeDateType.getExprType()) + ); } private static Stream getAllSupportedFormats() { @@ -218,11 +226,13 @@ private static Stream get_format_combinations_for_test() { Arguments.of(TIME, List.of("time"), "t"), Arguments.of(DATE, List.of("date"), "d"), Arguments.of(TIMESTAMP, List.of("date_time"), "dt"), - Arguments.of(STRING, List.of("unknown"), "unknown/incorrect"), - Arguments.of(STRING, List.of("uuuu"), "incomplete"), - Arguments.of(STRING, List.of("E-w"), "incomplete"), + Arguments.of(TIMESTAMP, List.of("unknown"), "unknown/incorrect"), + Arguments.of(DATE, List.of("uuuu"), "incomplete date"), + Arguments.of(TIME, List.of("HH"), "incomplete time"), + Arguments.of(DATE, List.of("E-w"), "incomplete"), // E - day of week, w - week of year - Arguments.of(STRING, List.of("uuuu", "E-w"), "incomplete"), + Arguments.of(DATE, List.of("uuuu", "E-w"), "incomplete with year"), + Arguments.of(TIMESTAMP, List.of("---"), "incorrect"), Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy", "HH:mm"), "custom date and time"), // D - day of year, N - nano of day Arguments.of(TIMESTAMP, List.of("dd.MM.yyyy N", "uuuu:D:HH:mm"), "custom datetime"), @@ -240,6 +250,11 @@ public void check_ExprCoreType_of_combinations_of_custom_and_predefined_formats( assertEquals(expected, OpenSearchDateType.of(String.join(" || ", formats)).getExprType()); } + @Test + public void dont_use_incorrect_format_as_custom() { + assertEquals(0, OpenSearchDateType.of(" ").getAllCustomFormatters().size()); + } + @Test public void check_if_date_type_compatible() { assertTrue(isDateTypeCompatible(DATE)); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index e8d0e98a30..1a9bbfdaf6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -35,14 +35,11 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.utils.DateTimeUtils.UTC_ZONE_ID; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalTime; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -80,12 +77,16 @@ class OpenSearchExprValueFactoryTest { .put("dateStringV", OpenSearchDateType.of("date")) .put("timeStringV", OpenSearchDateType.of("time")) .put("epochMillisV", OpenSearchDateType.of("epoch_millis")) - .put("dateOrEpochMillisV", OpenSearchDateType.of("date_time_no_millis||epoch_millis")) - .put("timeNoMillisOrTimeV", OpenSearchDateType.of("time_no_millis||time")) - .put("dateOrOrdinalDateV", OpenSearchDateType.of("date||ordinal_date")) + .put("epochSecondV", OpenSearchDateType.of("epoch_second")) + .put("timeCustomV", OpenSearchDateType.of("HHmmss")) + .put("dateCustomV", OpenSearchDateType.of("uuuuMMdd")) + .put("dateTimeCustomV", OpenSearchDateType.of("uuuuMMddHHmmss")) + .put("dateOrEpochMillisV", OpenSearchDateType.of("date_time_no_millis || epoch_millis")) + .put("timeNoMillisOrTimeV", OpenSearchDateType.of("time_no_millis || time")) + .put("dateOrOrdinalDateV", OpenSearchDateType.of("date || ordinal_date")) .put("customFormatV", OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss")) .put("customAndEpochMillisV", - OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss||epoch_millis")) + OpenSearchDateType.of("yyyy-MM-dd-HH-mm-ss || epoch_millis")) .put("incompleteFormatV", OpenSearchDateType.of("year")) .put("boolV", OpenSearchDataType.of(BOOLEAN)) .put("structV", OpenSearchDataType.of(STRUCT)) @@ -118,26 +119,32 @@ class OpenSearchExprValueFactoryTest { @Test public void constructNullValue() { - assertEquals(nullValue(), tupleValue("{\"intV\":null}").get("intV")); - assertEquals(nullValue(), constructFromObject("intV", null)); - assertTrue(new OpenSearchJsonContent(null).isNull()); + assertAll( + () -> assertEquals(nullValue(), tupleValue("{\"intV\":null}").get("intV")), + () -> assertEquals(nullValue(), constructFromObject("intV", null)), + () -> assertTrue(new OpenSearchJsonContent(null).isNull()) + ); } @Test public void iterateArrayValue() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\",\"bb\"]")).array(); - assertEquals("zz", arrayIt.next().stringValue()); - assertEquals("bb", arrayIt.next().stringValue()); - assertFalse(arrayIt.hasNext()); + assertAll( + () -> assertEquals("zz", arrayIt.next().stringValue()), + () -> assertEquals("bb", arrayIt.next().stringValue()), + () -> assertFalse(arrayIt.hasNext()) + ); } @Test public void iterateArrayValueWithOneElement() throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); var arrayIt = new OpenSearchJsonContent(mapper.readTree("[\"zz\"]")).array(); - assertEquals("zz", arrayIt.next().stringValue()); - assertFalse(arrayIt.hasNext()); + assertAll( + () -> assertEquals("zz", arrayIt.next().stringValue()), + () -> assertFalse(arrayIt.hasNext()) + ); } @Test @@ -147,23 +154,29 @@ public void constructNullArrayValue() { @Test public void constructByte() { - assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")); - assertEquals(byteValue((byte) 1), constructFromObject("byteV", 1)); - assertEquals(byteValue((byte) 1), constructFromObject("byteV", "1.0")); + assertAll( + () -> assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")), + () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", 1)), + () -> assertEquals(byteValue((byte) 1), constructFromObject("byteV", "1.0")) + ); } @Test public void constructShort() { - assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")); - assertEquals(shortValue((short) 1), constructFromObject("shortV", 1)); - assertEquals(shortValue((short) 1), constructFromObject("shortV", "1.0")); + assertAll( + () -> assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")), + () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", 1)), + () -> assertEquals(shortValue((short) 1), constructFromObject("shortV", "1.0")) + ); } @Test public void constructInteger() { - assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")); - assertEquals(integerValue(1), constructFromObject("intV", 1)); - assertEquals(integerValue(1), constructFromObject("intV", "1.0")); + assertAll( + () -> assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")), + () -> assertEquals(integerValue(1), constructFromObject("intV", 1)), + () -> assertEquals(integerValue(1), constructFromObject("intV", "1.0")) + ); } @Test @@ -173,138 +186,148 @@ public void constructIntegerValueInStringValue() { @Test public void constructLong() { - assertEquals(longValue(1L), tupleValue("{\"longV\":1}").get("longV")); - assertEquals(longValue(1L), constructFromObject("longV", 1L)); - assertEquals(longValue(1L), constructFromObject("longV", "1.0")); + assertAll( + () -> assertEquals(longValue(1L), tupleValue("{\"longV\":1}").get("longV")), + () -> assertEquals(longValue(1L), constructFromObject("longV", 1L)), + () -> assertEquals(longValue(1L), constructFromObject("longV", "1.0")) + ); } @Test public void constructFloat() { - assertEquals(floatValue(1f), tupleValue("{\"floatV\":1.0}").get("floatV")); - assertEquals(floatValue(1f), constructFromObject("floatV", 1f)); + assertAll( + () -> assertEquals(floatValue(1f), tupleValue("{\"floatV\":1.0}").get("floatV")), + () -> assertEquals(floatValue(1f), constructFromObject("floatV", 1f)) + ); } @Test public void constructDouble() { - assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":1.0}").get("doubleV")); - assertEquals(doubleValue(1d), constructFromObject("doubleV", 1d)); + assertAll( + () -> assertEquals(doubleValue(1d), tupleValue("{\"doubleV\":1.0}").get("doubleV")), + () -> assertEquals(doubleValue(1d), constructFromObject("doubleV", 1d)) + ); } @Test public void constructString() { - assertEquals(stringValue("text"), tupleValue("{\"stringV\":\"text\"}").get("stringV")); - assertEquals(stringValue("text"), constructFromObject("stringV", "text")); + assertAll( + () -> assertEquals(stringValue("text"), tupleValue("{\"stringV\":\"text\"}").get("stringV")), + () -> assertEquals(stringValue("text"), constructFromObject("stringV", "text")) + ); } @Test public void constructBoolean() { - assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")); - assertEquals(booleanValue(true), constructFromObject("boolV", true)); - assertEquals(booleanValue(true), constructFromObject("boolV", "true")); - assertEquals(booleanValue(true), constructFromObject("boolV", 1)); - assertEquals(booleanValue(false), constructFromObject("boolV", 0)); + assertAll( + () -> assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", true)), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", "true")), + () -> assertEquals(booleanValue(true), constructFromObject("boolV", 1)), + () -> assertEquals(booleanValue(false), constructFromObject("boolV", 0)) + ); } @Test public void constructText() { - assertEquals(new OpenSearchExprTextValue("text"), - tupleValue("{\"textV\":\"text\"}").get("textV")); - assertEquals(new OpenSearchExprTextValue("text"), - constructFromObject("textV", "text")); - - assertEquals(new OpenSearchExprTextValue("text"), - tupleValue("{\"textKeywordV\":\"text\"}").get("textKeywordV")); - assertEquals(new OpenSearchExprTextValue("text"), - constructFromObject("textKeywordV", "text")); + assertAll( + () -> assertEquals(new OpenSearchExprTextValue("text"), + tupleValue("{\"textV\":\"text\"}").get("textV")), + () -> assertEquals(new OpenSearchExprTextValue("text"), + constructFromObject("textV", "text")), + + () -> assertEquals(new OpenSearchExprTextValue("text"), + tupleValue("{\"textKeywordV\":\"text\"}").get("textKeywordV")), + () -> assertEquals(new OpenSearchExprTextValue("text"), + constructFromObject("textKeywordV", "text")) + ); } @Test public void constructDates() { ExprValue dateStringV = constructFromObject("dateStringV", "1984-04-12"); - assertEquals(new ExprDateValue("1984-04-12"), dateStringV); - - assertEquals( - new ExprDateValue(LocalDate.ofInstant(Instant.ofEpochMilli(450576000000L), - UTC_ZONE_ID)), - constructFromObject("dateV", 450576000000L)); - - assertEquals( - new ExprDateValue("1984-04-12"), - constructFromObject("dateOrOrdinalDateV", "1984-103")); - assertEquals( - new ExprDateValue("2015-01-01"), - tupleValue("{\"dateV\":\"2015-01-01\"}").get("dateV")); + assertAll( + () -> assertEquals(new ExprDateValue("1984-04-12"), dateStringV), + () -> assertEquals(new ExprDateValue("1984-04-12"), + constructFromObject("dateOrOrdinalDateV", "1984-103")), + () -> assertEquals(new ExprDateValue("2015-01-01"), + tupleValue("{\"dateV\":\"2015-01-01\"}").get("dateV")) + ); } @Test public void constructTimes() { ExprValue timeStringV = constructFromObject("timeStringV","12:10:30.000Z"); - assertTrue(timeStringV.isDateTime()); - assertTrue(timeStringV instanceof ExprTimeValue); - assertEquals(new ExprTimeValue("12:10:30"), timeStringV); - - assertEquals( - new ExprTimeValue(LocalTime.from(Instant.ofEpochMilli(1420070400001L).atZone(UTC_ZONE_ID))), - constructFromObject("timeV", 1420070400001L)); - assertEquals( - new ExprTimeValue("09:07:42.000"), - constructFromObject("timeNoMillisOrTimeV", "09:07:42.000Z")); - assertEquals( - new ExprTimeValue("09:07:42"), - tupleValue("{\"timeV\":\"09:07:42\"}").get("timeV")); + assertAll( + () -> assertTrue(timeStringV.isDateTime()), + () -> assertTrue(timeStringV instanceof ExprTimeValue), + () -> assertEquals(new ExprTimeValue("12:10:30"), timeStringV), + () -> assertEquals(new ExprTimeValue("09:07:42.000"), + constructFromObject("timeNoMillisOrTimeV", "09:07:42.000Z")), + () -> assertEquals(new ExprTimeValue("09:07:42"), + tupleValue("{\"timeV\":\"09:07:42\"}").get("timeV")) + ); } @Test public void constructDatetime() { - assertEquals( - new ExprTimestampValue("2015-01-01 00:00:00"), - tupleValue("{\"timestampV\":\"2015-01-01\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01T12:10:30Z\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - tupleValue("{\"timestampV\":1420070400001}").get("timestampV")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("timestampV", 1420070400001L)); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("epochMillisV", "1420070400001")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("epochMillisV", 1420070400001L)); - assertEquals( - new ExprTimestampValue("2015-01-01 12:10:30"), - constructFromObject("timestampV", "2015-01-01 12:10:30")); - assertEquals( - new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("datetimeV", "2015-01-01 12:10:30")); - assertEquals( - new ExprDatetimeValue("2015-01-01 12:10:30"), - constructFromObject("datetimeDefaultV", "2015-01-01 12:10:30")); - assertEquals( - new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), - constructFromObject("dateOrEpochMillisV", "1420070400001")); - - // case: timestamp-formatted field, but it only gets a time: should match a time - assertEquals( - new ExprTimeValue("19:36:22"), - tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")); - - // case: timestamp-formatted field, but it only gets a date: should match a date - assertEquals( - new ExprDateValue("2011-03-03"), - tupleValue("{\"timestampV\":\"2011-03-03\"}").get("timestampV")); + assertAll( + () -> assertEquals( + new ExprTimestampValue("2015-01-01 00:00:00"), + tupleValue("{\"timestampV\":\"2015-01-01\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01T12:10:30Z\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("epochMillisV", "1420070400001")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("epochMillisV", 1420070400001L)), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochSecond(142704001L)), + constructFromObject("epochSecondV", 142704001L)), + () -> assertEquals( + new ExprTimeValue("10:20:30"), + tupleValue("{ \"timeCustomV\" : 102030 }").get("timeCustomV")), + () -> assertEquals( + new ExprDateValue("1961-04-12"), + tupleValue("{ \"dateCustomV\" : 19610412 }").get("dateCustomV")), + () -> assertEquals( + new ExprTimestampValue("1984-05-10 20:30:40"), + tupleValue("{ \"dateTimeCustomV\" : 19840510203040 }").get("dateTimeCustomV")), + () -> assertEquals( + new ExprTimestampValue("2015-01-01 12:10:30"), + constructFromObject("timestampV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("datetimeV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprDatetimeValue("2015-01-01 12:10:30"), + constructFromObject("datetimeDefaultV", "2015-01-01 12:10:30")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("dateOrEpochMillisV", "1420070400001")), + + // case: timestamp-formatted field, but it only gets a time: should match a time + () -> assertEquals( + new ExprTimeValue("19:36:22"), + tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")), + + // case: timestamp-formatted field, but it only gets a date: should match a date + () -> assertEquals( + new ExprDateValue("2011-03-03"), + tupleValue("{\"timestampV\":\"2011-03-03\"}").get("timestampV")) + ); } @Test @@ -387,7 +410,7 @@ public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { @Test public void constructDateFromIncompleteFormat() { assertEquals( - new ExprStringValue("1984"), + new ExprDateValue("1984-01-01"), constructFromObject("incompleteFormatV", "1984")); } From c086daddb20a8ea5f7ebe2e1e8a287b16dbfd196 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 7 Jul 2023 12:56:20 -0700 Subject: [PATCH 8/8] More fixes for god of fixes. Signed-off-by: Yury-Fridlyand --- .../data/type/OpenSearchDateType.java | 53 +++++++++---------- .../value/OpenSearchExprValueFactory.java | 20 ++++--- .../value/OpenSearchExprValueFactoryTest.java | 15 +++++- 3 files changed, 52 insertions(+), 36 deletions(-) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java index 86d95348f4..76947bf720 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -137,11 +137,11 @@ public class OpenSearchDateType extends OpenSearchDataType { private static final String CUSTOM_FORMAT_DATE_SYMBOLS = "FecEWwYqQgdMLDyuG"; @EqualsAndHashCode.Exclude - String formatString; + private final List formats; private OpenSearchDateType() { super(MappingType.Date); - this.formatString = ""; + this.formats = List.of(); } private OpenSearchDateType(ExprCoreType exprCoreType) { @@ -156,16 +156,20 @@ private OpenSearchDateType(ExprType exprType) { private OpenSearchDateType(String format) { super(MappingType.Date); - this.formatString = format; + this.formats = getFormatList(format); this.exprCoreType = getExprTypeFromFormatString(format); } + public boolean hasFormats() { + return !formats.isEmpty(); + } + /** * Retrieves and splits a user defined format string from the mapping into a list of formats. * @return A list of format names and user defined formats. */ - private List getFormatList() { - String format = strip8Prefix(formatString); + private List getFormatList(String format) { + format = strip8Prefix(format); return splitCombinedPatterns(format).stream().map(String::trim).collect(Collectors.toList()); } @@ -174,7 +178,7 @@ private List getFormatList() { * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. */ public List getAllNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> FormatNames.forName(formatString) != null) .map(DateFormatter::forPattern).collect(Collectors.toList()); } @@ -184,7 +188,7 @@ public List getAllNamedFormatters() { * @return a list of DateFormatters that can be used to parse a Date. */ public List getNumericNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); return namedFormat != null && SUPPORTED_NAMED_NUMERIC_FORMATS.contains(namedFormat); @@ -197,7 +201,7 @@ public List getNumericNamedFormatters() { * @return a list of formats as strings that can be used to parse a Date/Time/Timestamp. */ public List getAllCustomFormats() { - return getFormatList().stream() + return formats.stream() .filter(format -> FormatNames.forName(format) == null) .map(format -> { try { @@ -217,17 +221,8 @@ public List getAllCustomFormats() { * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. */ public List getAllCustomFormatters() { - return getFormatList().stream() - .filter(format -> FormatNames.forName(format) == null) - .map(format -> { - try { - return DateFormatter.forPattern(format); - } catch (Exception ignored) { - // parsing failed - return null; - } - }) - .filter(Objects::nonNull) + return getAllCustomFormats().stream() + .map(DateFormatter::forPattern) .collect(Collectors.toList()); } @@ -236,7 +231,7 @@ public List getAllCustomFormatters() { * @return a list of DateFormatters that can be used to parse a Date. */ public List getDateNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); return namedFormat != null && SUPPORTED_NAMED_DATE_FORMATS.contains(namedFormat); @@ -249,7 +244,7 @@ public List getDateNamedFormatters() { * @return a list of DateFormatters that can be used to parse a Time. */ public List getTimeNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); return namedFormat != null && SUPPORTED_NAMED_TIME_FORMATS.contains(namedFormat); @@ -262,7 +257,7 @@ public List getTimeNamedFormatters() { * @return a list of DateFormatters that can be used to parse a DateTime. */ public List getDateTimeNamedFormatters() { - return getFormatList().stream() + return formats.stream() .filter(formatString -> { FormatNames namedFormat = FormatNames.forName(formatString); return namedFormat != null && SUPPORTED_NAMED_DATETIME_FORMATS.contains(namedFormat); @@ -274,9 +269,9 @@ private ExprCoreType getExprTypeFromCustomFormats(List formats) { boolean isDate = false; boolean isTime = false; - for (var format : formats) { + for (String format : formats) { if (!isTime) { - for (var symbol : CUSTOM_FORMAT_TIME_SYMBOLS.toCharArray()) { + for (char symbol : CUSTOM_FORMAT_TIME_SYMBOLS.toCharArray()) { if (format.contains(String.valueOf(symbol))) { isTime = true; break; @@ -284,7 +279,7 @@ private ExprCoreType getExprTypeFromCustomFormats(List formats) { } } if (!isDate) { - for (var symbol : CUSTOM_FORMAT_DATE_SYMBOLS.toCharArray()) { + for (char symbol : CUSTOM_FORMAT_DATE_SYMBOLS.toCharArray()) { if (format.contains(String.valueOf(symbol))) { isDate = true; break; @@ -393,7 +388,7 @@ public static OpenSearchDateType of() { @Override public List getParent() { - return List.of(this.exprCoreType); + return List.of(exprCoreType); } @Override @@ -403,9 +398,9 @@ public boolean shouldCast(ExprType other) { @Override protected OpenSearchDataType cloneEmpty() { - if (this.formatString.isEmpty()) { - return OpenSearchDateType.of(this.exprCoreType); + if (formats.isEmpty()) { + return OpenSearchDateType.of(exprCoreType); } - return OpenSearchDateType.of(this.formatString); + return OpenSearchDateType.of(String.join(" || ", formats)); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index e79bd37e66..4b5db4ac51 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -243,6 +243,7 @@ private ExprValue parseTimestampString(String value, OpenSearchDateType dateType parsed = DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant(); return new ExprTimestampValue(parsed); } catch (DateTimeParseException ignored) { + // ignored } // otherwise, throw an error that no formatters worked @@ -279,6 +280,7 @@ private ExprValue parseTimeString(String value, OpenSearchDateType dateType) { return new ExprTimeValue( DateFormatters.from(STRICT_HOUR_MINUTE_SECOND_FORMATTER.parse(value)).toLocalTime()); } catch (DateTimeParseException ignored) { + // ignored } throw new IllegalArgumentException("Construct ExprTimeValue from \"" + value @@ -313,6 +315,7 @@ private ExprValue parseDateString(String value, OpenSearchDateType dateType) { return new ExprDateValue( DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate()); } catch (DateTimeParseException ignored) { + // ignored } throw new IllegalArgumentException("Construct ExprDateValue from \"" + value @@ -325,19 +328,24 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { if (value.isNumber()) { // isNumber var numFormatters = dt.getNumericNamedFormatters(); - if (numFormatters.size() > 0) { + if (numFormatters.size() > 0 || !dt.hasFormats()) { long epochMillis = 0; if (numFormatters.contains(DateFormatter.forPattern( - FormatNames.EPOCH_MILLIS.getSnakeCaseName()))) { + FormatNames.EPOCH_SECOND.getSnakeCaseName()))) { // no CamelCase for `EPOCH_*` formats - epochMillis = value.longValue(); - } else /* EPOCH_SECOND */ { epochMillis = value.longValue() * 1000; + } else /* EPOCH_MILLIS */ { + epochMillis = value.longValue(); + } + Instant instant = Instant.ofEpochMilli(epochMillis); + switch ((ExprCoreType) returnFormat) { + case TIME: return new ExprTimeValue(LocalTime.from(instant.atZone(UTC_ZONE_ID))); + case DATE: return new ExprDateValue(LocalDate.ofInstant(instant, UTC_ZONE_ID)); + default: return new ExprTimestampValue(instant); } - return new ExprTimestampValue(Instant.ofEpochMilli(epochMillis)); } else { // custom format - switch ((ExprCoreType) dt.getExprType()) { + switch ((ExprCoreType) returnFormat) { case TIME: return parseTimeString(value.stringValue(), dt); case DATE: return parseDateString(value.stringValue(), dt); default: return parseTimestampString(value.stringValue(), dt); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 1a9bbfdaf6..6b1b10dfa1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -35,11 +35,14 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.utils.DateTimeUtils.UTC_ZONE_ID; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -212,7 +215,8 @@ public void constructDouble() { @Test public void constructString() { assertAll( - () -> assertEquals(stringValue("text"), tupleValue("{\"stringV\":\"text\"}").get("stringV")), + () -> assertEquals(stringValue("text"), + tupleValue("{\"stringV\":\"text\"}").get("stringV")), () -> assertEquals(stringValue("text"), constructFromObject("stringV", "text")) ); } @@ -248,6 +252,9 @@ public void constructDates() { ExprValue dateStringV = constructFromObject("dateStringV", "1984-04-12"); assertAll( () -> assertEquals(new ExprDateValue("1984-04-12"), dateStringV), + () -> assertEquals(new ExprDateValue( + LocalDate.ofInstant(Instant.ofEpochMilli(450576000000L), UTC_ZONE_ID)), + constructFromObject("dateV", 450576000000L)), () -> assertEquals(new ExprDateValue("1984-04-12"), constructFromObject("dateOrOrdinalDateV", "1984-103")), () -> assertEquals(new ExprDateValue("2015-01-01"), @@ -262,6 +269,9 @@ public void constructTimes() { () -> assertTrue(timeStringV.isDateTime()), () -> assertTrue(timeStringV instanceof ExprTimeValue), () -> assertEquals(new ExprTimeValue("12:10:30"), timeStringV), + () -> assertEquals(new ExprTimeValue(LocalTime.from( + Instant.ofEpochMilli(1420070400001L).atZone(UTC_ZONE_ID))), + constructFromObject("timeV", 1420070400001L)), () -> assertEquals(new ExprTimeValue("09:07:42.000"), constructFromObject("timeNoMillisOrTimeV", "09:07:42.000Z")), () -> assertEquals(new ExprTimeValue("09:07:42"), @@ -284,6 +294,9 @@ public void constructDatetime() { () -> assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")), + () -> assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("timestampV", 1420070400001L)), () -> assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))),