diff --git a/docs/changelog/118603.yaml b/docs/changelog/118603.yaml new file mode 100644 index 0000000000000..d61619adfa5f6 --- /dev/null +++ b/docs/changelog/118603.yaml @@ -0,0 +1,6 @@ +pr: 118603 +summary: Allow DATE_PARSE to read the timezones +area: ES|QL +type: bug +issues: + - 117680 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 87963077e3b3d..7245488c7d7e5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -495,6 +495,60 @@ b:datetime null ; +evalDateParseWithTimezone +required_capability: date_parse_tz +row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s); + +s:keyword | d:datetime +12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z +; + +evalDateParseWithTimezoneCrossingDayBoundary +required_capability: date_parse_tz +row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s); + +s:keyword | d:datetime +12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z +; + +evalDateParseWithTimezone2 +required_capability: date_parse_tz +row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800" +| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2) +| eval eq = d1 == d2 +| keep d1, eq +; + +d1:datetime | eq:boolean +2022-07-12T01:24:10.000Z | true +; + +evalDateParseWithAndWithoutTimezone +required_capability: date_parse_tz +row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss" +| eval no_tz = date_parse(format, s) +| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900")) +| keep s, no_tz, with_tz +; + +s:keyword | no_tz:datetime | with_tz:datetime +2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z +; + +evalDateParseWithOtherTimezoneSpecifiers +required_capability: date_parse_tz +row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss" +| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900")) +| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09")) +| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900")) +| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9")) +| keep s, with_tz* +; + +s:keyword | with_tz1:datetime | with_tz2:datetime | with_tz3:datetime | with_tz4:datetime +2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z +; + evalDateParseDynamic from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no | eval birth_date_string = date_format("yyyy-MM-dd", birth_date) diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java index 3ea782931a0a3..6c432855e38fb 100644 --- a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java @@ -7,7 +7,6 @@ import java.lang.IllegalArgumentException; import java.lang.Override; import java.lang.String; -import java.time.ZoneId; import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; @@ -31,18 +30,15 @@ public final class DateParseEvaluator implements EvalOperator.ExpressionEvaluato private final EvalOperator.ExpressionEvaluator formatter; - private final ZoneId zoneId; - private final DriverContext driverContext; private Warnings warnings; public DateParseEvaluator(Source source, EvalOperator.ExpressionEvaluator val, - EvalOperator.ExpressionEvaluator formatter, ZoneId zoneId, DriverContext driverContext) { + EvalOperator.ExpressionEvaluator formatter, DriverContext driverContext) { this.source = source; this.val = val; this.formatter = formatter; - this.zoneId = zoneId; this.driverContext = driverContext; } @@ -91,7 +87,7 @@ public LongBlock eval(int positionCount, BytesRefBlock valBlock, BytesRefBlock f continue position; } try { - result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch), this.zoneId)); + result.appendLong(DateParse.process(valBlock.getBytesRef(valBlock.getFirstValueIndex(p), valScratch), formatterBlock.getBytesRef(formatterBlock.getFirstValueIndex(p), formatterScratch))); } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); @@ -108,7 +104,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector, BytesRef formatterScratch = new BytesRef(); position: for (int p = 0; p < positionCount; p++) { try { - result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch), this.zoneId)); + result.appendLong(DateParse.process(valVector.getBytesRef(p, valScratch), formatterVector.getBytesRef(p, formatterScratch))); } catch (IllegalArgumentException e) { warnings().registerException(e); result.appendNull(); @@ -120,7 +116,7 @@ public LongBlock eval(int positionCount, BytesRefVector valVector, @Override public String toString() { - return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]"; + return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; } @Override @@ -147,24 +143,21 @@ static class Factory implements EvalOperator.ExpressionEvaluator.Factory { private final EvalOperator.ExpressionEvaluator.Factory formatter; - private final ZoneId zoneId; - public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val, - EvalOperator.ExpressionEvaluator.Factory formatter, ZoneId zoneId) { + EvalOperator.ExpressionEvaluator.Factory formatter) { this.source = source; this.val = val; this.formatter = formatter; - this.zoneId = zoneId; } @Override public DateParseEvaluator get(DriverContext context) { - return new DateParseEvaluator(source, val.get(context), formatter.get(context), zoneId, context); + return new DateParseEvaluator(source, val.get(context), formatter.get(context), context); } @Override public String toString() { - return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + ", zoneId=" + zoneId + "]"; + return "DateParseEvaluator[" + "val=" + val + ", formatter=" + formatter + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 95bbe9520f4bf..c964bd5986d05 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -373,6 +373,11 @@ public enum Cap { */ DATE_NANOS_AGGREGATIONS(), + /** + * DATE_PARSE supports reading timezones + */ + DATE_PARSE_TZ(), + /** * Support for datetime in least and greatest functions */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index 1aaa227c3846e..e09fabab98d0f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -28,14 +28,12 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; -import java.time.ZoneId; import java.util.List; import static org.elasticsearch.common.time.DateFormatter.forPattern; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; -import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC; import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; @@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws } @Evaluator(warnExceptions = { IllegalArgumentException.class }) - static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException { - return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId)); + static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException { + return dateTimeToLong(val.utf8ToString(), toFormatter(formatter)); } @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - ZoneId zone = UTC; // TODO session timezone? ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field); if (format == null) { return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER); @@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } if (format.foldable()) { try { - DateFormatter formatter = toFormatter(format.fold(), zone); + DateFormatter formatter = toFormatter(format.fold()); return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } catch (IllegalArgumentException e) { throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage()); } } ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format); - return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone); + return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator); } - private static DateFormatter toFormatter(Object format, ZoneId zone) { - return forPattern(((BytesRef) format).utf8ToString()).withZone(zone); + private static DateFormatter toFormatter(Object format) { + return forPattern(((BytesRef) format).utf8ToString()); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java index 8da01fc1989ba..04683ecb65467 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import java.util.List; +import java.util.Locale; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -46,11 +47,26 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"), new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, equalTo(1683244800000L) ) ), + new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> { + long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z + int hours = randomIntBetween(0, 23); + String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours); + long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L; + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"), + new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second") + ), + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", + DataType.DATETIME, + equalTo(expected_ts) + ); + }), new TestCaseSupplier( "With Text", List.of(DataType.KEYWORD, DataType.TEXT), @@ -59,7 +75,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"), new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, equalTo(1683244800000L) ) @@ -72,7 +88,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"), new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, equalTo(1683244800000L) ) @@ -85,7 +101,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"), new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, equalTo(1683244800000L) ) @@ -98,7 +114,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, is(nullValue()) ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") @@ -118,7 +134,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second") ), - "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]", + "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]", DataType.DATETIME, is(nullValue()) ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")