From 3704dc47048f7c1636cdb25d8461489745691cb7 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 21 Jul 2023 14:23:50 -0700 Subject: [PATCH 1/4] Added support of timestamp/date/time using curly brackets (#297) * added bracketed time/date/timestamp input, tests, and documentation Signed-off-by: Matthew Wells --- docs/user/dql/expressions.rst | 11 ++- .../sql/sql/DateTimeFunctionIT.java | 99 +++++++++++++++++++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 4 +- .../common/antlr/SyntaxParserTestBase.java | 2 - .../sql/sql/antlr/BracketedTimestampTest.java | 41 ++++++++ .../sql/sql/antlr/SQLParserTest.java | 6 ++ 6 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 sql/src/test/java/org/opensearch/sql/sql/antlr/BracketedTimestampTest.java diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 275795707c..af264b2f16 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -25,7 +25,7 @@ A literal is a symbol that represents a value. The most common literal values in 1. Numeric literals: specify numeric values such as integer and floating-point numbers. 2. String literals: specify a string enclosed by single or double quotes. 3. Boolean literals: ``true`` or ``false``. -4. Date and Time literals: DATE 'YYYY-MM-DD' represent the date, TIME 'hh:mm:ss' represent the time, TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the timestamp. +4. Date and Time literals: DATE 'YYYY-MM-DD' represent the date, TIME 'hh:mm:ss' represent the time, TIMESTAMP 'YYYY-MM-DD hh:mm:ss' represent the timestamp. You can also surround the literals with curly brackets, if you do, you can replace date with d, time with t, and timestamp with ts Examples -------- @@ -49,6 +49,15 @@ Here is an example for different type of literals:: | Hello | Hello | It"s | It's | It's | "Its" | It's | It\'s | \I\t\s | +-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------+ + + os> SELECT {DATE '2020-07-07'}, {D '2020-07-07'}, {TIME '01:01:01'}, {T '01:01:01'}, {TIMESTAMP '2020-07-07 01:01:01'}, {TS '2020-07-07 01:01:01'} + fetched rows / total rows = 1/1 + +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ + | {DATE '2020-07-07'} | {D '2020-07-07'} | {TIME '01:01:01'} | {T '01:01:01'} | {TIMESTAMP '2020-07-07 01:01:01'} | {TS '2020-07-07 01:01:01'} | + |-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------| + | 2020-07-07 | 2020-07-07 | 01:01:01 | 01:01:01 | 2020-07-07 01:01:01 | 2020-07-07 01:01:01 | + +-----------------------+--------------------+---------------------+------------------+-------------------------------------+------------------------------+ + Limitations ----------- diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 91457296d6..4b9019a484 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -1288,4 +1288,103 @@ protected JSONObject executeQuery(String query) throws IOException { Response response = client().performRequest(request); return new JSONObject(getResponseBody(response)); } + + @Test + public void testTimestampBracket() throws IOException { + JSONObject result = executeQuery("select {timestamp '2020-09-16 17:30:00'}"); + verifySchema(result, schema("{timestamp '2020-09-16 17:30:00'}", null, "timestamp")); + verifyDataRows(result, rows("2020-09-16 17:30:00")); + + result = executeQuery("select {ts '2020-09-16 17:30:00'}"); + verifySchema(result, schema("{ts '2020-09-16 17:30:00'}", null, "timestamp")); + verifyDataRows(result, rows("2020-09-16 17:30:00")); + + result = executeQuery("select {timestamp '2020-09-16 17:30:00.123'}"); + verifySchema(result, schema("{timestamp '2020-09-16 17:30:00.123'}", null, "timestamp")); + verifyDataRows(result, rows("2020-09-16 17:30:00.123")); + + result = executeQuery("select {ts '2020-09-16 17:30:00.123'}"); + verifySchema(result, schema("{ts '2020-09-16 17:30:00.123'}", null, "timestamp")); + verifyDataRows(result, rows("2020-09-16 17:30:00.123")); + } + + @Test + public void testTimeBracket() throws IOException { + JSONObject result = executeQuery("select {time '17:30:00'}"); + verifySchema(result, schema("{time '17:30:00'}", null, "time")); + verifyDataRows(result, rows("17:30:00")); + + result = executeQuery("select {t '17:30:00'}"); + verifySchema(result, schema("{t '17:30:00'}", null, "time")); + verifyDataRows(result, rows("17:30:00")); + + result = executeQuery("select {time '17:30:00'}"); + verifySchema(result, schema("{time '17:30:00'}", null, "time")); + verifyDataRows(result, rows("17:30:00")); + + result = executeQuery("select {t '17:30:00'}"); + verifySchema(result, schema("{t '17:30:00'}", null, "time")); + verifyDataRows(result, rows("17:30:00")); + } + + @Test + public void testDateBracket() throws IOException { + JSONObject result = executeQuery("select {date '2020-09-16'}"); + verifySchema(result, schema("{date '2020-09-16'}", null, "date")); + verifyDataRows(result, rows("2020-09-16")); + + result = executeQuery("select {d '2020-09-16'}"); + verifySchema(result, schema("{d '2020-09-16'}", null, "date")); + verifyDataRows(result, rows("2020-09-16")); + } + + private void compareBrackets(String query1, String query2, String datetime) throws IOException { + JSONObject result1 = executeQuery("select " + query1 + " '" + datetime + "'"); + JSONObject result2 = executeQuery("select {" + query2 + " '" + datetime + "'}"); + + verifyDataRows(result1, rows(datetime)); + verifyDataRows(result2, rows(datetime)); + } + + @Test + public void testBracketedEquivalent() throws IOException { + compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00"); + compareBrackets("timestamp", "ts", "2020-09-16 17:30:00"); + compareBrackets("timestamp", "timestamp", "2020-09-16 17:30:00.123"); + compareBrackets("timestamp", "ts", "2020-09-16 17:30:00.123"); + compareBrackets("date", "date", "2020-09-16"); + compareBrackets("date", "d", "2020-09-16"); + compareBrackets("time", "time", "17:30:00"); + compareBrackets("time", "t", "17:30:00"); + } + + private void queryFails(String query) { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + boolean fails = false; + + try { + client().performRequest(request); + } catch(Exception ignored) { + fails = true; + } + + assertTrue(fails); + } + + @Test + public void testBracketFails() { + queryFails("select {time 'failure'}"); + queryFails("select {t 'failure'}"); + queryFails("select {date 'failure'}"); + queryFails("select {d 'failure'}"); + queryFails("select {timestamp 'failure'}"); + queryFails("select {ts 'failure'}"); + + } } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index e68edbbc58..2c3defb9f1 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -185,7 +185,6 @@ constant // Doesn't support the following types for now //| BIT_STRING //| NOT? nullLiteral=(NULL_LITERAL | NULL_SPEC_LITERAL) - //| LEFT_BRACE dateType=(D | T | TS | DATE | TIME | TIMESTAMP) stringLiteral RIGHT_BRACE ; decimalLiteral @@ -227,14 +226,17 @@ datetimeLiteral dateLiteral : DATE date=stringLiteral + | LEFT_BRACE (DATE | D) date=stringLiteral RIGHT_BRACE ; timeLiteral : TIME time=stringLiteral + | LEFT_BRACE (TIME | T) time=stringLiteral RIGHT_BRACE ; timestampLiteral : TIMESTAMP timestamp=stringLiteral + | LEFT_BRACE (TIMESTAMP | TS) timestamp=stringLiteral RIGHT_BRACE ; // Actually, these constants are shortcuts to the corresponding functions diff --git a/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java b/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java index 526dc4e816..63d7666c62 100644 --- a/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java +++ b/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java @@ -1,13 +1,11 @@ package org.opensearch.sql.common.antlr; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.sql.antlr.SQLSyntaxParser; /** * A base class for tests for SQL or PPL parser. diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/BracketedTimestampTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/BracketedTimestampTest.java new file mode 100644 index 0000000000..0f7a284aa7 --- /dev/null +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/BracketedTimestampTest.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.sql.antlr; + +import org.junit.jupiter.api.Test; + +public class BracketedTimestampTest extends SQLParserTest { + @Test + void date_shortened_test() { + acceptQuery("SELECT {d '2001-05-07'}"); + } + + @Test + void date_test() { + acceptQuery("SELECT {date '2001-05-07'}"); + } + + @Test + void time_shortened_test() { + acceptQuery("SELECT {t '10:11:12'}"); + } + + @Test + void time_test() { + acceptQuery("SELECT {time '10:11:12'}"); + } + + @Test + void timestamp_shortened_test() { + acceptQuery("SELECT {ts '2001-05-07 10:11:12'}"); + } + + @Test + void timestamp_test() { + acceptQuery("SELECT {timestamp '2001-05-07 10:11:12'}"); + } +} diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLParserTest.java index 7b8b415ee7..3f323725ab 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLParserTest.java @@ -1,3 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + package org.opensearch.sql.sql.antlr; import org.opensearch.sql.common.antlr.SyntaxParserTestBase; From 3a11f9d680d16a6d81c859de2d75ea0600264f19 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 21 Jul 2023 14:47:28 -0700 Subject: [PATCH 2/4] improved failing tests Signed-off-by: Matthew Wells --- .../sql/sql/DateTimeFunctionIT.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 4b9019a484..ec8f899a9a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -29,6 +29,7 @@ import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; @@ -1359,22 +1360,7 @@ public void testBracketedEquivalent() throws IOException { } private void queryFails(String query) { - Request request = new Request("POST", QUERY_API_ENDPOINT); - request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); - - RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); - restOptionsBuilder.addHeader("Content-Type", "application/json"); - request.setOptions(restOptionsBuilder); - - boolean fails = false; - - try { - client().performRequest(request); - } catch(Exception ignored) { - fails = true; - } - - assertTrue(fails); + assertThrows(ResponseException.class, ()->executeQuery(query)); } @Test @@ -1385,6 +1371,5 @@ public void testBracketFails() { queryFails("select {d 'failure'}"); queryFails("select {timestamp 'failure'}"); queryFails("select {ts 'failure'}"); - } } From a71a881aebca188225e90ddca37124c4efa5855c Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Fri, 21 Jul 2023 15:25:48 -0700 Subject: [PATCH 3/4] simplified tests for checking for failure Signed-off-by: Matthew Wells --- .../opensearch/sql/sql/DateTimeFunctionIT.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index ec8f899a9a..c503e68fef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -1358,18 +1358,14 @@ public void testBracketedEquivalent() throws IOException { compareBrackets("time", "time", "17:30:00"); compareBrackets("time", "t", "17:30:00"); } - - private void queryFails(String query) { - assertThrows(ResponseException.class, ()->executeQuery(query)); - } - + @Test public void testBracketFails() { - queryFails("select {time 'failure'}"); - queryFails("select {t 'failure'}"); - queryFails("select {date 'failure'}"); - queryFails("select {d 'failure'}"); - queryFails("select {timestamp 'failure'}"); - queryFails("select {ts 'failure'}"); + assertThrows(ResponseException.class, ()->executeQuery("select {time 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {t 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {date 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {d 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {timestamp 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {ts 'failure'}")); } } From 25a8c01fe8e237748b9b8823887f63b9cb765382 Mon Sep 17 00:00:00 2001 From: Matthew Wells Date: Mon, 24 Jul 2023 14:53:56 -0700 Subject: [PATCH 4/4] fixed redundant tests and improved tests that should fail Signed-off-by: Matthew Wells --- .../sql/sql/DateTimeFunctionIT.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index c503e68fef..2696a9a0d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -1319,13 +1319,13 @@ public void testTimeBracket() throws IOException { verifySchema(result, schema("{t '17:30:00'}", null, "time")); verifyDataRows(result, rows("17:30:00")); - result = executeQuery("select {time '17:30:00'}"); - verifySchema(result, schema("{time '17:30:00'}", null, "time")); - verifyDataRows(result, rows("17:30:00")); + result = executeQuery("select {time '17:30:00.123'}"); + verifySchema(result, schema("{time '17:30:00.123'}", null, "time")); + verifyDataRows(result, rows("17:30:00.123")); - result = executeQuery("select {t '17:30:00'}"); - verifySchema(result, schema("{t '17:30:00'}", null, "time")); - verifyDataRows(result, rows("17:30:00")); + result = executeQuery("select {t '17:30:00.123'}"); + verifySchema(result, schema("{t '17:30:00.123'}", null, "time")); + verifyDataRows(result, rows("17:30:00.123")); } @Test @@ -1361,11 +1361,13 @@ public void testBracketedEquivalent() throws IOException { @Test public void testBracketFails() { - assertThrows(ResponseException.class, ()->executeQuery("select {time 'failure'}")); - assertThrows(ResponseException.class, ()->executeQuery("select {t 'failure'}")); - assertThrows(ResponseException.class, ()->executeQuery("select {date 'failure'}")); - assertThrows(ResponseException.class, ()->executeQuery("select {d 'failure'}")); - assertThrows(ResponseException.class, ()->executeQuery("select {timestamp 'failure'}")); - assertThrows(ResponseException.class, ()->executeQuery("select {ts 'failure'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {time '2020-09-16'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {t '2020-09-16'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {date '17:30:00'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {d '17:30:00'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {timestamp '2020-09-16'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {ts '2020-09-16'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {timestamp '17:30:00'}")); + assertThrows(ResponseException.class, ()->executeQuery("select {ts '17:30:00'}")); } }